messageStore now uses log like rotation on files

This commit is contained in:
Dan Ballard 2018-09-20 14:01:46 -07:00
parent f217533044
commit 4e95427f44
5 changed files with 114 additions and 60 deletions

View File

@ -1,2 +0,0 @@
{"ciphertext":"SGVsbG8gdGhpcyBpcyBhIGZhaXJseSBhdmVyYWdlIGxlbmd0aCBtZXNzYWdlIHRoYXQgd2UgYXJlIHdyaXRpbmcgaGVyZS4="}
{"ciphertext":"SGVsbG8gdGhpcyBpcyBhIGZhaXJseSBhdmVyYWdlIGxlbmd0aCBtZXNzYWdlIHRoYXQgd2UgYXJlIHdyaXRpbmcgaGVyZS4="}

View File

@ -34,7 +34,10 @@ func (s *Server) Run(serverConfig *Config) {
af := application.ApplicationInstanceFactory{} af := application.ApplicationInstanceFactory{}
af.Init() af.Init()
ms := new(storage.MessageStore) ms := new(storage.MessageStore)
ms.Init("cwtch.messages", s.config.MaxBufferLines, s.metricsPack.MessageCounter) err = ms.Init(".", s.config.MaxBufferLines, s.metricsPack.MessageCounter)
if err != nil {
log.Fatal(err)
}
af.AddHandler("im.cwtch.server.listen", func(rai *application.ApplicationInstance) func() channels.Handler { af.AddHandler("im.cwtch.server.listen", func(rai *application.ApplicationInstance) func() channels.Handler {
return func() channels.Handler { return func() channels.Handler {
cslc := new(listen.CwtchServerListenChannel) cslc := new(listen.CwtchServerListenChannel)

View File

@ -15,8 +15,8 @@ func TestServerInstance(t *testing.T) {
ai := new(application.ApplicationInstance) ai := new(application.ApplicationInstance)
ra := new(application.RicochetApplication) ra := new(application.RicochetApplication)
msi := new(storage.MessageStore) msi := new(storage.MessageStore)
os.Remove("ms.test") os.RemoveAll("messages")
msi.Init("ms.test", 5, metrics.NewCounter()) msi.Init(".", 5, metrics.NewCounter())
gm := protocol.GroupMessage{ gm := protocol.GroupMessage{
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here."), Ciphertext: []byte("Hello this is a fairly average length message that we are writing here."),
Spamguard: []byte{}, Spamguard: []byte{},

View File

@ -11,6 +11,12 @@ import (
"sync" "sync"
) )
const (
fileStorePartitions = 10
fileStoreFilename = "cwtch.messages"
directory = "messages"
)
// MessageStoreInterface defines an interface to interact with a store of cwtch messages. // MessageStoreInterface defines an interface to interact with a store of cwtch messages.
type MessageStoreInterface interface { type MessageStoreInterface interface {
AddMessage(protocol.GroupMessage) AddMessage(protocol.GroupMessage)
@ -19,72 +25,116 @@ type MessageStoreInterface interface {
// MessageStore is a file-backed implementation of MessageStoreInterface // MessageStore is a file-backed implementation of MessageStoreInterface
type MessageStore struct { type MessageStore struct {
file *os.File activeLogFile *os.File
filePos int
storeDirectory string
lock sync.Mutex lock sync.Mutex
messages []*protocol.GroupMessage messages []*protocol.GroupMessage
messageCounter metrics.Counter messageCounter metrics.Counter
bufferSize int maxBufferLines int
pos int bufferPos int
rotated bool bufferRotated bool
} }
// Close closes the message store and underlying resources. // Close closes the message store and underlying resources.
func (ms *MessageStore) Close() { func (ms *MessageStore) Close() {
ms.lock.Lock() ms.lock.Lock()
ms.messages = nil ms.messages = nil
ms.file.Close() ms.activeLogFile.Close()
ms.lock.Unlock() ms.lock.Unlock()
} }
func (ms *MessageStore) updateBuffer(gm *protocol.GroupMessage) { func (ms *MessageStore) updateBuffer(gm *protocol.GroupMessage) {
ms.messages[ms.pos] = gm ms.messages[ms.bufferPos] = gm
ms.pos++ ms.bufferPos++
if ms.pos == ms.bufferSize { if ms.bufferPos == ms.maxBufferLines {
ms.pos = 0 ms.bufferPos = 0
ms.rotated = true ms.bufferRotated = true
} }
} }
// Init sets up a MessageStore of size bufferSize backed by filename func (ms *MessageStore) initAndLoadFiles() error {
func (ms *MessageStore) Init(filename string, bufferSize int, messageCounter metrics.Counter) { ms.activeLogFile = nil
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600) for i := fileStorePartitions - 1; i >= 0; i-- {
if err != nil { ms.filePos = 0
panic(err) filename := fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i)
} f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
ms.file = f if err != nil {
ms.pos = 0 log.Printf("Error: MessageStore could not open: %v: %v", filename, err)
ms.bufferSize = bufferSize continue
ms.messages = make([]*protocol.GroupMessage, bufferSize) }
ms.rotated = false ms.activeLogFile = f
ms.messageCounter = messageCounter
scanner := bufio.NewScanner(f) scanner := bufio.NewScanner(f)
for scanner.Scan() { for scanner.Scan() {
gms := scanner.Text() gms := scanner.Text()
gm := &protocol.GroupMessage{} ms.filePos++
err := json.Unmarshal([]byte(gms), gm) gm := &protocol.GroupMessage{}
if err == nil { err := json.Unmarshal([]byte(gms), gm)
ms.updateBuffer(gm) if err == nil {
} else { ms.updateBuffer(gm)
panic(err) }
} }
} }
if ms.activeLogFile == nil {
return fmt.Errorf("Could not create log file to write to in %s", ms.storeDirectory)
}
return nil
}
if err := scanner.Err(); err != nil { func (ms *MessageStore) updateFile(gm *protocol.GroupMessage) {
panic(err) s, err := json.Marshal(gm)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal group message %v\n", err)
}
fmt.Fprintf(ms.activeLogFile, "%s\n", s)
ms.filePos++
if ms.filePos >= ms.maxBufferLines/fileStorePartitions {
ms.rotateFileStore()
}
}
func (ms *MessageStore) rotateFileStore() {
ms.activeLogFile.Close()
os.Remove(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, fileStorePartitions-1))
for i := fileStorePartitions - 2; i >= 0; i-- {
os.Rename(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i), fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i+1))
} }
f, err := os.OpenFile(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, 0), os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
if err != nil {
log.Printf("ERROR: Could not open new message store file in: %s", ms.storeDirectory)
}
ms.filePos = 0
ms.activeLogFile = f
}
// Init sets up a MessageStore of size maxBufferLines (# of messages) backed by filename
func (ms *MessageStore) Init(appDirectory string, maxBufferLines int, messageCounter metrics.Counter) error {
ms.storeDirectory = appDirectory + "/" + directory + "/"
os.Mkdir(ms.storeDirectory, 0700)
ms.bufferPos = 0
ms.maxBufferLines = maxBufferLines
ms.messages = make([]*protocol.GroupMessage, maxBufferLines)
ms.bufferRotated = false
ms.messageCounter = messageCounter
err := ms.initAndLoadFiles()
return err
} }
// FetchMessages returns all messages from the backing file. // FetchMessages returns all messages from the backing file.
func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) { func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) {
ms.lock.Lock() ms.lock.Lock()
if !ms.rotated { if !ms.bufferRotated {
messages = make([]*protocol.GroupMessage, ms.pos) messages = make([]*protocol.GroupMessage, ms.bufferPos)
copy(messages, ms.messages[0:ms.pos]) copy(messages, ms.messages[0:ms.bufferPos])
} else { } else {
messages = make([]*protocol.GroupMessage, ms.bufferSize) messages = make([]*protocol.GroupMessage, ms.maxBufferLines)
copy(messages, ms.messages) copy(messages, ms.messages[ms.bufferPos:ms.maxBufferLines])
copy(messages[ms.bufferPos:], ms.messages[0:ms.bufferPos])
} }
ms.lock.Unlock() ms.lock.Unlock()
return return
@ -95,10 +145,7 @@ func (ms *MessageStore) AddMessage(gm protocol.GroupMessage) {
ms.messageCounter.Add(1) ms.messageCounter.Add(1)
ms.lock.Lock() ms.lock.Lock()
ms.updateBuffer(&gm) ms.updateBuffer(&gm)
s, err := json.Marshal(gm) ms.updateFile(&gm)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal group message %v\n", err)
}
fmt.Fprintf(ms.file, "%s\n", s)
ms.lock.Unlock() ms.lock.Unlock()
} }

View File

@ -12,27 +12,27 @@ func TestMessageStore(t *testing.T) {
os.Remove("ms.test") os.Remove("ms.test")
ms := new(MessageStore) ms := new(MessageStore)
counter := metrics.NewCounter() counter := metrics.NewCounter()
ms.Init("ms.test", 100000, counter) ms.Init("./", 1000, counter)
for i := 0; i < 50000; i++ { for i := 0; i < 499; i++ {
gm := protocol.GroupMessage{ gm := protocol.GroupMessage{
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)), Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
Spamguard: []byte{}, Spamguard: []byte{},
} }
ms.AddMessage(gm) ms.AddMessage(gm)
} }
if counter.Count() != 50000 { if counter.Count() != 499 {
t.Errorf("Counter should be at 50000 was %v", counter.Count()) t.Errorf("Counter should be at 499 was %v", counter.Count())
} }
ms.Close() ms.Close()
ms.Init("ms.test", 100000, counter) ms.Init("./", 1000, counter)
m := ms.FetchMessages() m := ms.FetchMessages()
if len(m) != 50000 { if len(m) != 499 {
t.Errorf("Should have been 50000 was %v", len(m)) t.Errorf("Should have been 499 was %v", len(m))
} }
counter.Reset() counter.Reset()
for i := 0; i < 100000; i++ { for i := 0; i < 1000; i++ {
gm := protocol.GroupMessage{ gm := protocol.GroupMessage{
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)), Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
Spamguard: []byte{}, Spamguard: []byte{},
@ -41,10 +41,16 @@ func TestMessageStore(t *testing.T) {
} }
m = ms.FetchMessages() m = ms.FetchMessages()
if len(m) != 100000 { if len(m) != 1000 {
t.Errorf("Should have been 100000 was %v", len(m)) t.Errorf("Should have been 1000 was %v", len(m))
} }
ms.Close() ms.Close()
os.Remove("ms.test") ms.Init("./", 1000, counter)
m = ms.FetchMessages()
if len(m) != 999 {
t.Errorf("Should have been 999 was %v", len(m))
}
ms.Close()
os.RemoveAll("./messages")
} }