forked from cwtch.im/cwtch
messageStore now uses log like rotation on files
This commit is contained in:
parent
f217533044
commit
4e95427f44
|
@ -1,2 +0,0 @@
|
||||||
{"ciphertext":"SGVsbG8gdGhpcyBpcyBhIGZhaXJseSBhdmVyYWdlIGxlbmd0aCBtZXNzYWdlIHRoYXQgd2UgYXJlIHdyaXRpbmcgaGVyZS4="}
|
|
||||||
{"ciphertext":"SGVsbG8gdGhpcyBpcyBhIGZhaXJseSBhdmVyYWdlIGxlbmd0aCBtZXNzYWdlIHRoYXQgd2UgYXJlIHdyaXRpbmcgaGVyZS4="}
|
|
|
@ -34,7 +34,10 @@ func (s *Server) Run(serverConfig *Config) {
|
||||||
af := application.ApplicationInstanceFactory{}
|
af := application.ApplicationInstanceFactory{}
|
||||||
af.Init()
|
af.Init()
|
||||||
ms := new(storage.MessageStore)
|
ms := new(storage.MessageStore)
|
||||||
ms.Init("cwtch.messages", s.config.MaxBufferLines, s.metricsPack.MessageCounter)
|
err = ms.Init(".", s.config.MaxBufferLines, s.metricsPack.MessageCounter)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
af.AddHandler("im.cwtch.server.listen", func(rai *application.ApplicationInstance) func() channels.Handler {
|
af.AddHandler("im.cwtch.server.listen", func(rai *application.ApplicationInstance) func() channels.Handler {
|
||||||
return func() channels.Handler {
|
return func() channels.Handler {
|
||||||
cslc := new(listen.CwtchServerListenChannel)
|
cslc := new(listen.CwtchServerListenChannel)
|
||||||
|
|
|
@ -15,8 +15,8 @@ func TestServerInstance(t *testing.T) {
|
||||||
ai := new(application.ApplicationInstance)
|
ai := new(application.ApplicationInstance)
|
||||||
ra := new(application.RicochetApplication)
|
ra := new(application.RicochetApplication)
|
||||||
msi := new(storage.MessageStore)
|
msi := new(storage.MessageStore)
|
||||||
os.Remove("ms.test")
|
os.RemoveAll("messages")
|
||||||
msi.Init("ms.test", 5, metrics.NewCounter())
|
msi.Init(".", 5, metrics.NewCounter())
|
||||||
gm := protocol.GroupMessage{
|
gm := protocol.GroupMessage{
|
||||||
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here."),
|
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here."),
|
||||||
Spamguard: []byte{},
|
Spamguard: []byte{},
|
||||||
|
|
|
@ -11,6 +11,12 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
fileStorePartitions = 10
|
||||||
|
fileStoreFilename = "cwtch.messages"
|
||||||
|
directory = "messages"
|
||||||
|
)
|
||||||
|
|
||||||
// MessageStoreInterface defines an interface to interact with a store of cwtch messages.
|
// MessageStoreInterface defines an interface to interact with a store of cwtch messages.
|
||||||
type MessageStoreInterface interface {
|
type MessageStoreInterface interface {
|
||||||
AddMessage(protocol.GroupMessage)
|
AddMessage(protocol.GroupMessage)
|
||||||
|
@ -19,72 +25,116 @@ type MessageStoreInterface interface {
|
||||||
|
|
||||||
// MessageStore is a file-backed implementation of MessageStoreInterface
|
// MessageStore is a file-backed implementation of MessageStoreInterface
|
||||||
type MessageStore struct {
|
type MessageStore struct {
|
||||||
file *os.File
|
activeLogFile *os.File
|
||||||
|
filePos int
|
||||||
|
storeDirectory string
|
||||||
lock sync.Mutex
|
lock sync.Mutex
|
||||||
messages []*protocol.GroupMessage
|
messages []*protocol.GroupMessage
|
||||||
messageCounter metrics.Counter
|
messageCounter metrics.Counter
|
||||||
bufferSize int
|
maxBufferLines int
|
||||||
pos int
|
bufferPos int
|
||||||
rotated bool
|
bufferRotated bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the message store and underlying resources.
|
// Close closes the message store and underlying resources.
|
||||||
func (ms *MessageStore) Close() {
|
func (ms *MessageStore) Close() {
|
||||||
ms.lock.Lock()
|
ms.lock.Lock()
|
||||||
ms.messages = nil
|
ms.messages = nil
|
||||||
ms.file.Close()
|
ms.activeLogFile.Close()
|
||||||
ms.lock.Unlock()
|
ms.lock.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ms *MessageStore) updateBuffer(gm *protocol.GroupMessage) {
|
func (ms *MessageStore) updateBuffer(gm *protocol.GroupMessage) {
|
||||||
ms.messages[ms.pos] = gm
|
ms.messages[ms.bufferPos] = gm
|
||||||
ms.pos++
|
ms.bufferPos++
|
||||||
if ms.pos == ms.bufferSize {
|
if ms.bufferPos == ms.maxBufferLines {
|
||||||
ms.pos = 0
|
ms.bufferPos = 0
|
||||||
ms.rotated = true
|
ms.bufferRotated = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Init sets up a MessageStore of size bufferSize backed by filename
|
func (ms *MessageStore) initAndLoadFiles() error {
|
||||||
func (ms *MessageStore) Init(filename string, bufferSize int, messageCounter metrics.Counter) {
|
ms.activeLogFile = nil
|
||||||
|
for i := fileStorePartitions - 1; i >= 0; i-- {
|
||||||
|
ms.filePos = 0
|
||||||
|
filename := fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i)
|
||||||
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
log.Printf("Error: MessageStore could not open: %v: %v", filename, err)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
ms.file = f
|
ms.activeLogFile = f
|
||||||
ms.pos = 0
|
|
||||||
ms.bufferSize = bufferSize
|
|
||||||
ms.messages = make([]*protocol.GroupMessage, bufferSize)
|
|
||||||
ms.rotated = false
|
|
||||||
ms.messageCounter = messageCounter
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(f)
|
scanner := bufio.NewScanner(f)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
gms := scanner.Text()
|
gms := scanner.Text()
|
||||||
|
ms.filePos++
|
||||||
gm := &protocol.GroupMessage{}
|
gm := &protocol.GroupMessage{}
|
||||||
err := json.Unmarshal([]byte(gms), gm)
|
err := json.Unmarshal([]byte(gms), gm)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
ms.updateBuffer(gm)
|
ms.updateBuffer(gm)
|
||||||
} else {
|
|
||||||
panic(err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if ms.activeLogFile == nil {
|
||||||
|
return fmt.Errorf("Could not create log file to write to in %s", ms.storeDirectory)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ms *MessageStore) updateFile(gm *protocol.GroupMessage) {
|
||||||
|
s, err := json.Marshal(gm)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[ERROR] Failed to unmarshal group message %v\n", err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(ms.activeLogFile, "%s\n", s)
|
||||||
|
ms.filePos++
|
||||||
|
if ms.filePos >= ms.maxBufferLines/fileStorePartitions {
|
||||||
|
ms.rotateFileStore()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ms *MessageStore) rotateFileStore() {
|
||||||
|
ms.activeLogFile.Close()
|
||||||
|
os.Remove(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, fileStorePartitions-1))
|
||||||
|
|
||||||
|
for i := fileStorePartitions - 2; i >= 0; i-- {
|
||||||
|
os.Rename(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i), fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, i+1))
|
||||||
|
}
|
||||||
|
|
||||||
if err := scanner.Err(); err != nil {
|
f, err := os.OpenFile(fmt.Sprintf("%s%s.%d", ms.storeDirectory, fileStoreFilename, 0), os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
|
||||||
panic(err)
|
if err != nil {
|
||||||
|
log.Printf("ERROR: Could not open new message store file in: %s", ms.storeDirectory)
|
||||||
}
|
}
|
||||||
|
ms.filePos = 0
|
||||||
|
ms.activeLogFile = f
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init sets up a MessageStore of size maxBufferLines (# of messages) backed by filename
|
||||||
|
func (ms *MessageStore) Init(appDirectory string, maxBufferLines int, messageCounter metrics.Counter) error {
|
||||||
|
ms.storeDirectory = appDirectory + "/" + directory + "/"
|
||||||
|
os.Mkdir(ms.storeDirectory, 0700)
|
||||||
|
|
||||||
|
ms.bufferPos = 0
|
||||||
|
ms.maxBufferLines = maxBufferLines
|
||||||
|
ms.messages = make([]*protocol.GroupMessage, maxBufferLines)
|
||||||
|
ms.bufferRotated = false
|
||||||
|
ms.messageCounter = messageCounter
|
||||||
|
|
||||||
|
err := ms.initAndLoadFiles()
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// FetchMessages returns all messages from the backing file.
|
// FetchMessages returns all messages from the backing file.
|
||||||
func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) {
|
func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) {
|
||||||
ms.lock.Lock()
|
ms.lock.Lock()
|
||||||
if !ms.rotated {
|
if !ms.bufferRotated {
|
||||||
messages = make([]*protocol.GroupMessage, ms.pos)
|
messages = make([]*protocol.GroupMessage, ms.bufferPos)
|
||||||
copy(messages, ms.messages[0:ms.pos])
|
copy(messages, ms.messages[0:ms.bufferPos])
|
||||||
} else {
|
} else {
|
||||||
messages = make([]*protocol.GroupMessage, ms.bufferSize)
|
messages = make([]*protocol.GroupMessage, ms.maxBufferLines)
|
||||||
copy(messages, ms.messages)
|
copy(messages, ms.messages[ms.bufferPos:ms.maxBufferLines])
|
||||||
|
copy(messages[ms.bufferPos:], ms.messages[0:ms.bufferPos])
|
||||||
}
|
}
|
||||||
ms.lock.Unlock()
|
ms.lock.Unlock()
|
||||||
return
|
return
|
||||||
|
@ -95,10 +145,7 @@ func (ms *MessageStore) AddMessage(gm protocol.GroupMessage) {
|
||||||
ms.messageCounter.Add(1)
|
ms.messageCounter.Add(1)
|
||||||
ms.lock.Lock()
|
ms.lock.Lock()
|
||||||
ms.updateBuffer(&gm)
|
ms.updateBuffer(&gm)
|
||||||
s, err := json.Marshal(gm)
|
ms.updateFile(&gm)
|
||||||
if err != nil {
|
|
||||||
log.Printf("[ERROR] Failed to unmarshal group message %v\n", err)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(ms.file, "%s\n", s)
|
|
||||||
ms.lock.Unlock()
|
ms.lock.Unlock()
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,27 +12,27 @@ func TestMessageStore(t *testing.T) {
|
||||||
os.Remove("ms.test")
|
os.Remove("ms.test")
|
||||||
ms := new(MessageStore)
|
ms := new(MessageStore)
|
||||||
counter := metrics.NewCounter()
|
counter := metrics.NewCounter()
|
||||||
ms.Init("ms.test", 100000, counter)
|
ms.Init("./", 1000, counter)
|
||||||
for i := 0; i < 50000; i++ {
|
for i := 0; i < 499; i++ {
|
||||||
gm := protocol.GroupMessage{
|
gm := protocol.GroupMessage{
|
||||||
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
|
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
|
||||||
Spamguard: []byte{},
|
Spamguard: []byte{},
|
||||||
}
|
}
|
||||||
ms.AddMessage(gm)
|
ms.AddMessage(gm)
|
||||||
}
|
}
|
||||||
if counter.Count() != 50000 {
|
if counter.Count() != 499 {
|
||||||
t.Errorf("Counter should be at 50000 was %v", counter.Count())
|
t.Errorf("Counter should be at 499 was %v", counter.Count())
|
||||||
}
|
}
|
||||||
ms.Close()
|
ms.Close()
|
||||||
ms.Init("ms.test", 100000, counter)
|
ms.Init("./", 1000, counter)
|
||||||
m := ms.FetchMessages()
|
m := ms.FetchMessages()
|
||||||
if len(m) != 50000 {
|
if len(m) != 499 {
|
||||||
t.Errorf("Should have been 50000 was %v", len(m))
|
t.Errorf("Should have been 499 was %v", len(m))
|
||||||
}
|
}
|
||||||
|
|
||||||
counter.Reset()
|
counter.Reset()
|
||||||
|
|
||||||
for i := 0; i < 100000; i++ {
|
for i := 0; i < 1000; i++ {
|
||||||
gm := protocol.GroupMessage{
|
gm := protocol.GroupMessage{
|
||||||
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
|
Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
|
||||||
Spamguard: []byte{},
|
Spamguard: []byte{},
|
||||||
|
@ -41,10 +41,16 @@ func TestMessageStore(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
m = ms.FetchMessages()
|
m = ms.FetchMessages()
|
||||||
if len(m) != 100000 {
|
if len(m) != 1000 {
|
||||||
t.Errorf("Should have been 100000 was %v", len(m))
|
t.Errorf("Should have been 1000 was %v", len(m))
|
||||||
}
|
}
|
||||||
|
|
||||||
ms.Close()
|
ms.Close()
|
||||||
os.Remove("ms.test")
|
ms.Init("./", 1000, counter)
|
||||||
|
m = ms.FetchMessages()
|
||||||
|
if len(m) != 999 {
|
||||||
|
t.Errorf("Should have been 999 was %v", len(m))
|
||||||
|
}
|
||||||
|
ms.Close()
|
||||||
|
|
||||||
|
os.RemoveAll("./messages")
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue