Merge pull request 'Storage v1 massively increase storage capacity' (#371) from dan/cwtch:storageEmbiggen into master
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
Reviewed-on: #371 Reviewed-by: Sarah Jamie Lewis <sarah@openprivacy.ca>
This commit is contained in:
commit
149f64b030
|
@ -30,8 +30,8 @@ type Message struct {
|
||||||
Flags uint64
|
Flags uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// MessageBaseSize is a rough estimate of the base number of bytes the struct uses before strings are populated
|
// MessageBaseSize 2021.06 byte size of an *empty* message json serialized
|
||||||
const MessageBaseSize = 104
|
const MessageBaseSize float64 = 463
|
||||||
|
|
||||||
func compareSignatures(a []byte, b []byte) bool {
|
func compareSignatures(a []byte, b []byte) bool {
|
||||||
if len(a) != len(b) {
|
if len(a) != len(b) {
|
||||||
|
|
|
@ -114,7 +114,7 @@ func (ss *streamStore) initBuffer() {
|
||||||
|
|
||||||
func (ss *streamStore) updateBuffer(m model.Message) {
|
func (ss *streamStore) updateBuffer(m model.Message) {
|
||||||
ss.messages = append(ss.messages, m)
|
ss.messages = append(ss.messages, m)
|
||||||
ss.bufferByteCount += (model.MessageBaseSize * 1.5) + len(m.Message)
|
ss.bufferByteCount += (104 * 1.5) + len(m.Message)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ss *streamStore) updateFile() error {
|
func (ss *streamStore) updateFile() error {
|
||||||
|
|
|
@ -6,14 +6,19 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.openprivacy.ca/openprivacy/log"
|
"git.openprivacy.ca/openprivacy/log"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// This number is larger that the recommend chunk size of libsodium secretbox by an order of magnitude.
|
||||||
|
// Since this code is not performance-sensitive (and is unlikely to gain any significant performance benefit from
|
||||||
|
// cache-efficient chunking) this size isn’t currently a concern.
|
||||||
|
// TODO: revise and evaluate better storage options after beta”
|
||||||
const (
|
const (
|
||||||
fileStorePartitions = 16
|
fileStorePartitions = 128
|
||||||
bytesPerFile = 15 * 1024
|
bytesPerFile = 128 * 1024
|
||||||
)
|
)
|
||||||
|
|
||||||
// streamStore is a file-backed implementation of StreamStore using an in memory buffer of ~16KB and a rotating set of files
|
// streamStore is a file-backed implementation of StreamStore using an in memory buffer of ~16KB and a rotating set of files
|
||||||
|
@ -73,7 +78,7 @@ func (ss *streamStore) initBufferFromStorage() error {
|
||||||
|
|
||||||
func (ss *streamStore) updateBuffer(m model.Message) {
|
func (ss *streamStore) updateBuffer(m model.Message) {
|
||||||
ss.messages = append(ss.messages, m)
|
ss.messages = append(ss.messages, m)
|
||||||
ss.bufferByteCount += (model.MessageBaseSize * 1.5) + len(m.Message)
|
ss.bufferByteCount += int(math.Ceil(model.MessageBaseSize*1.5)) + len(m.Message)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ss *streamStore) updateFile() error {
|
func (ss *streamStore) updateFile() error {
|
||||||
|
|
Loading…
Reference in New Issue