Merge pull request 'dep upgrades; server app wait for ACN' (#34) from networkAfterOnline into trunk

Reviewed-on: #34

.drone.yml

@@ -5,7 +5,7 @@ name: linux-test
 
 steps:
   - name: fetch
-    image: golang
+    image: golang:1.17.5
     volumes:
       - name: deps
         path: /go
@@ -18,7 +18,7 @@ steps:
       - echo `date +%G-%m-%d-%H-%M` > BUILDDATE
 
   - name: quality
-    image: golang
+    image: golang:1.17.5
     volumes:
       - name: deps
         path: /go
@@ -27,7 +27,7 @@ steps:
       - staticcheck ./...
 
   - name: units-tests
-    image: golang
+    image: golang:1.17.5
     volumes:
       - name: deps
         path: /go
@@ -35,7 +35,7 @@ steps:
       - sh testing/tests.sh
 
   - name: test-builda-app
-    image: golang
+    image: golang:1.17.5
     volumes:
       - name: deps
         path: /go
@@ -43,17 +43,8 @@ steps:
       - cd app
       - go build
 
-  - name: notify-email
-    image: drillster/drone-email
-    settings:
-      host: build.openprivacy.ca
-      port: 25
-      skip_verify: true
-      from: drone@openprivacy.ca
-    when:
-      status: [failure]
-
   - name: notify-gogs
+    pull: if-not-exists
     image: openpriv/drone-gogs
     when:
       event: pull_request

LICENSE

@@ -0,0 +1,9 @@
+MIT License
+Copyright (c) 2021 Open Privacy Research Society
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

README.md

@@ -19,16 +19,18 @@ The server package relies on sqlite which in turn requires the use of CGO. As pe
 The app takes the following arguments
 - -debug: enabled debug logging
 - -exportServerBundle: Export the server bundle to a file called serverbundle
-
+- -disableMetrics: Disable metrics reporting to serverMonitor.txt and associated tracking routines
+- -dir [directory]: specify a directory to store server files (default is current directory) 
 
 The app takes the following environment variables
 - CWTCH_HOME: sets the config dir for the app
+- DISABLE_METRICS: if set to any value ('1') it disables metrics reporting to serverMonitor.txt and associated tracking routines 
 
 `env CONFIG_HOME=./conf ./app`
 
 ## Using the Server
 
-When run the app will output standard log lines, one of which will contain the `tofubundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
+When run the app will output standard log lines, one of which will contain the `serverbundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
 
 ## Docker
 
@@ -38,4 +40,10 @@ or run our prebuild ones with
 
 `pull openpriv/cwtch-server`
 
-and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch`
+and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch` so if you want the server data to persist you would run
+
+`docker run -v /var/lib/cwtch/server01:/var/lib/cwtch openpriv/cwtch-server`
+
+to create a persistent container you might try a command like:
+
+`docker run --name cwtch -v /var/lib/cwtch/server01:/var/lib/cwtch --restart always openpriv/cwtch-server`

app/main.go

@@ -8,6 +8,7 @@ import (
 	"git.openprivacy.ca/cwtch.im/tapir/primitives"
 	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"git.openprivacy.ca/openprivacy/log"
+	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
 	"io/ioutil"
 	mrand "math/rand"
 	"os"
@@ -20,6 +21,8 @@ import (
 func main() {
 	flagDebug := flag.Bool("debug", false, "Enable debug logging")
 	flagExportServer := flag.Bool("exportServerBundle", false, "Export the server bundle to a file called serverbundle")
+	flagDir := flag.String("dir", ".", "Directory to store server files in (config, encrypted messages, metrics)")
+	flagDisableMetrics := flag.Bool("disableMetrics", false, "Disable metrics reporting")
 	flag.Parse()
 
 	log.AddEverythingFromPattern("server/app/main")
@@ -31,7 +34,9 @@ func main() {
 		log.SetLevel(log.LevelDebug)
 	}
 	configDir := os.Getenv("CWTCH_HOME")
-
+	if configDir == "" {
+		configDir = *flagDir
+	}
 	if len(os.Args) == 2 && os.Args[1] == "gen1" {
 		config := new(cwtchserver.Config)
 		id, pk := primitives.InitializeEphemeralIdentity()
@@ -42,9 +47,7 @@ func main() {
 		config.TokenServerPublicKey = tid.PublicKey()
 		config.MaxBufferLines = 100000
 		config.ServerReporting = cwtchserver.Reporting{
-			LogMetricsToFile:    true,
-			ReportingGroupID:    "",
-			ReportingServerAddr: "",
+			LogMetricsToFile: true,
 		}
 		config.ConfigDir = "."
 		config.FilePath = cwtchserver.ServerConfigFile
@@ -53,11 +56,16 @@ func main() {
 		return
 	}
 
-	serverConfig, err := cwtchserver.LoadCreateDefaultConfigFile(configDir, cwtchserver.ServerConfigFile, false, "")
+	disableMetrics := *flagDisableMetrics
+	if os.Getenv("DISABLE_METRICS") != "" {
+		disableMetrics = true
+	}
+	serverConfig, err := cwtchserver.LoadCreateDefaultConfigFile(configDir, cwtchserver.ServerConfigFile, false, "", !disableMetrics)
 	if err != nil {
 		log.Errorf("Could not load/create config file: %s\n", err)
 		return
 	}
+	serverConfig.ServerReporting.LogMetricsToFile = !disableMetrics
 	// we don't need real randomness for the port, just to avoid a possible conflict...
 	mrand.Seed(int64(time.Now().Nanosecond()))
 	controlPort := mrand.Intn(1000) + 9052
@@ -70,9 +78,8 @@ func main() {
 	}
 
 	os.MkdirAll("tordir/tor", 0700)
-	tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).Build("./tordir/tor/torrc")
-	acn, err := tor.NewTorACNWithAuth("tordir", "", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
-
+	tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).WithSocksPort(controlPort + 1).Build("./tordir/tor/torrc")
+	acn, err := tor.NewTorACNWithAuth("tordir", "", "tordir/tor", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
 	if err != nil {
 		log.Errorf("\nError connecting to Tor: %v\n", err)
 		os.Exit(1)
@@ -87,7 +94,7 @@ func main() {
 
 	if *flagExportServer {
 		// Todo: change all to server export
-		ioutil.WriteFile(path.Join(serverConfig.ConfigDir, "serverbundle"), []byte(server.TofuBundle()), 0600)
+		ioutil.WriteFile(path.Join(serverConfig.ConfigDir, "serverbundle"), []byte(server.ServerBundle()), 0600)
 	}
 
 	// Graceful Stop
@@ -100,8 +107,27 @@ func main() {
 		os.Exit(1)
 	}()
 
-	server.Run(acn)
+	running := false
+	lastStatus := -2
 	for {
+		status, msg := acn.GetBootstrapStatus()
+		if status == 100 && !running {
+			log.Infoln("ACN is online, Running Server")
+			server.Run(acn)
+			running = true
+		}
+		if status != 100 {
+			if running {
+				log.Infoln("ACN is offline, Stopping Server")
+				server.Stop()
+				running = false
+			} else {
+				if lastStatus != status {
+					log.Infof("ACN booting... Status %v%%: %v\n", status, msg)
+					lastStatus = status
+				}
+			}
+		}
 		time.Sleep(time.Second)
 	}
 }

go.mod

@@ -3,12 +3,11 @@ module git.openprivacy.ca/cwtch.im/server
 go 1.14
 
 require (
-	cwtch.im/cwtch v0.12.2
-	git.openprivacy.ca/cwtch.im/tapir v0.4.9
-	git.openprivacy.ca/openprivacy/connectivity v1.5.0
+	cwtch.im/cwtch v0.18.0
+	git.openprivacy.ca/cwtch.im/tapir v0.5.5
+	git.openprivacy.ca/openprivacy/connectivity v1.8.6
 	git.openprivacy.ca/openprivacy/log v1.0.3
-	github.com/gtank/ristretto255 v0.1.2
+	github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c
 	github.com/mattn/go-sqlite3 v1.14.7
-	github.com/struCoder/pidusage v0.2.1
-	golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
-)
+	golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d
+)

go.sum

@@ -1,114 +1,168 @@
-cwtch.im/cwtch v0.7.3 h1:3f2Q2XFgNtq8hVbEz7K++exS1gDsO+zmSO7X7Fqz8Mo=
-cwtch.im/cwtch v0.7.3/go.mod h1:S0JRLSTwFM+kRrpOPKgUQn/loKe/fc6lLDnOFopRKq8=
-cwtch.im/cwtch v0.8.0 h1:QDRaDBTXefFRPZPqUMtxoNhOcgXv0rl0bGjysSOmJX0=
-cwtch.im/cwtch v0.8.0/go.mod h1:+SY/4ueF1U7mK+CX8hZFbtd+GC1lx/cReo110KgtQAw=
-cwtch.im/cwtch v0.8.5 h1:W67jAF2oRwqWytbZEv1UeCqW0cU2x69tgUw8iy27xFA=
-cwtch.im/cwtch v0.8.5/go.mod h1:5GHxaaeVnKeXSU64IvtCKzkqhU8DRiLoVM+tiBT8kkc=
-cwtch.im/cwtch v0.12.2 h1:I+ndKadCRCITw4SPbd+1cpRv+z/7iHjjTUv8OzRwTrE=
-cwtch.im/cwtch v0.12.2/go.mod h1:QpTkQK7MqNt0dQK9/pBk5VpkvFhy6xuoxJIn401B8fM=
-filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU=
+cwtch.im/cwtch v0.18.0 h1:7Gz9FwoxMEFKvBzSnf/TLd7PkOgHsIq2IvFPHWAvBaU=
+cwtch.im/cwtch v0.18.0/go.mod h1:StheazFFY7PKqBbEyDVLhzWW6WOat41zV0ckC240c5Y=
 filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
-git.openprivacy.ca/cwtch.im/tapir v0.4.0 h1:clG8uORt0NKEhT4P+Dpw1pzyUuYzYBMevGqn2pciKk8=
-git.openprivacy.ca/cwtch.im/tapir v0.4.0/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
-git.openprivacy.ca/cwtch.im/tapir v0.4.1 h1:9LMpQX41IzecNNlRc1FZKXHg6wlFss679tFsa3vzb3Y=
-git.openprivacy.ca/cwtch.im/tapir v0.4.1/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
-git.openprivacy.ca/cwtch.im/tapir v0.4.2 h1:bxMWZnVJXX4dqqOFS7ELW4iFkVL4GS8wiRkjRv5rJe8=
-git.openprivacy.ca/cwtch.im/tapir v0.4.2/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
-git.openprivacy.ca/cwtch.im/tapir v0.4.4 h1:KyuTVmr9GYptTCeR7JDODjmhBBbnIBf9V3NSC4+6bHc=
-git.openprivacy.ca/cwtch.im/tapir v0.4.4/go.mod h1:qMFTdmDZITc1BLP1jSW0gVpLmvpg+Zjsh5ek8StwbFE=
-git.openprivacy.ca/cwtch.im/tapir v0.4.9 h1:LXonlztwvI1F1++0IyomIcDH1/Bxzo+oN8YjGonNvjM=
-git.openprivacy.ca/cwtch.im/tapir v0.4.9/go.mod h1:p4bHo3DAO8wwimU6JAeZXbfPQ4jnoA2bV+4YvknWTNQ=
+filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
+filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
+git.openprivacy.ca/cwtch.im/tapir v0.5.5 h1:km6UDrLYH/GCEn2s+S299/TiRHhxKCIAipYr9GbG3Hk=
+git.openprivacy.ca/cwtch.im/tapir v0.5.5/go.mod h1:bWWHrDYBtHvxMri59RwIB/w7Eg1aC0BrQ/ycKlnbB5k=
 git.openprivacy.ca/openprivacy/bine v0.0.4 h1:CO7EkGyz+jegZ4ap8g5NWRuDHA/56KKvGySR6OBPW+c=
 git.openprivacy.ca/openprivacy/bine v0.0.4/go.mod h1:13ZqhKyqakDsN/ZkQkIGNULsmLyqtXc46XBcnuXm/mU=
-git.openprivacy.ca/openprivacy/connectivity v1.4.3 h1:i2Ad/U9FlL9dKr2bhRck7lJ8NoWyGtoEfUwoCyMT0fU=
-git.openprivacy.ca/openprivacy/connectivity v1.4.3/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
-git.openprivacy.ca/openprivacy/connectivity v1.4.5 h1:UYMdCWPzEAP7LbqdMXGNXmfKjWlvfnKdmewBtnbgQRI=
-git.openprivacy.ca/openprivacy/connectivity v1.4.5/go.mod h1:JVRCIdL+lAG6ohBFWiKeC/MN42nnC0sfFszR9XG6vPQ=
-git.openprivacy.ca/openprivacy/connectivity v1.5.0 h1:ZxsR/ZaVKXIkD2x6FlajZn62ciNQjamrI4i/5xIpdoQ=
-git.openprivacy.ca/openprivacy/connectivity v1.5.0/go.mod h1:UjQiGBnWbotmBzIw59B8H6efwDadjkKzm3RPT1UaIRw=
-git.openprivacy.ca/openprivacy/log v1.0.1/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
-git.openprivacy.ca/openprivacy/log v1.0.2 h1:HLP4wsw4ljczFAelYnbObIs821z+jgMPCe8uODPnGQM=
-git.openprivacy.ca/openprivacy/log v1.0.2/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+git.openprivacy.ca/openprivacy/connectivity v1.8.6 h1:g74PyDGvpMZ3+K0dXy3mlTJh+e0rcwNk0XF8owzkmOA=
+git.openprivacy.ca/openprivacy/connectivity v1.8.6/go.mod h1:Hn1gpOx/bRZp5wvCtPQVJPXrfeUH0EGiG/Aoa0vjGLg=
 git.openprivacy.ca/openprivacy/log v1.0.3 h1:E/PMm4LY+Q9s3aDpfySfEDq/vYQontlvNj/scrPaga0=
 git.openprivacy.ca/openprivacy/log v1.0.3/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is=
 github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
-github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc=
-github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
+github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c h1:gkfmnY4Rlt3VINCo4uKdpvngiibQyoENVj5Q88sxXhE=
+github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c/go.mod h1:tDPFhGdt3hJWqtKwx57i9baiB1Cj0yAg22VOPUqm5vY=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
 github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0=
 github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk=
+github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b/go.mod h1:xxLb2ip6sSUts3g1irPVHyk/DGslwQsNOo9I7smJfNU=
+github.com/mutecomm/go-sqlcipher/v4 v4.4.2/go.mod h1:mF2UmIpBnzFeBdu/ypTDb/LdbS0nk0dfSN1WUsWTjMA=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY=
+github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q=
+github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/struCoder/pidusage v0.1.3 h1:pZcSa6asBE38TJtW0Nui6GeCjLTpaT/jAnNP7dUTLSQ=
-github.com/struCoder/pidusage v0.1.3/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI=
-github.com/struCoder/pidusage v0.2.1 h1:dFiEgUDkubeIj0XA1NpQ6+8LQmKrLi7NiIQl86E6BoY=
-github.com/struCoder/pidusage v0.2.1/go.mod h1:bewtP2KUA1TBUyza5+/PCpSQ6sc/H6jJbIKAzqW86BA=
-github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
-go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI=
 golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d h1:3qF+Z8Hkrw9sOhrFHti9TlB1Hkac1x+DNRkv0XQiFjo=
+golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
+golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/metrics.go

@@ -9,9 +9,13 @@ import (
 	"time"
 )
 
+// Fields must be in this order because go compiler has problems with 64bit fields on 32 bit arches (arm32 raspberry pi):
+//   https://git.openprivacy.ca/cwtch.im/server/pulls/30
+//   https://github.com/golang/go/issues/599
+//   https://github.com/census-instrumentation/opencensus-go/issues/587
 type counter struct {
-	startTime time.Time
 	count     uint64
+	startTime time.Time
 }
 
 // Counter providers a threadsafe counter to use for storing long running counts
@@ -71,6 +75,7 @@ const (
 	Average
 )
 
+// TODO port to SQLite for persistence between runs?
 type monitorHistory struct {
 	monitorType         MonitorType
 	monitorAccumulation MonitorAccumulation
@@ -150,13 +155,25 @@ func (mh *monitorHistory) Months() []float64 {
 	return mh.returnCopy(mh.perMonthForYear[:])
 }
 
+const timeDay = time.Hour * 24
+const timeWeek = timeDay * 7
+const timeMonth = timeDay * 28
+
 func (mh *monitorHistory) Report(w *bufio.Writer) {
 	mh.lock.Lock()
 	fmt.Fprintln(w, "Minutes:", reportLine(mh.monitorType, mh.perMinutePerHour[:]))
-	fmt.Fprintln(w, "Hours:  ", reportLine(mh.monitorType, mh.perHourForDay[:]))
-	fmt.Fprintln(w, "Days:   ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
-	fmt.Fprintln(w, "Weeks:  ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
-	fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
+	if time.Since(mh.starttime) >= time.Hour {
+		fmt.Fprintln(w, "Hours:  ", reportLine(mh.monitorType, mh.perHourForDay[:]))
+	}
+	if time.Since(mh.starttime) >= timeDay {
+		fmt.Fprintln(w, "Days:   ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
+	}
+	if time.Since(mh.starttime) >= timeWeek {
+		fmt.Fprintln(w, "Weeks:  ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
+	}
+	if time.Since(mh.starttime) >= timeMonth {
+		fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
+	}
 	mh.lock.Unlock()
 }
 
@@ -215,10 +232,10 @@ func (mh *monitorHistory) monitorThread() {
 		case <-time.After(time.Minute):
 			mh.lock.Lock()
 
-			minuteAvg := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
+			minuteAcc := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
 
 			if time.Since(mh.timeLastHourRotate) > time.Hour {
-				rotateAndAccumulate(mh.perHourForDay[:], minuteAvg, mh.monitorAccumulation)
+				rotateAndAccumulate(mh.perHourForDay[:], minuteAcc, mh.monitorAccumulation)
 				mh.timeLastHourRotate = time.Now()
 			}
 

metrics/monitors.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"git.openprivacy.ca/cwtch.im/tapir"
 	"git.openprivacy.ca/openprivacy/log"
-	"github.com/struCoder/pidusage"
 	"os"
 	"path"
 	"runtime"
@@ -17,70 +16,48 @@ const (
 	reportFile = "serverMonitorReport.txt"
 )
 
+type MessageCountFn func() int
+
 // Monitors is a package of metrics for a Cwtch Server including message count, CPU, Mem, and conns
 type Monitors struct {
-	MessageCounter      Counter
-	TotalMessageCounter Counter
-	Messages            MonitorHistory
-	CPU                 MonitorHistory
-	Memory              MonitorHistory
-	ClientConns         MonitorHistory
-	starttime           time.Time
-	breakChannel        chan bool
-	log                 bool
-	configDir           string
+	MessageCounter Counter
+	Messages       MonitorHistory
+	Memory         MonitorHistory
+	ClientConns    MonitorHistory
+	messageCountFn MessageCountFn
+	starttime      time.Time
+	breakChannel   chan bool
+	log            bool
+	configDir      string
+	running        bool
+	lock           sync.Mutex
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
 }
 
 // Start initializes a Monitors's monitors
-func (mp *Monitors) Start(ts tapir.Service, configDir string, doLogging bool) {
+func (mp *Monitors) Start(ts tapir.Service, mcfn MessageCountFn, configDir string, doLogging bool) {
 	mp.log = doLogging
 	mp.configDir = configDir
 	mp.starttime = time.Now()
 	mp.breakChannel = make(chan bool)
 	mp.MessageCounter = NewCounter()
+	mp.messageCountFn = mcfn
 
-	// Maintain a count of total messages
-	mp.TotalMessageCounter = NewCounter()
 	mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) {
 		c = float64(mp.MessageCounter.Count())
-		mp.TotalMessageCounter.Add(int(c))
 		mp.MessageCounter.Reset()
 		return
 	})
 
-	var pidUsageLock sync.Mutex
-	// pidusage doesn't support windows
-	if runtime.GOOS != "windows" {
-		mp.CPU = NewMonitorHistory(Percent, Average, func() float64 {
-			pidUsageLock.Lock()
-			defer pidUsageLock.Unlock()
-			sysInfo, err := pidusage.GetStat(os.Getpid())
-			if err != nil {
-				log.Errorf("pidusage.GetStat failed with: %s", err)
-				return 0.0
-			}
-			return float64(sysInfo.CPU)
-		})
-		mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
-			pidUsageLock.Lock()
-			defer pidUsageLock.Unlock()
-			sysInfo, err := pidusage.GetStat(os.Getpid())
-			if err != nil {
-				log.Errorf("pidusage.GetStat failed with: %s", err)
-				return 0.0
-			}
-			return float64(sysInfo.Memory)
-		})
-	} else {
-		mp.CPU = NewMonitorHistory(Percent, Average, func() float64 {
-			return 0.0
-		})
-		mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
-			return 0.0
-		})
-	}
+	mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		return float64(bToMb(m.Sys))
+	})
 
-	// TODO: replace with ts.
 	mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ts.Metrics().ConnectionCount) })
 
 	if mp.log {
@@ -89,16 +66,34 @@ func (mp *Monitors) Start(ts tapir.Service, configDir string, doLogging bool) {
 }
 
 func (mp *Monitors) run() {
+	mp.running = true
 	for {
 		select {
 		case <-time.After(time.Minute):
+			mp.lock.Lock()
 			mp.report()
+			mp.lock.Unlock()
 		case <-mp.breakChannel:
+			mp.lock.Lock()
+			mp.running = false
+			mp.lock.Unlock()
 			return
 		}
 	}
 }
 
+func FormatDuration(ts time.Duration) string {
+	const (
+		Day = 24 * time.Hour
+	)
+	d := ts / Day
+	ts = ts % Day
+	h := ts / time.Hour
+	ts = ts % time.Hour
+	m := ts / time.Minute
+	return fmt.Sprintf("%dd%dh%dm", d, h, m)
+}
+
 func (mp *Monitors) report() {
 	f, err := os.Create(path.Join(mp.configDir, reportFile))
 	if err != nil {
@@ -109,18 +104,16 @@ func (mp *Monitors) report() {
 
 	w := bufio.NewWriter(f)
 
-	fmt.Fprintf(w, "Uptime: %v\n\n", time.Since(mp.starttime))
+	fmt.Fprintf(w, "Uptime: %v \n", FormatDuration(time.Since(mp.starttime)))
+	fmt.Fprintf(w, "Total Messages: %v \n\n", mp.messageCountFn())
 
-	fmt.Fprintln(w, "messages:")
+	fmt.Fprintln(w, "Messages:")
 	mp.Messages.Report(w)
 
 	fmt.Fprintln(w, "\nClient Connections:")
 	mp.ClientConns.Report(w)
 
-	fmt.Fprintln(w, "\nCPU:")
-	mp.CPU.Report(w)
-
-	fmt.Fprintln(w, "\nMemory:")
+	fmt.Fprintln(w, "\nSys Memory:")
 	mp.Memory.Report(w)
 
 	w.Flush()
@@ -128,11 +121,15 @@ func (mp *Monitors) report() {
 
 // Stop stops all the monitors in a Monitors
 func (mp *Monitors) Stop() {
-	if mp.log {
-		mp.breakChannel <- true
+	mp.lock.Lock()
+	running := mp.running
+	mp.lock.Unlock()
+	if running {
+		if mp.log {
+			mp.breakChannel <- true
+		}
+		mp.Messages.Stop()
+		mp.Memory.Stop()
+		mp.ClientConns.Stop()
 	}
-	mp.Messages.Stop()
-	mp.CPU.Stop()
-	mp.Memory.Stop()
-	mp.ClientConns.Stop()
 }

metrics/monitors_test.go

@@ -15,13 +15,14 @@ func TestMonitors(t *testing.T) {
 	os.Mkdir("testLog", 0700)
 	service := new(tor2.BaseOnionService)
 	mp := Monitors{}
-	mp.Start(service, "testLog", true)
+	mp.Start(service, func() int { return 1 }, "testLog", true)
+	mp.MessageCounter.Add(1)
 	log.Infof("sleeping for minute to give to for monitors to trigger...")
 	// wait a minute for it to trigger
 	time.Sleep(62 * time.Second)
 
 	// it didn't segfault? that's good, did it create a log file?
-	if _, err := os.Stat(filepath.Join("testLog",  "serverMonitorReport.txt")); err != nil {
+	if _, err := os.Stat(filepath.Join("testLog", "serverMonitorReport.txt")); err != nil {
 		t.Errorf("serverMonitorReport.txt not generated")
 	}
 

server.go

@@ -42,21 +42,22 @@ type Server interface {
 	TofuBundle() string
 	GetAttribute(string) string
 	SetAttribute(string, string)
+	SetMonitorLogging(bool)
 }
 
 type server struct {
-	service              tapir.Service
-	config               *Config
-	metricsPack          metrics.Monitors
-	tokenTapirService    tapir.Service
-	tokenServer          *privacypass.TokenServer
-	tokenService         primitives.Identity
-	tokenServicePrivKey  ed25519.PrivateKey
-	tokenServiceStopped  bool
-	onionServiceStopped  bool
-	running              bool
-	existingMessageCount int
-	lock                 sync.RWMutex
+	config              *Config
+	service             tapir.Service
+	messageStore        storage.MessageStoreInterface
+	metricsPack         metrics.Monitors
+	tokenTapirService   tapir.Service
+	tokenServer         *privacypass.TokenServer
+	tokenService        primitives.Identity
+	tokenServicePrivKey ed25519.PrivateKey
+	tokenServiceStopped bool
+	onionServiceStopped bool
+	running             bool
+	lock                sync.RWMutex
 }
 
 // NewServer creates and configures a new server based on the supplied configuration
@@ -78,6 +79,21 @@ func (s *server) Identity() primitives.Identity {
 	return s.config.Identity()
 }
 
+// helper fn to pass to metrics
+func (s *server) getStorageTotalMessageCount() int {
+	if s.messageStore != nil {
+		return s.messageStore.MessagesCount()
+	}
+	return 0
+}
+
+// helper fn to pass to storage
+func (s *server) incMessageCount() {
+	if s.metricsPack.MessageCounter != nil {
+		s.metricsPack.MessageCounter.Add(1)
+	}
+}
+
 // Run starts a server with the given privateKey
 func (s *server) Run(acn connectivity.ACN) error {
 	s.lock.Lock()
@@ -91,17 +107,17 @@ func (s *server) Run(acn connectivity.ACN) error {
 	service.Init(acn, s.config.PrivateKey, &identity)
 	s.service = service
 	log.Infof("cwtch server running on cwtch:%s\n", s.Onion())
-	s.metricsPack.Start(service, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
 
-	ms, err := storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.metricsPack.MessageCounter)
+	if s.config.ServerReporting.LogMetricsToFile {
+		s.metricsPack.Start(service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
+	}
+
+	var err error
+	s.messageStore, err = storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.config.GetMaxMessages(), s.incMessageCount)
 	if err != nil {
 		return fmt.Errorf("could not open database: %v", err)
 	}
 
-	// Needed because we only collect metrics on a per-session basis
-	// TODO fix metrics so they persist across sessions?
-	s.existingMessageCount = len(ms.FetchMessages())
-
 	s.tokenTapirService = new(tor2.BaseOnionService)
 	s.tokenTapirService.Init(acn, s.tokenServicePrivKey, &s.tokenService)
 	tokenApplication := new(applications.TokenApplication)
@@ -114,7 +130,7 @@ func (s *server) Run(acn connectivity.ACN) error {
 		s.tokenServiceStopped = true
 	}()
 	go func() {
-		s.service.Listen(NewTokenBoardServer(ms, s.tokenServer))
+		s.service.Listen(NewTokenBoardServer(s.messageStore, s.tokenServer))
 		s.onionServiceStopped = true
 	}()
 
@@ -144,13 +160,14 @@ func (s *server) CheckStatus() (bool, error) {
 }
 
 // Stop turns off the server so it cannot receive connections and frees most resourses.
-// The server is still in a reRunable state and tokenServer still has an active persistance
+// The server is still in a reRunable state and tokenServer still has an active persistence
 func (s *server) Stop() {
 	log.Infof("Shutting down server")
 	s.lock.Lock()
 	defer s.lock.Unlock()
 	if s.running {
 		s.service.Shutdown()
+		s.messageStore.Close()
 		s.tokenTapirService.Shutdown()
 		log.Infof("Closing Token server Database...")
 
@@ -159,7 +176,7 @@ func (s *server) Stop() {
 	}
 }
 
-// Destroy frees the last of the resources the server has active (toklenServer persistance) leaving it un-re-runable and completely shutdown
+// Destroy frees the last of the resources the server has active (tokenServer persistence) leaving it un-re-runable and completely shutdown
 func (s *server) Destroy() {
 	s.Stop()
 	s.lock.Lock()
@@ -169,21 +186,20 @@ func (s *server) Destroy() {
 
 // Statistics is an encapsulation of information about the server that an operator might want to know at a glance.
 type Statistics struct {
-	TotalMessages int
+	TotalMessages    int
+	TotalConnections int
 }
 
 // GetStatistics is a stub method for providing some high level information about
 // the server operation to bundling applications (e.g. the UI)
 func (s *server) GetStatistics() Statistics {
-	// TODO Statistics from Metrics is very awkward. Metrics needs an overhaul to make safe
-	total := s.existingMessageCount
-	if s.metricsPack.TotalMessageCounter != nil {
-		total += s.metricsPack.TotalMessageCounter.Count()
-	}
-
-	return Statistics{
-		TotalMessages: total,
+	if s.running {
+		return Statistics{
+			TotalMessages:    s.messageStore.MessagesCount(),
+			TotalConnections: s.service.Metrics().ConnectionCount,
+		}
 	}
+	return Statistics{}
 }
 
 func (s *server) Delete(password string) error {
@@ -229,3 +245,25 @@ func (s *server) GetAttribute(key string) string {
 func (s *server) SetAttribute(key, val string) {
 	s.config.SetAttribute(key, val)
 }
+
+// GetMessageCap gets a server's MaxStorageMBs value
+func (s *server) GetMaxStorageMBs() int {
+	return s.config.GetMaxMessageMBs()
+}
+
+// SetMaxStorageMBs sets a server's MaxStorageMBs and sets MaxMessages for storage (which can trigger a prune)
+func (s *server) SetMaxStorageMBs(val int) {
+	s.config.SetMaxMessageMBs(val)
+	s.messageStore.SetMessageCap(s.config.GetMaxMessages())
+}
+
+// SetMonitorLogging turns on or off the monitor logging suite, and logging to a file in the server dir
+func (s *server) SetMonitorLogging(do bool) {
+	s.config.ServerReporting.LogMetricsToFile = do
+	s.config.Save()
+	if do {
+		s.metricsPack.Start(s.service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
+	} else {
+		s.metricsPack.Stop()
+	}
+}

serverConfig.go

@@ -2,8 +2,8 @@ package server
 
 import (
 	"crypto/rand"
-	v1 "cwtch.im/cwtch/storage/v1"
 	"encoding/json"
+	"git.openprivacy.ca/cwtch.im/server/storage"
 	"git.openprivacy.ca/cwtch.im/tapir/primitives"
 	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"git.openprivacy.ca/openprivacy/log"
@@ -16,9 +16,6 @@ import (
 )
 
 const (
-	// SaltFile is the standard filename to store an encrypted config's SALT under beside it
-	SaltFile = "SALT"
-
 	// AttrAutostart is the attribute key for autostart setting
 	AttrAutostart = "autostart"
 
@@ -42,11 +39,12 @@ const (
 
 // Reporting is a struct for storing a the config a server needs to be a peer, and connect to a group to report
 type Reporting struct {
-	LogMetricsToFile    bool   `json:"logMetricsToFile"`
-	ReportingGroupID    string `json:"reportingGroupId"`
-	ReportingServerAddr string `json:"reportingServerAddr"`
+	LogMetricsToFile bool `json:"logMetricsToFile"`
 }
 
+// messages are ~4kb of storage
+const MessagesPerMB = 250
+
 // Config is a struct for storing basic server configuration
 type Config struct {
 	ConfigDir      string `json:"-"`
@@ -67,8 +65,12 @@ type Config struct {
 
 	Attributes map[string]string `json:"attributes"`
 
+	// messages are ~4kb of storage
+	// -1 == infinite
+	MaxStorageMBs int `json:"maxStorageMBs"`
+
 	lock         sync.Mutex
-	encFileStore v1.FileStore
+	encFileStore storage.FileStore
 }
 
 // Identity returns an encapsulation of the servers keys
@@ -92,11 +94,10 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
 	config.TokenServerPublicKey = tid.PublicKey()
 	config.MaxBufferLines = 100000
 	config.ServerReporting = Reporting{
-		LogMetricsToFile:    true,
-		ReportingGroupID:    "",
-		ReportingServerAddr: "",
+		LogMetricsToFile: false,
 	}
 	config.Attributes[AttrAutostart] = "false"
+	config.MaxStorageMBs = -1
 
 	k := new(ristretto255.Scalar)
 	b := make([]byte, 64)
@@ -105,34 +106,35 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
 		// unable to generate secure random numbers
 		panic("unable to generate secure random numbers")
 	}
-	k.FromUniformBytes(b)
+	k.SetUniformBytes(b)
 	config.TokenServiceK = *k
 	return config
 }
 
 // LoadCreateDefaultConfigFile loads a Config from or creates a default config and saves it to a json file specified by filename
 // if the encrypted flag is true the config is store encrypted by password
-func LoadCreateDefaultConfigFile(configDir, filename string, encrypted bool, password string) (*Config, error) {
+func LoadCreateDefaultConfigFile(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
 	if _, err := os.Stat(path.Join(configDir, filename)); os.IsNotExist(err) {
-		return CreateConfig(configDir, filename, encrypted, password)
+		return CreateConfig(configDir, filename, encrypted, password, defaultLogToFile)
 	}
 	return LoadConfig(configDir, filename, encrypted, password)
 }
 
 // CreateConfig creates a default config and saves it to a json file specified by filename
 // if the encrypted flag is true the config is store encrypted by password
-func CreateConfig(configDir, filename string, encrypted bool, password string) (*Config, error) {
+func CreateConfig(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
 	log.Debugf("CreateConfig for server with configDir: %s\n", configDir)
 	os.MkdirAll(configDir, 0700)
 	config := initDefaultConfig(configDir, filename, encrypted)
+	config.ServerReporting.LogMetricsToFile = defaultLogToFile
 	if encrypted {
-		key, _, err := v1.InitV1Directory(configDir, password)
+		key, _, err := storage.InitV1Directory(configDir, password)
 		if err != nil {
 			log.Errorf("could not create server directory: %s", err)
 			return nil, err
 		}
 		config.key = key
-		config.encFileStore = v1.NewFileStore(configDir, ServerConfigFile, key)
+		config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, key)
 	}
 
 	config.Save()
@@ -145,12 +147,12 @@ func LoadConfig(configDir, filename string, encrypted bool, password string) (*C
 	var raw []byte
 	var err error
 	if encrypted {
-		salt, err := ioutil.ReadFile(path.Join(configDir, SaltFile))
+		salt, err := ioutil.ReadFile(path.Join(configDir, storage.SaltFile))
 		if err != nil {
 			return nil, err
 		}
-		config.key = v1.CreateKey(password, salt)
-		config.encFileStore = v1.NewFileStore(configDir, ServerConfigFile, config.key)
+		config.key = storage.CreateKey(password, salt)
+		config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, config.key)
 		raw, err = config.encFileStore.Read()
 		if err != nil {
 			// Not an error to log as load config is called blindly across all dirs with a password to see what it applies to
@@ -189,11 +191,11 @@ func (config *Config) Save() error {
 func (config *Config) CheckPassword(checkpass string) bool {
 	config.lock.Lock()
 	defer config.lock.Unlock()
-	salt, err := ioutil.ReadFile(path.Join(config.ConfigDir, SaltFile))
+	salt, err := ioutil.ReadFile(path.Join(config.ConfigDir, storage.SaltFile))
 	if err != nil {
 		return false
 	}
-	oldkey := v1.CreateKey(checkpass, salt[:])
+	oldkey := storage.CreateKey(checkpass, salt[:])
 	return oldkey == config.key
 }
 
@@ -218,3 +220,26 @@ func (config *Config) GetAttribute(key string) string {
 	defer config.lock.Unlock()
 	return config.Attributes[key]
 }
+
+// GetMaxMessages returns the config setting for Max messages converting from MaxMB to messages
+// or -1 for infinite
+func (config *Config) GetMaxMessages() int {
+	config.lock.Lock()
+	defer config.lock.Unlock()
+	if config.MaxStorageMBs == -1 {
+		return -1
+	}
+	return config.MaxStorageMBs * MessagesPerMB
+}
+
+func (config *Config) GetMaxMessageMBs() int {
+	config.lock.Lock()
+	defer config.lock.Unlock()
+	return config.MaxStorageMBs
+}
+
+func (config *Config) SetMaxMessageMBs(newval int) {
+	config.lock.Lock()
+	defer config.lock.Unlock()
+	config.MaxStorageMBs = newval
+}

servers.go

@@ -1,9 +1,9 @@
 package server
 
 import (
-	"cwtch.im/cwtch/model"
 	"errors"
 	"fmt"
+	"git.openprivacy.ca/cwtch.im/server/storage"
 	"git.openprivacy.ca/openprivacy/connectivity"
 	"git.openprivacy.ca/openprivacy/log"
 	"io/ioutil"
@@ -63,15 +63,14 @@ func (s *servers) LoadServers(password string) ([]string, error) {
 			}
 		}
 	}
-	log.Infof("LoadServers returning: %s\n", loadedServers)
 	return loadedServers, nil
 }
 
 // CreateServer creates a new server and stores it, also returns an interface to it
 func (s *servers) CreateServer(password string) (Server, error) {
-	newLocalID := model.GenerateRandomID()
+	newLocalID := storage.GenerateRandomID()
 	directory := path.Join(s.directory, newLocalID)
-	config, err := CreateConfig(directory, ServerConfigFile, true, password)
+	config, err := CreateConfig(directory, ServerConfigFile, true, password, false)
 	if err != nil {
 		return nil, err
 	}

storage/file_store.go

@@ -0,0 +1,157 @@
+package storage
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"git.openprivacy.ca/openprivacy/log"
+	"golang.org/x/crypto/nacl/secretbox"
+	"golang.org/x/crypto/pbkdf2"
+	"golang.org/x/crypto/sha3"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+// SaltFile is the standard filename to store an encrypted config's SALT under beside it
+const SaltFile = "SALT"
+
+const version = "1"
+const versionFile = "VERSION"
+
+// GenerateRandomID generates a random 16 byte hex id code
+func GenerateRandomID() string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return hex.EncodeToString(randBytes)
+}
+
+// InitV1Directory generates a key and salt from a password, writes a SALT and VERSION file and returns the key and salt
+func InitV1Directory(directory, password string) ([32]byte, [128]byte, error) {
+	os.Mkdir(directory, 0700)
+
+	key, salt, err := CreateKeySalt(password)
+	if err != nil {
+		log.Errorf("Could not create key for profile store from password: %v\n", err)
+		return [32]byte{}, [128]byte{}, err
+	}
+
+	if err = ioutil.WriteFile(path.Join(directory, versionFile), []byte(version), 0600); err != nil {
+		log.Errorf("Could not write version file: %v", err)
+		return [32]byte{}, [128]byte{}, err
+	}
+
+	if err = ioutil.WriteFile(path.Join(directory, SaltFile), salt[:], 0600); err != nil {
+		log.Errorf("Could not write salt file: %v", err)
+		return [32]byte{}, [128]byte{}, err
+	}
+
+	return key, salt, nil
+}
+
+// fileStore stores a cwtchPeer in an encrypted file
+type fileStore struct {
+	directory string
+	filename  string
+	key       [32]byte
+}
+
+// FileStore is a primitive around storing encrypted files
+type FileStore interface {
+	Write([]byte) error
+	Read() ([]byte, error)
+	Delete()
+	ChangeKey(newkey [32]byte)
+}
+
+// NewFileStore instantiates a fileStore given a filename and a password
+func NewFileStore(directory string, filename string, key [32]byte) FileStore {
+	filestore := new(fileStore)
+	filestore.key = key
+	filestore.filename = filename
+	filestore.directory = directory
+	return filestore
+}
+
+// CreateKeySalt derives a key and salt from a password: returns key, salt, err
+func CreateKeySalt(password string) ([32]byte, [128]byte, error) {
+	var salt [128]byte
+	if _, err := io.ReadFull(rand.Reader, salt[:]); err != nil {
+		log.Errorf("Cannot read from random: %v\n", err)
+		return [32]byte{}, salt, err
+	}
+	dk := pbkdf2.Key([]byte(password), salt[:], 4096, 32, sha3.New512)
+
+	var dkr [32]byte
+	copy(dkr[:], dk)
+	return dkr, salt, nil
+}
+
+// CreateKey derives a key from a password and salt
+func CreateKey(password string, salt []byte) [32]byte {
+	dk := pbkdf2.Key([]byte(password), salt, 4096, 32, sha3.New512)
+
+	var dkr [32]byte
+	copy(dkr[:], dk)
+	return dkr
+}
+
+//EncryptFileData encrypts the data with the supplied key
+func EncryptFileData(data []byte, key [32]byte) ([]byte, error) {
+	var nonce [24]byte
+
+	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
+		log.Errorf("Cannot read from random: %v\n", err)
+		return nil, err
+	}
+
+	encrypted := secretbox.Seal(nonce[:], data, &nonce, &key)
+	return encrypted, nil
+}
+
+//DecryptFile decrypts the passed ciphertext with the supplied key.
+func DecryptFile(ciphertext []byte, key [32]byte) ([]byte, error) {
+	var decryptNonce [24]byte
+	copy(decryptNonce[:], ciphertext[:24])
+	decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &key)
+	if ok {
+		return decrypted, nil
+	}
+	return nil, errors.New("failed to decrypt")
+}
+
+// ReadEncryptedFile reads data from an encrypted file in directory with key
+func ReadEncryptedFile(directory, filename string, key [32]byte) ([]byte, error) {
+	encryptedbytes, err := ioutil.ReadFile(path.Join(directory, filename))
+	if err == nil {
+		return DecryptFile(encryptedbytes, key)
+	}
+	return nil, err
+}
+
+// write serializes a cwtchPeer to a file
+func (fps *fileStore) Write(data []byte) error {
+	encryptedbytes, err := EncryptFileData(data, fps.key)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path.Join(fps.directory, fps.filename), encryptedbytes, 0600)
+	return err
+}
+
+func (fps *fileStore) Read() ([]byte, error) {
+	return ReadEncryptedFile(fps.directory, fps.filename, fps.key)
+}
+
+func (fps *fileStore) Delete() {
+	err := os.Remove(path.Join(fps.directory, fps.filename))
+	if err != nil {
+		log.Errorf("Deleting file %v\n", err)
+	}
+}
+
+func (fps *fileStore) ChangeKey(newkey [32]byte) {
+	fps.key = newkey
+}

storage/message_store.go

@@ -5,26 +5,36 @@ import (
 	"database/sql"
 	"encoding/base64"
 	"fmt"
-	"git.openprivacy.ca/cwtch.im/server/metrics"
 	"git.openprivacy.ca/openprivacy/log"
-	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
+	"sync"
 )
 
 // MessageStoreInterface defines an interface to interact with a store of cwtch messages.