.drone.yml

@@ -1,120 +0,0 @@
----
-kind: pipeline
-type: docker
-name: linux-test
-
-steps:
-  - name: fetch
-    image: golang:1.21.5
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - go install honnef.co/go/tools/cmd/staticcheck@latest
-      - go install go.uber.org/nilaway/cmd/nilaway@latest
-      - git fetch --tags
-      - go get
-      - echo `git describe --tags` > VERSION
-      - echo `date +%G-%m-%d-%H-%M` > BUILDDATE
-
-  - name: quality
-    image: golang:1.21.5
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - go list ./... | xargs go vet
-      - staticcheck ./...
-
-  - name: units-tests
-    image: golang:1.21.5
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - sh testing/tests.sh
-
-  - name: test-builda-app
-    image: golang:1.21.5
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - cd app
-      - go build
-
-  - name: notify-gogs
-    pull: if-not-exists
-    image: openpriv/drone-gogs
-    when:
-      event: pull_request
-      status: [success, changed, failure]
-    environment:
-      GOGS_ACCOUNT_TOKEN:
-        from_secret: gogs_account_token
-    settings:
-      gogs_url: https://git.openprivacy.ca
-
-volumes:
-  # gopath where bin and pkg lives to persist across steps
-  - name: deps
-    temp: {}
-
-trigger:
-  repo: cwtch.im/server
-  branch: trunk
-  event:
-    - push
-    - pull_request
-    - tag
-
----
-kind: pipeline
-type: exec
-name: mac-test
-
-platform:
-  os: darwin
-  arch: amd64
-
-steps:
-  - name: fetch
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - export PATH=$PATH:/usr/local/go/bin
-      - go get
-
-  - name: units-tests
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - export PATH=$PATH:/usr/local/go/bin
-      - sh testing/tests.sh
-
-  - name: test-builda-app
-    volumes:
-      - name: deps
-        path: /go
-    commands:
-      - export PATH=$PATH:/usr/local/go/bin
-      - cd app
-      - go build
-
-volumes:
-  # gopath where bin and pkg lives to persist across steps
-  - name: deps
-    temp: {}
-
-trigger:
-  repo: cwtch.im/server
-  branch: trunk
-  event:
-    - push
-    - pull_request
-    - tag
-
-# TODO: windows: but need windows docker containers with
-#  go + gcc to compile sqlite. Will prolly come from likewise work for Cwtch

LICENSE

@@ -1,9 +0,0 @@
-MIT License
-Copyright (c) 2021 Open Privacy Research Society
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-

README.md

@@ -1,15 +1,5 @@
 # Cwtch Server
 
-## Building
-
-Pretty straight forward:
-- build the app in `app/` with `go build`
-- build the docker container in `docker/` with `docker build . -t openpriv/server`
-
-### Windows
-
-The server package relies on sqlite which in turn requires the use of CGO. As per [this issue](https://github.com/golang/go/issues/12029) that means [TDM-GCC](https://jmeubank.github.io/tdm-gcc/download/) is required to be installed and used to compile on Windows
-
 ## Running
 
 - cd app
@@ -18,32 +8,23 @@ The server package relies on sqlite which in turn requires the use of CGO. As pe
 
 The app takes the following arguments
 - -debug: enabled debug logging
-- -exportServerBundle: Export the server bundle to a file called serverbundle
+- -exportTofuBundle: Export the tofubundle to a file called tofubundle
-- -disableMetrics: Disable metrics reporting to serverMonitor.txt and associated tracking routines
+
-- -dir [directory]: specify a directory to store server files (default is current directory) 
 
 The app takes the following environment variables
 - CWTCH_HOME: sets the config dir for the app
-- DISABLE_METRICS: if set to any value ('1') it disables metrics reporting to serverMonitor.txt and associated tracking routines 
-
-`env CONFIG_HOME=./conf ./app`
 
 ## Using the Server
 
-When run the app will output standard log lines, one of which will contain the `serverbundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
+When run the app will output standard log lines, one of which will contain the `tofubundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
 
 ## Docker
 
 Build by executing `docker build -f docker/Dockerfile .`
 
-or run our prebuild ones with
+or run our prebuild ones with 
 
-`pull openpriv/cwtch-server`
+`pull openpriv:cwtch-server`
 
-and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch` so if you want the server data to persist you would run
+and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch`
 
-`docker run -v /var/lib/cwtch/server01:/var/lib/cwtch openpriv/cwtch-server`
-
-to create a persistent container you might try a command like:
-
-`docker run --name cwtch -v /var/lib/cwtch/server01:/var/lib/cwtch --restart always openpriv/cwtch-server`

app/main.go

@@ -2,13 +2,15 @@ package main
 
 import (
 	"crypto/rand"
+	"cwtch.im/cwtch/model"
 	"encoding/base64"
 	"flag"
+	"fmt"
 	cwtchserver "git.openprivacy.ca/cwtch.im/server"
 	"git.openprivacy.ca/cwtch.im/tapir/primitives"
 	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"git.openprivacy.ca/openprivacy/log"
-	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
+	"io/ioutil"
 	mrand "math/rand"
 	"os"
 	"os/signal"
@@ -17,11 +19,13 @@ import (
 	"time"
 )
 
+const (
+	serverConfigFile = "serverConfig.json"
+)
+
 func main() {
 	flagDebug := flag.Bool("debug", false, "Enable debug logging")
-	flagExportServer := flag.Bool("exportServerBundle", false, "Export the server bundle to a file called serverbundle")
+	flagExportTofu := flag.Bool("exportTofuBundle", false, "Export the tofubundle to a file called tofubundle")
-	flagDir := flag.String("dir", ".", "Directory to store server files in (config, encrypted messages, metrics)")
-	flagDisableMetrics := flag.Bool("disableMetrics", false, "Disable metrics reporting")
 	flag.Parse()
 
 	log.AddEverythingFromPattern("server/app/main")
@@ -33,9 +37,7 @@ func main() {
 		log.SetLevel(log.LevelDebug)
 	}
 	configDir := os.Getenv("CWTCH_HOME")
-	if configDir == "" {
+
-		configDir = *flagDir
-	}
 	if len(os.Args) == 2 && os.Args[1] == "gen1" {
 		config := new(cwtchserver.Config)
 		id, pk := primitives.InitializeEphemeralIdentity()
@@ -44,88 +46,72 @@ func main() {
 		config.PublicKey = id.PublicKey()
 		config.TokenServerPrivateKey = tpk
 		config.TokenServerPublicKey = tid.PublicKey()
+		config.MaxBufferLines = 100000
 		config.ServerReporting = cwtchserver.Reporting{
-			LogMetricsToFile: true,
+			LogMetricsToFile:    true,
+			ReportingGroupID:    "",
+			ReportingServerAddr: "",
 		}
-		config.ConfigDir = "."
+		config.Save(".", "serverConfig.json")
-		config.FilePath = cwtchserver.ServerConfigFile
-		config.Encrypted = false
-		config.Save()
 		return
 	}
 
-	disableMetrics := *flagDisableMetrics
+	serverConfig := cwtchserver.LoadConfig(configDir, serverConfigFile)
-	if os.Getenv("DISABLE_METRICS") != "" {
+
-		disableMetrics = true
-	}
-	serverConfig, err := cwtchserver.LoadCreateDefaultConfigFile(configDir, cwtchserver.ServerConfigFile, false, "", !disableMetrics)
-	if err != nil {
-		log.Errorf("Could not load/create config file: %s\n", err)
-		return
-	}
-	serverConfig.ServerReporting.LogMetricsToFile = !disableMetrics
 	// we don't need real randomness for the port, just to avoid a possible conflict...
-	r := mrand.New(mrand.NewSource(int64(time.Now().Nanosecond())))
+	mrand.Seed(int64(time.Now().Nanosecond()))
-	controlPort := r.Intn(1000) + 9052
+	controlPort := mrand.Intn(1000) + 9052
 
 	// generate a random password
 	key := make([]byte, 64)
-	_, err = rand.Read(key)
+	_, err := rand.Read(key)
 	if err != nil {
 		panic(err)
 	}
 
 	os.MkdirAll("tordir/tor", 0700)
-	tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).WithSocksPort(controlPort + 1).Build("./tordir/tor/torrc")
+	tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).Build("./tordir/tor/torrc")
-	acn, err := tor.NewTorACNWithAuth("tordir", "", "tordir/tor", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
+	acn, err := tor.NewTorACNWithAuth("tordir", "", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
+
 	if err != nil {
 		log.Errorf("\nError connecting to Tor: %v\n", err)
 		os.Exit(1)
 	}
 	defer acn.Close()
 
-	server := cwtchserver.NewServer(serverConfig)
+	server := new(cwtchserver.Server)
 	log.Infoln("starting cwtch server...")
-	log.Infof("Server %s\n", server.Onion())
 
-	log.Infof("Server bundle (import into client to use server): %s\n", log.Magenta(server.ServerBundle()))
+	server.Setup(serverConfig)
 
-	if *flagExportServer {
+	// TODO create a random group for testing
-		// Todo: change all to server export
+	group, _ := model.NewGroup(tor.GetTorV3Hostname(serverConfig.PublicKey))
-		os.WriteFile(path.Join(serverConfig.ConfigDir, "serverbundle"), []byte(server.ServerBundle()), 0600)
+	invite, err := group.Invite()
+	if err != nil {
+		panic(err)
 	}
 
-	// Graceful Stop
+	bundle := server.KeyBundle().Serialize()
+	tofubundle := fmt.Sprintf("tofubundle:server:%s||%s", base64.StdEncoding.EncodeToString(bundle), invite)
+	log.Infof("Server Tofu Bundle (import into client to use server): %s", log.Magenta(tofubundle))
+	log.Infof("Server Config: server address:%s", base64.StdEncoding.EncodeToString(bundle))
+
+	if *flagExportTofu {
+		ioutil.WriteFile(path.Join(serverConfig.ConfigDir, "tofubundle"), []byte(tofubundle), 0600)
+	}
+
+	// Graceful Shutdown
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-c
-		server.Destroy()
 		acn.Close()
+		server.Close()
 		os.Exit(1)
 	}()
 
-	running := false
+	server.Run(acn)
-	lastStatus := -2
 	for {
-		status, msg := acn.GetBootstrapStatus()
-		if status == 100 && !running {
-			log.Infoln("ACN is online, Running Server")
-			server.Run(acn)
-			running = true
-		}
-		if status != 100 {
-			if running {
-				log.Infoln("ACN is offline, Stopping Server")
-				server.Stop()
-				running = false
-			} else {
-				if lastStatus != status {
-					log.Infof("ACN booting... Status %v%%: %v\n", status, msg)
-					lastStatus = status
-				}
-			}
-		}
 		time.Sleep(time.Second)
 	}
 }

docker/Dockerfile

@@ -64,5 +64,5 @@ COPY ./docker/docker-entrypoint /usr/local/bin/
 VOLUME /etc/tor /var/lib/tor /var/lib/cwtch
 
 ENTRYPOINT ["docker-entrypoint"]
-CMD ["/usr/local/bin/cwtch","--exportServerBundle"]
+CMD ["/usr/local/bin/cwtch","--exportTofuBundle"]
 

go.mod

@@ -1,23 +1,14 @@
 module git.openprivacy.ca/cwtch.im/server
 
-go 1.20
+go 1.14
 
 require (
-	cwtch.im/cwtch v0.27.0
+	cwtch.im/cwtch v0.8.5
-	git.openprivacy.ca/cwtch.im/tapir v0.6.0
+	git.openprivacy.ca/cwtch.im/tapir v0.4.2
-	git.openprivacy.ca/openprivacy/connectivity v1.11.0
+	git.openprivacy.ca/openprivacy/connectivity v1.4.3
-	git.openprivacy.ca/openprivacy/log v1.0.3
+	git.openprivacy.ca/openprivacy/log v1.0.2
-	github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c
+	github.com/gtank/ristretto255 v0.1.2
 	github.com/mattn/go-sqlite3 v1.14.7
-	golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d
+	github.com/struCoder/pidusage v0.2.1
-)
+	golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
-
-require (
-	filippo.io/edwards25519 v1.0.0 // indirect
-	git.openprivacy.ca/openprivacy/bine v0.0.5 // indirect
-	github.com/gtank/merlin v0.1.1 // indirect
-	github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
-	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
-	golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect
 )

go.sum

@@ -1,47 +1,100 @@
-cwtch.im/cwtch v0.27.0 h1:MkNIZ+pT5ZwiGlKHk20GMUTlzBsmCWJEibbj6hr+WHw=
+cwtch.im/cwtch v0.7.3 h1:3f2Q2XFgNtq8hVbEz7K++exS1gDsO+zmSO7X7Fqz8Mo=
-cwtch.im/cwtch v0.27.0/go.mod h1:A3i92aFuhyHI2DYO2Qnvl5iqEw0Cox22pdiypHnMOy4=
+cwtch.im/cwtch v0.7.3/go.mod h1:S0JRLSTwFM+kRrpOPKgUQn/loKe/fc6lLDnOFopRKq8=
-filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
+cwtch.im/cwtch v0.8.0 h1:QDRaDBTXefFRPZPqUMtxoNhOcgXv0rl0bGjysSOmJX0=
-filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
+cwtch.im/cwtch v0.8.0/go.mod h1:+SY/4ueF1U7mK+CX8hZFbtd+GC1lx/cReo110KgtQAw=
-git.openprivacy.ca/cwtch.im/tapir v0.6.0 h1:TtnKjxitkIDMM7Qn0n/u+mOHRLJzuQUYjYRu5n0/QFY=
+cwtch.im/cwtch v0.8.5 h1:W67jAF2oRwqWytbZEv1UeCqW0cU2x69tgUw8iy27xFA=
-git.openprivacy.ca/cwtch.im/tapir v0.6.0/go.mod h1:iQIq4y7N+DuP3CxyG66WNEC/d6vzh+wXvvOmelB+KoY=
+cwtch.im/cwtch v0.8.5/go.mod h1:5GHxaaeVnKeXSU64IvtCKzkqhU8DRiLoVM+tiBT8kkc=
-git.openprivacy.ca/openprivacy/bine v0.0.5 h1:DJs5gqw3SkvLSgRDvroqJxZ7F+YsbxbBRg5t0rU5gYE=
+git.openprivacy.ca/cwtch.im/tapir v0.4.0 h1:clG8uORt0NKEhT4P+Dpw1pzyUuYzYBMevGqn2pciKk8=
-git.openprivacy.ca/openprivacy/bine v0.0.5/go.mod h1:fwdeq6RO08WDkV0k7HfArsjRvurVULoUQmT//iaABZM=
+git.openprivacy.ca/cwtch.im/tapir v0.4.0/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
-git.openprivacy.ca/openprivacy/connectivity v1.11.0 h1:roASjaFtQLu+HdH5fa2wx6F00NL3YsUTlmXBJh8aLZk=
+git.openprivacy.ca/cwtch.im/tapir v0.4.1 h1:9LMpQX41IzecNNlRc1FZKXHg6wlFss679tFsa3vzb3Y=
-git.openprivacy.ca/openprivacy/connectivity v1.11.0/go.mod h1:OQO1+7OIz/jLxDrorEMzvZA6SEbpbDyLGpjoFqT3z1Y=
+git.openprivacy.ca/cwtch.im/tapir v0.4.1/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
-git.openprivacy.ca/openprivacy/log v1.0.3 h1:E/PMm4LY+Q9s3aDpfySfEDq/vYQontlvNj/scrPaga0=
+git.openprivacy.ca/cwtch.im/tapir v0.4.2 h1:bxMWZnVJXX4dqqOFS7ELW4iFkVL4GS8wiRkjRv5rJe8=
-git.openprivacy.ca/openprivacy/log v1.0.3/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+git.openprivacy.ca/cwtch.im/tapir v0.4.2/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
+git.openprivacy.ca/openprivacy/bine v0.0.4 h1:CO7EkGyz+jegZ4ap8g5NWRuDHA/56KKvGySR6OBPW+c=
+git.openprivacy.ca/openprivacy/bine v0.0.4/go.mod h1:13ZqhKyqakDsN/ZkQkIGNULsmLyqtXc46XBcnuXm/mU=
+git.openprivacy.ca/openprivacy/connectivity v1.4.3 h1:i2Ad/U9FlL9dKr2bhRck7lJ8NoWyGtoEfUwoCyMT0fU=
+git.openprivacy.ca/openprivacy/connectivity v1.4.3/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
+git.openprivacy.ca/openprivacy/log v1.0.1/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+git.openprivacy.ca/openprivacy/log v1.0.2 h1:HLP4wsw4ljczFAelYnbObIs821z+jgMPCe8uODPnGQM=
+git.openprivacy.ca/openprivacy/log v1.0.2/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is=
 github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
-github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c h1:gkfmnY4Rlt3VINCo4uKdpvngiibQyoENVj5Q88sxXhE=
+github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc=
-github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c/go.mod h1:tDPFhGdt3hJWqtKwx57i9baiB1Cj0yAg22VOPUqm5vY=
+github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
 github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0=
 github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
-github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
-github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b/go.mod h1:xxLb2ip6sSUts3g1irPVHyk/DGslwQsNOo9I7smJfNU=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+github.com/struCoder/pidusage v0.1.3 h1:pZcSa6asBE38TJtW0Nui6GeCjLTpaT/jAnNP7dUTLSQ=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
+github.com/struCoder/pidusage v0.1.3/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI=
+github.com/struCoder/pidusage v0.2.1 h1:dFiEgUDkubeIj0XA1NpQ6+8LQmKrLi7NiIQl86E6BoY=
+github.com/struCoder/pidusage v0.2.1/go.mod h1:bewtP2KUA1TBUyza5+/PCpSQ6sc/H6jJbIKAzqW86BA=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI=
 golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d h1:3qF+Z8Hkrw9sOhrFHti9TlB1Hkac1x+DNRkv0XQiFjo=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
-golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

metrics/metrics.go

@@ -9,14 +9,9 @@ import (
 	"time"
 )
 
-// Fields must be in this order because go compiler has problems with 64bit fields on 32 bit arches (arm32 raspberry pi):
-//
-//	https://git.openprivacy.ca/cwtch.im/server/pulls/30
-//	https://github.com/golang/go/issues/599
-//	https://github.com/census-instrumentation/opencensus-go/issues/587
 type counter struct {
-	count     uint64
 	startTime time.Time
+	count     uint64
 }
 
 // Counter providers a threadsafe counter to use for storing long running counts
@@ -76,7 +71,6 @@ const (
 	Average
 )
 
-// TODO port to SQLite for persistence between runs?
 type monitorHistory struct {
 	monitorType         MonitorType
 	monitorAccumulation MonitorAccumulation
@@ -156,25 +150,13 @@ func (mh *monitorHistory) Months() []float64 {
 	return mh.returnCopy(mh.perMonthForYear[:])
 }
 
-const timeDay = time.Hour * 24
-const timeWeek = timeDay * 7
-const timeMonth = timeDay * 28
-
 func (mh *monitorHistory) Report(w *bufio.Writer) {
 	mh.lock.Lock()
 	fmt.Fprintln(w, "Minutes:", reportLine(mh.monitorType, mh.perMinutePerHour[:]))
-	if time.Since(mh.starttime) >= time.Hour {
+	fmt.Fprintln(w, "Hours:  ", reportLine(mh.monitorType, mh.perHourForDay[:]))
-		fmt.Fprintln(w, "Hours:  ", reportLine(mh.monitorType, mh.perHourForDay[:]))
+	fmt.Fprintln(w, "Days:   ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
-	}
+	fmt.Fprintln(w, "Weeks:  ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
-	if time.Since(mh.starttime) >= timeDay {
+	fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
-		fmt.Fprintln(w, "Days:   ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
-	}
-	if time.Since(mh.starttime) >= timeWeek {
-		fmt.Fprintln(w, "Weeks:  ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
-	}
-	if time.Since(mh.starttime) >= timeMonth {
-		fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
-	}
 	mh.lock.Unlock()
 }
 
@@ -197,7 +179,9 @@ func reportLine(t MonitorType, array []float64) string {
 func (mh *monitorHistory) returnCopy(slice []float64) []float64 {
 	retSlice := make([]float64, len(slice))
 	mh.lock.Lock()
-	copy(retSlice, slice)
+	for i, v := range slice {
+		retSlice[i] = v
+	}
 	mh.lock.Unlock()
 	return retSlice
 }
@@ -233,24 +217,24 @@ func (mh *monitorHistory) monitorThread() {
 		case <-time.After(time.Minute):
 			mh.lock.Lock()
 
-			minuteAcc := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
+			minuteAvg := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
 
-			if time.Since(mh.timeLastHourRotate) > time.Hour {
+			if time.Now().Sub(mh.timeLastHourRotate) > time.Hour {
-				rotateAndAccumulate(mh.perHourForDay[:], minuteAcc, mh.monitorAccumulation)
+				rotateAndAccumulate(mh.perHourForDay[:], minuteAvg, mh.monitorAccumulation)
 				mh.timeLastHourRotate = time.Now()
 			}
 
-			if time.Since(mh.timeLastDayRotate) > time.Hour*24 {
+			if time.Now().Sub(mh.timeLastDayRotate) > time.Hour*24 {
 				rotateAndAccumulate(mh.perDayForWeek[:], accumulate(mh.perHourForDay[:], mh.monitorAccumulation), mh.monitorAccumulation)
 				mh.timeLastDayRotate = time.Now()
 			}
 
-			if time.Since(mh.timeLastWeekRotate) > time.Hour*24*7 {
+			if time.Now().Sub(mh.timeLastWeekRotate) > time.Hour*24*7 {
 				rotateAndAccumulate(mh.perWeekForMonth[:], accumulate(mh.perDayForWeek[:], mh.monitorAccumulation), mh.monitorAccumulation)
 				mh.timeLastWeekRotate = time.Now()
 			}
 
-			if time.Since(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
+			if time.Now().Sub(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
 				rotateAndAccumulate(mh.perMonthForYear[:], accumulate(mh.perWeekForMonth[:], mh.monitorAccumulation), mh.monitorAccumulation)
 				mh.timeLastMonthRotate = time.Now()
 			}

metrics/monitors.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"git.openprivacy.ca/cwtch.im/tapir"
 	"git.openprivacy.ca/openprivacy/log"
+	"github.com/struCoder/pidusage"
 	"os"
 	"path"
-	"runtime"
 	"sync"
 	"time"
 )
@@ -16,48 +16,52 @@ const (
 	reportFile = "serverMonitorReport.txt"
 )
 
-type MessageCountFn func() int
-
 // Monitors is a package of metrics for a Cwtch Server including message count, CPU, Mem, and conns
 type Monitors struct {
-	MessageCounter Counter
+	MessageCounter      Counter
-	Messages       MonitorHistory
+	TotalMessageCounter Counter
-	Memory         MonitorHistory
+	Messages            MonitorHistory
-	ClientConns    MonitorHistory
+	CPU                 MonitorHistory
-	messageCountFn MessageCountFn
+	Memory              MonitorHistory
-	starttime      time.Time
+	ClientConns         MonitorHistory
-	breakChannel   chan bool
+	starttime           time.Time
-	log            bool
+	breakChannel        chan bool
-	configDir      string
+	log                 bool
-	running        bool
+	configDir           string
-	lock           sync.Mutex
-}
-
-func bToMb(b uint64) uint64 {
-	return b / 1024 / 1024
 }
 
 // Start initializes a Monitors's monitors
-func (mp *Monitors) Start(ts tapir.Service, mcfn MessageCountFn, configDir string, doLogging bool) {
+func (mp *Monitors) Start(ts tapir.Service, configDir string, log bool) {
-	mp.log = doLogging
+	mp.log = log
 	mp.configDir = configDir
 	mp.starttime = time.Now()
 	mp.breakChannel = make(chan bool)
 	mp.MessageCounter = NewCounter()
-	mp.messageCountFn = mcfn
 
+	// Maintain a count of total messages
+	mp.TotalMessageCounter = NewCounter()
 	mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) {
 		c = float64(mp.MessageCounter.Count())
+		mp.TotalMessageCounter.Add(int(c))
 		mp.MessageCounter.Reset()
 		return
 	})
 
+	var pidUsageLock sync.Mutex
+	mp.CPU = NewMonitorHistory(Percent, Average, func() float64 {
+		pidUsageLock.Lock()
+		defer pidUsageLock.Unlock()
+		sysInfo, _ := pidusage.GetStat(os.Getpid())
+		return float64(sysInfo.CPU)
+	})
 	mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
-		var m runtime.MemStats
+		pidUsageLock.Lock()
-		runtime.ReadMemStats(&m)
+		defer pidUsageLock.Unlock()
-		return float64(bToMb(m.Sys))
+		sysInfo, _ := pidusage.GetStat(os.Getpid())
+		return float64(sysInfo.Memory)
 	})
 
+	// TODO: replace with ts.
 	mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ts.Metrics().ConnectionCount) })
 
 	if mp.log {
@@ -66,34 +70,16 @@ func (mp *Monitors) Start(ts tapir.Service, mcfn MessageCountFn, configDir strin
 }
 
 func (mp *Monitors) run() {
-	mp.running = true
 	for {
 		select {
 		case <-time.After(time.Minute):
-			mp.lock.Lock()
 			mp.report()
-			mp.lock.Unlock()
 		case <-mp.breakChannel:
-			mp.lock.Lock()
-			mp.running = false
-			mp.lock.Unlock()
 			return
 		}
 	}
 }
 
-func FormatDuration(ts time.Duration) string {
-	const (
-		Day = 24 * time.Hour
-	)
-	d := ts / Day
-	ts = ts % Day
-	h := ts / time.Hour
-	ts = ts % time.Hour
-	m := ts / time.Minute
-	return fmt.Sprintf("%dd%dh%dm", d, h, m)
-}
-
 func (mp *Monitors) report() {
 	f, err := os.Create(path.Join(mp.configDir, reportFile))
 	if err != nil {
@@ -104,16 +90,18 @@ func (mp *Monitors) report() {
 
 	w := bufio.NewWriter(f)
 
-	fmt.Fprintf(w, "Uptime: %v \n", FormatDuration(time.Since(mp.starttime)))
+	fmt.Fprintf(w, "Uptime: %v\n\n", time.Now().Sub(mp.starttime))
-	fmt.Fprintf(w, "Total Messages: %v \n\n", mp.messageCountFn())
 
-	fmt.Fprintln(w, "Messages:")
+	fmt.Fprintln(w, "messages:")
 	mp.Messages.Report(w)
 
 	fmt.Fprintln(w, "\nClient Connections:")
 	mp.ClientConns.Report(w)
 
-	fmt.Fprintln(w, "\nSys Memory:")
+	fmt.Fprintln(w, "\nCPU:")
+	mp.CPU.Report(w)
+
+	fmt.Fprintln(w, "\nMemory:")
 	mp.Memory.Report(w)
 
 	w.Flush()
@@ -121,15 +109,11 @@ func (mp *Monitors) report() {
 
 // Stop stops all the monitors in a Monitors
 func (mp *Monitors) Stop() {
-	mp.lock.Lock()
+	if mp.log {
-	running := mp.running
+		mp.breakChannel <- true
-	mp.lock.Unlock()
-	if running {
-		if mp.log {
-			mp.breakChannel <- true
-		}
-		mp.Messages.Stop()
-		mp.Memory.Stop()
-		mp.ClientConns.Stop()
 	}
+	mp.Messages.Stop()
+	mp.CPU.Stop()
+	mp.Memory.Stop()
+	mp.ClientConns.Stop()
 }

metrics/monitors_test.go

@@ -1,31 +0,0 @@
-package metrics
-
-import (
-	tor2 "git.openprivacy.ca/cwtch.im/tapir/networks/tor"
-	"git.openprivacy.ca/openprivacy/log"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-)
-
-func TestMonitors(t *testing.T) {
-	log.SetLevel(log.LevelInfo)
-	os.RemoveAll("testLog")
-	os.Mkdir("testLog", 0700)
-	service := new(tor2.BaseOnionService)
-	mp := Monitors{}
-	mp.Start(service, func() int { return 1 }, "testLog", true)
-	mp.MessageCounter.Add(1)
-	log.Infof("sleeping for minute to give to for monitors to trigger...")
-	// wait a minute for it to trigger
-	time.Sleep(62 * time.Second)
-
-	// it didn't segfault? that's good, did it create a log file?
-	if _, err := os.Stat(filepath.Join("testLog", "serverMonitorReport.txt")); err != nil {
-		t.Errorf("serverMonitorReport.txt not generated")
-	}
-
-	mp.Stop()
-	os.RemoveAll("testLog")
-}

server.go

@@ -3,8 +3,6 @@ package server
 import (
 	"crypto/ed25519"
 	"cwtch.im/cwtch/model"
-	"encoding/base64"
-	"errors"
 	"fmt"
 	"git.openprivacy.ca/cwtch.im/server/metrics"
 	"git.openprivacy.ca/cwtch.im/server/storage"
@@ -17,107 +15,62 @@ import (
 	"git.openprivacy.ca/openprivacy/connectivity"
 	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"git.openprivacy.ca/openprivacy/log"
-	"os"
 	"path"
 	"sync"
 )
 
-const (
-	// ServerConfigFile is the standard filename for a server's config to be written to in a directory
-	ServerConfigFile = "serverConfig.json"
-)
-
 // Server encapsulates a complete, compliant Cwtch server.
-type Server interface {
+type Server struct {
-	Identity() primitives.Identity
+	service              tapir.Service
-	Run(acn connectivity.ACN) error
+	config               Config
-	KeyBundle() *model.KeyBundle
+	metricsPack          metrics.Monitors
-	CheckStatus() (bool, error)
+	tokenTapirService    tapir.Service
-	Stop()
+	tokenServer          *privacypass.TokenServer
-	Destroy()
+	tokenService         primitives.Identity
-	GetStatistics() Statistics
+	tokenServicePrivKey  ed25519.PrivateKey
-	Delete(password string) error
+	tokenServiceStopped  bool
-	Onion() string
+	onionServiceStopped  bool
-	ServerBundle() string
+	running              bool
-	TofuBundle() string
+	existingMessageCount int
-	GetAttribute(string) string
+	lock                 sync.RWMutex
-	SetAttribute(string, string)
-	SetMonitorLogging(bool)
 }
 
-type server struct {
+// Setup initialized a server from a given configuration
-	config              *Config
+func (s *Server) Setup(serverConfig Config) {
-	service             tapir.Service
+	s.config = serverConfig
-	messageStore        storage.MessageStoreInterface
-	metricsPack         metrics.Monitors
-	tokenTapirService   tapir.Service
-	tokenServer         *privacypass.TokenServer
-	tokenService        primitives.Identity
-	tokenServicePrivKey ed25519.PrivateKey
-	tokenServiceStopped bool
-	onionServiceStopped bool
-	running             bool
-	lock                sync.RWMutex
-}
-
-// NewServer creates and configures a new server based on the supplied configuration
-func NewServer(serverConfig *Config) Server {
-	server := new(server)
-	server.running = false
-	server.config = serverConfig
-	server.tokenService = server.config.TokenServiceIdentity()
-	server.tokenServicePrivKey = server.config.TokenServerPrivateKey
 	bs := new(persistence.BoltPersistence)
 	bs.Open(path.Join(serverConfig.ConfigDir, "tokens.db"))
-	server.tokenServer = privacypass.NewTokenServerFromStore(&serverConfig.TokenServiceK, bs)
+	s.tokenServer = privacypass.NewTokenServerFromStore(&serverConfig.TokenServiceK, bs)
-	log.Infof("Y: %v", server.tokenServer.Y)
+	log.Infof("Y: %v", s.tokenServer.Y)
-	return server
+	s.tokenService = s.config.TokenServiceIdentity()
+	s.tokenServicePrivKey = s.config.TokenServerPrivateKey
 }
 
 // Identity returns the main onion identity of the server
-func (s *server) Identity() primitives.Identity {
+func (s *Server) Identity() primitives.Identity {
 	return s.config.Identity()
 }
 
-// helper fn to pass to metrics
-func (s *server) getStorageTotalMessageCount() int {
-	if s.messageStore != nil {
-		return s.messageStore.MessagesCount()
-	}
-	return 0
-}
-
-// helper fn to pass to storage
-func (s *server) incMessageCount() {
-	if s.metricsPack.MessageCounter != nil {
-		s.metricsPack.MessageCounter.Add(1)
-	}
-}
-
 // Run starts a server with the given privateKey
-func (s *server) Run(acn connectivity.ACN) error {
+func (s *Server) Run(acn connectivity.ACN) error {
-	s.lock.Lock()
+	addressIdentity := tor.GetTorV3Hostname(s.config.PublicKey)
-	defer s.lock.Unlock()
-	if s.running {
-		return nil
-	}
-
 	identity := primitives.InitializeIdentity("", &s.config.PrivateKey, &s.config.PublicKey)
-	service := new(tor2.BaseOnionService)
+	var service tapir.Service
+	service = new(tor2.BaseOnionService)
 	service.Init(acn, s.config.PrivateKey, &identity)
 	s.service = service
-	log.Infof("cwtch server running on cwtch:%s\n", s.Onion())
+	log.Infof("cwtch server running on cwtch:%s\n", addressIdentity+".onion:")
+	s.metricsPack.Start(service, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
 
-	if s.config.ServerReporting.LogMetricsToFile {
+	ms, err := storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.metricsPack.MessageCounter)
-		s.metricsPack.Start(service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
-	}
-
-	var err error
-	s.messageStore, err = storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.config.GetMaxMessages(), s.incMessageCount)
 	if err != nil {
 		return fmt.Errorf("could not open database: %v", err)
 	}
 
+	// Needed because we only collect metrics on a per-session basis
+	// TODO fix metrics so they persist across sessions?
+	s.existingMessageCount = len(ms.FetchMessages())
+
 	s.tokenTapirService = new(tor2.BaseOnionService)
 	s.tokenTapirService.Init(acn, s.tokenServicePrivKey, &s.tokenService)
 	tokenApplication := new(applications.TokenApplication)
@@ -130,16 +83,18 @@ func (s *server) Run(acn connectivity.ACN) error {
 		s.tokenServiceStopped = true
 	}()
 	go func() {
-		s.service.Listen(NewTokenBoardServer(s.messageStore, s.tokenServer))
+		s.service.Listen(NewTokenBoardServer(ms, s.tokenServer))
 		s.onionServiceStopped = true
 	}()
 
+	s.lock.Lock()
 	s.running = true
+	s.lock.Unlock()
 	return nil
 }
 
 // KeyBundle provides the signed keybundle of the server
-func (s *server) KeyBundle() *model.KeyBundle {
+func (s *Server) KeyBundle() *model.KeyBundle {
 	kb := model.NewKeyBundle()
 	identity := s.config.Identity()
 	kb.Keys[model.KeyTypeServerOnion] = model.Key(identity.Hostname())
@@ -150,120 +105,56 @@ func (s *server) KeyBundle() *model.KeyBundle {
 }
 
 // CheckStatus returns true if the server is running and/or an error if any part of the server needs to be restarted.
-func (s *server) CheckStatus() (bool, error) {
+func (s *Server) CheckStatus() (bool, error) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
-	if s.onionServiceStopped || s.tokenServiceStopped {
+	if s.onionServiceStopped == true || s.tokenServiceStopped == true {
 		return s.running, fmt.Errorf("one of more server components are down: onion:%v token service: %v", s.onionServiceStopped, s.tokenServiceStopped)
 	}
 	return s.running, nil
 }
 
-// Stop turns off the server so it cannot receive connections and frees most resourses.
+// Shutdown kills the app closing all connections and freeing all goroutines
-// The server is still in a reRunable state and tokenServer still has an active persistence
+func (s *Server) Shutdown() {
-func (s *server) Stop() {
-	log.Infof("Shutting down server")
 	s.lock.Lock()
 	defer s.lock.Unlock()
-	if s.running {
+	s.service.Shutdown()
-		s.service.Shutdown()
+	s.tokenTapirService.Shutdown()
-		s.messageStore.Close()
+	s.metricsPack.Stop()
-		s.tokenTapirService.Shutdown()
+	s.running = true
-		log.Infof("Closing Token server Database...")
 
-		s.metricsPack.Stop()
-		s.running = false
-	}
-}
-
-// Destroy frees the last of the resources the server has active (tokenServer persistence) leaving it un-re-runable and completely shutdown
-func (s *server) Destroy() {
-	s.Stop()
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	s.tokenServer.Close()
 }
 
 // Statistics is an encapsulation of information about the server that an operator might want to know at a glance.
 type Statistics struct {
-	TotalMessages    int
+	TotalMessages int
-	TotalConnections int
 }
 
 // GetStatistics is a stub method for providing some high level information about
 // the server operation to bundling applications (e.g. the UI)
-func (s *server) GetStatistics() Statistics {
+func (s *Server) GetStatistics() Statistics {
-	if s.running {
+	// TODO Statistics from Metrics is very awkward. Metrics needs an overhaul to make safe
-		return Statistics{
+	total := s.existingMessageCount
-			TotalMessages:    s.messageStore.MessagesCount(),
+	if s.metricsPack.TotalMessageCounter != nil {
-			TotalConnections: s.service.Metrics().ConnectionCount,
+		total += s.metricsPack.TotalMessageCounter.Count()
-		}
+	}
+
+	return Statistics{
+		TotalMessages: total,
 	}
-	return Statistics{}
 }
 
-func (s *server) Delete(password string) error {
+// ConfigureAutostart sets whether this server should autostart (in the Cwtch UI/bundling application)
+func (s *Server) ConfigureAutostart(autostart bool) {
+	s.config.AutoStart = autostart
+	s.config.Save(s.config.ConfigDir, s.config.FilePath)
+}
+
+// Close shuts down the cwtch server in a safe way.
+func (s *Server) Close() {
+	log.Infof("Shutting down server")
 	s.lock.Lock()
-	if s.config.Encrypted && !s.config.CheckPassword(password) {
+	defer s.lock.Unlock()
-		s.lock.Unlock()
+	log.Infof("Closing Token Server Database...")
-		log.Errorf("encryped and checkpassword failed")
+	s.tokenServer.Close()
-		return errors.New("cannot delete server, passwords do not match")
-	}
-	s.lock.Unlock()
-	s.Destroy()
-	os.RemoveAll(s.config.ConfigDir)
-	return nil
-}
-
-func (s *server) Onion() string {
-	return s.config.Onion()
-}
-
-// ServerBundle returns a bundle of the server keys required to access it (torv3 keys are addresses)
-func (s *server) ServerBundle() string {
-	bundle := s.KeyBundle().Serialize()
-	return fmt.Sprintf("server:%s", base64.StdEncoding.EncodeToString(bundle))
-}
-
-// TofuBundle returns a Server Bundle + a newly created group invite
-func (s *server) TofuBundle() string {
-	group, _ := model.NewGroup(tor.GetTorV3Hostname(s.config.PublicKey))
-	invite, err := group.Invite()
-	if err != nil {
-		panic(err)
-	}
-	bundle := s.KeyBundle().Serialize()
-	return fmt.Sprintf("tofubundle:server:%s||%s", base64.StdEncoding.EncodeToString(bundle), invite)
-}
-
-// GetAttribute gets a server attribute
-func (s *server) GetAttribute(key string) string {
-	return s.config.GetAttribute(key)
-}
-
-// SetAttribute sets a server attribute
-func (s *server) SetAttribute(key, val string) {
-	s.config.SetAttribute(key, val)
-}
-
-// GetMessageCap gets a server's MaxStorageMBs value
-func (s *server) GetMaxStorageMBs() int {
-	return s.config.GetMaxMessageMBs()
-}
-
-// SetMaxStorageMBs sets a server's MaxStorageMBs and sets MaxMessages for storage (which can trigger a prune)
-func (s *server) SetMaxStorageMBs(val int) {
-	s.config.SetMaxMessageMBs(val)
-	s.messageStore.SetMessageCap(s.config.GetMaxMessages())
-}
-
-// SetMonitorLogging turns on or off the monitor logging suite, and logging to a file in the server dir
-func (s *server) SetMonitorLogging(do bool) {
-	s.config.ServerReporting.LogMetricsToFile = do
-	s.config.Save()
-	if do {
-		s.metricsPack.Start(s.service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
-	} else {
-		s.metricsPack.Stop()
-	}
 }

serverConfig.go

@@ -3,53 +3,26 @@ package server
 import (
 	"crypto/rand"
 	"encoding/json"
-	"git.openprivacy.ca/cwtch.im/server/storage"
 	"git.openprivacy.ca/cwtch.im/tapir/primitives"
-	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"git.openprivacy.ca/openprivacy/log"
 	"github.com/gtank/ristretto255"
 	"golang.org/x/crypto/ed25519"
-	"os"
+	"io/ioutil"
 	"path"
-	"sync"
-)
-
-const (
-	// AttrAutostart is the attribute key for autostart setting
-	AttrAutostart = "autostart"
-
-	// AttrDescription is the attribute key for a user set server description
-	AttrDescription = "description"
-
-	// AttrStorageType is used by clients that may need info about stored server config types/styles
-	AttrStorageType = "storageType"
-)
-
-const (
-	// StorageTypeDefaultPassword is a AttrStorageType that indicated a app default password was used
-	StorageTypeDefaultPassword = "storage-default-password"
-
-	// StorageTypePassword is a AttrStorageType that indicated a user password was used to protect the profile
-	StorageTypePassword = "storage-password"
-
-	// StoreageTypeNoPassword is a AttrStorageType that indicated a no password was used to protect the profile
-	StoreageTypeNoPassword = "storage-no-password"
 )
 
 // Reporting is a struct for storing a the config a server needs to be a peer, and connect to a group to report
 type Reporting struct {
-	LogMetricsToFile bool `json:"logMetricsToFile"`
+	LogMetricsToFile    bool   `json:"logMetricsToFile"`
+	ReportingGroupID    string `json:"reportingGroupId"`
+	ReportingServerAddr string `json:"reportingServerAddr"`
 }
 
-// messages are ~4kb of storage
-const MessagesPerMB = 250
-
 // Config is a struct for storing basic server configuration
 type Config struct {
-	ConfigDir string `json:"-"`
+	ConfigDir      string `json:"-"`
-	FilePath  string `json:"-"`
+	FilePath       string `json:"-"`
-	Encrypted bool   `json:"-"`
+	MaxBufferLines int    `json:"maxBufferLines"`
-	key       [32]byte
 
 	PublicKey  ed25519.PublicKey  `json:"publicKey"`
 	PrivateKey ed25519.PrivateKey `json:"privateKey"`
@@ -60,15 +33,7 @@ type Config struct {
 	TokenServiceK ristretto255.Scalar `json:"tokenServiceK"`
 
 	ServerReporting Reporting `json:"serverReporting"`
-
+	AutoStart       bool      `json:"autostart"`
-	Attributes map[string]string `json:"attributes"`
-
-	// messages are ~4kb of storage
-	// -1 == infinite
-	MaxStorageMBs int `json:"maxStorageMBs"`
-
-	lock         sync.Mutex
-	encFileStore storage.FileStore
 }
 
 // Identity returns an encapsulation of the servers keys
@@ -81,8 +46,17 @@ func (config *Config) TokenServiceIdentity() primitives.Identity {
 	return primitives.InitializeIdentity("", &config.TokenServerPrivateKey, &config.TokenServerPublicKey)
 }
 
-func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
+// Save dumps the latest version of the config to a json file given by filename
-	config := &Config{Encrypted: encrypted, ConfigDir: configDir, FilePath: filename, Attributes: make(map[string]string)}
+func (config *Config) Save(dir, filename string) {
+	log.Infof("Saving config to %s\n", path.Join(dir, filename))
+	bytes, _ := json.MarshalIndent(config, "", "\t")
+	ioutil.WriteFile(path.Join(dir, filename), bytes, 0600)
+}
+
+// LoadConfig loads a Config from a json file specified by filename
+func LoadConfig(configDir, filename string) Config {
+	log.Infof("Loading config from %s\n", path.Join(configDir, filename))
+	config := Config{}
 
 	id, pk := primitives.InitializeEphemeralIdentity()
 	tid, tpk := primitives.InitializeEphemeralIdentity()
@@ -90,11 +64,15 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
 	config.PublicKey = id.PublicKey()
 	config.TokenServerPrivateKey = tpk
 	config.TokenServerPublicKey = tid.PublicKey()
+	config.MaxBufferLines = 100000
 	config.ServerReporting = Reporting{
-		LogMetricsToFile: false,
+		LogMetricsToFile:    true,
+		ReportingGroupID:    "",
+		ReportingServerAddr: "",
 	}
-	config.Attributes[AttrAutostart] = "false"
+	config.AutoStart = false
-	config.MaxStorageMBs = -1
+	config.ConfigDir = configDir
+	config.FilePath = filename
 
 	k := new(ristretto255.Scalar)
 	b := make([]byte, 64)
@@ -103,140 +81,18 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
 		// unable to generate secure random numbers
 		panic("unable to generate secure random numbers")
 	}
-	k.SetUniformBytes(b)
+	k.FromUniformBytes(b)
 	config.TokenServiceK = *k
+
+	raw, err := ioutil.ReadFile(path.Join(configDir, filename))
+	if err == nil {
+		err = json.Unmarshal(raw, &config)
+
+		if err != nil {
+			log.Errorf("reading config: %v", err)
+		}
+	}
+	// Always save (first time generation, new version with new variables populated)
+	config.Save(configDir, filename)
 	return config
 }
-
-// LoadCreateDefaultConfigFile loads a Config from or creates a default config and saves it to a json file specified by filename
-// if the encrypted flag is true the config is store encrypted by password
-func LoadCreateDefaultConfigFile(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
-	if _, err := os.Stat(path.Join(configDir, filename)); os.IsNotExist(err) {
-		return CreateConfig(configDir, filename, encrypted, password, defaultLogToFile)
-	}
-	return LoadConfig(configDir, filename, encrypted, password)
-}
-
-// CreateConfig creates a default config and saves it to a json file specified by filename
-// if the encrypted flag is true the config is store encrypted by password
-func CreateConfig(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
-	log.Debugf("CreateConfig for server with configDir: %s\n", configDir)
-	os.MkdirAll(configDir, 0700)
-	config := initDefaultConfig(configDir, filename, encrypted)
-	config.ServerReporting.LogMetricsToFile = defaultLogToFile
-	if encrypted {
-		key, _, err := storage.InitV1Directory(configDir, password)
-		if err != nil {
-			log.Errorf("could not create server directory: %s", err)
-			return nil, err
-		}
-		config.key = key
-		config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, key)
-	}
-
-	config.Save()
-	return config, nil
-}
-
-// LoadConfig loads a Config from a json file specified by filename
-func LoadConfig(configDir, filename string, encrypted bool, password string) (*Config, error) {
-	config := initDefaultConfig(configDir, filename, encrypted)
-	var raw []byte
-	var err error
-	if encrypted {
-		salt, err := os.ReadFile(path.Join(configDir, storage.SaltFile))
-		if err != nil {
-			return nil, err
-		}
-		config.key = storage.CreateKey(password, salt)
-		config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, config.key)
-		raw, err = config.encFileStore.Read()
-		if err != nil {
-			// Not an error to log as load config is called blindly across all dirs with a password to see what it applies to
-			log.Debugf("read enc bytes failed: %s\n", err)
-			return nil, err
-		}
-	} else {
-		raw, err = os.ReadFile(path.Join(configDir, filename))
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if err = json.Unmarshal(raw, &config); err != nil {
-		log.Errorf("reading config: %v", err)
-		return nil, err
-	}
-
-	// Always save (first time generation, new version with new variables populated)
-	config.Save()
-	return config, nil
-}
-
-// Save dumps the latest version of the config to a json file given by filename
-func (config *Config) Save() error {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	bytes, _ := json.MarshalIndent(config, "", "\t")
-	if config.Encrypted {
-		return config.encFileStore.Write(bytes)
-	}
-	return os.WriteFile(path.Join(config.ConfigDir, config.FilePath), bytes, 0600)
-}
-
-// CheckPassword returns true if the given password produces the same key as the current stored key, otherwise false.
-func (config *Config) CheckPassword(checkpass string) bool {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	salt, err := os.ReadFile(path.Join(config.ConfigDir, storage.SaltFile))
-	if err != nil {
-		return false
-	}
-	oldkey := storage.CreateKey(checkpass, salt[:])
-	return oldkey == config.key
-}
-
-// Onion returns the .onion url for the server
-func (config *Config) Onion() string {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	return tor.GetTorV3Hostname(config.PublicKey) + ".onion"
-}
-
-// SetAttribute sets a server attribute
-func (config *Config) SetAttribute(key, val string) {
-	config.lock.Lock()
-	config.Attributes[key] = val
-	config.lock.Unlock()
-	config.Save()
-}
-
-// GetAttribute gets a server attribute
-func (config *Config) GetAttribute(key string) string {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	return config.Attributes[key]
-}
-
-// GetMaxMessages returns the config setting for Max messages converting from MaxMB to messages
-// or -1 for infinite
-func (config *Config) GetMaxMessages() int {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	if config.MaxStorageMBs == -1 {
-		return -1
-	}
-	return config.MaxStorageMBs * MessagesPerMB
-}
-
-func (config *Config) GetMaxMessageMBs() int {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	return config.MaxStorageMBs
-}
-
-func (config *Config) SetMaxMessageMBs(newval int) {
-	config.lock.Lock()
-	defer config.lock.Unlock()
-	config.MaxStorageMBs = newval
-}

server_tokenboard.go

@@ -50,14 +50,14 @@ func (ta *TokenboardServer) Listen() {
 	for {
 		data := ta.connection.Expect()
 		if len(data) == 0 {
-			log.Debugf("server Closing Connection")
+			log.Debugf("Server Closing Connection")
 			ta.connection.Close()
 			return // connection is closed
 		}
 
 		var message groups.Message
 		if err := json.Unmarshal(data, &message); err != nil {
-			log.Debugf("server Closing Connection Because of Malformed Client Packet %v", err)
+			log.Debugf("Server Closing Connection Because of Malformed Client Packet %v", err)
 			ta.connection.Close()
 			return // connection is closed
 		}
@@ -69,7 +69,7 @@ func (ta *TokenboardServer) Listen() {
 				log.Debugf("Received a Post Message Request: %v", ta.connection.Hostname())
 				ta.postMessageRequest(postrequest)
 			} else {
-				log.Debugf("server Closing Connection Because of PostRequestMessage Client Packet")
+				log.Debugf("Server Closing Connection Because of PostRequestMessage Client Packet")
 				ta.connection.Close()
 				return // connection is closed
 			}
@@ -97,7 +97,7 @@ func (ta *TokenboardServer) Listen() {
 					ta.connection.Send(data)
 				}
 			} else {
-				log.Debugf("server Closing Connection Because of Malformed ReplayRequestMessage Packet")
+				log.Debugf("Server Closing Connection Because of Malformed ReplayRequestMessage Packet")
 				ta.connection.Close()
 				return // connection is closed
 			}

servers.go

@@ -1,151 +0,0 @@
-package server
-
-import (
-	"errors"
-	"fmt"
-	"git.openprivacy.ca/cwtch.im/server/storage"
-	"git.openprivacy.ca/openprivacy/connectivity"
-	"git.openprivacy.ca/openprivacy/log"
-	"os"
-	"path"
-	"sync"
-)
-
-// Servers is an interface to manage multiple Cwtch servers
-// Unlike a standalone server, server's dirs will be under one "$CwtchDir/servers" and use a cwtch style localID to obscure
-// what servers are hosted. Users are of course free to use a default password. This means Config file will be encrypted
-// with cwtch/storage/v1/file_enc and monitor files will not be generated
-type Servers interface {
-	LoadServers(password string) ([]string, error)
-	CreateServer(password string) (Server, error)
-
-	GetServer(onion string) Server
-	ListServers() []string
-	DeleteServer(onion string, currentPassword string) error
-
-	LaunchServer(string)
-	StopServer(string)
-	Stop()
-	Destroy()
-}
-
-type servers struct {
-	lock      sync.Mutex
-	servers   map[string]Server
-	directory string
-	acn       connectivity.ACN
-}
-
-// NewServers returns a Servers interface to manage a collection of servers
-// expecting directory: $CWTCH_HOME/servers
-func NewServers(acn connectivity.ACN, directory string) Servers {
-	return &servers{acn: acn, directory: directory, servers: make(map[string]Server)}
-}
-
-// LoadServers will attempt to load any servers in the servers directory that are encrypted with the supplied password
-// returns a list of onions identifiers for servers loaded or an error
-func (s *servers) LoadServers(password string) ([]string, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	dirs, err := os.ReadDir(s.directory)
-	if err != nil {
-		return nil, fmt.Errorf("error: cannot read server directory: %v", err)
-	}
-	loadedServers := []string{}
-	for _, dir := range dirs {
-		newConfig, err := LoadConfig(path.Join(s.directory, dir.Name()), ServerConfigFile, true, password)
-		if err == nil {
-			if _, exists := s.servers[newConfig.Onion()]; !exists {
-				log.Debugf("Loaded config, building server for %s\n", newConfig.Onion())
-				server := NewServer(newConfig)
-				s.servers[server.Onion()] = server
-				loadedServers = append(loadedServers, server.Onion())
-			}
-		}
-	}
-	return loadedServers, nil
-}
-
-// CreateServer creates a new server and stores it, also returns an interface to it
-func (s *servers) CreateServer(password string) (Server, error) {
-	newLocalID := storage.GenerateRandomID()
-	directory := path.Join(s.directory, newLocalID)
-	config, err := CreateConfig(directory, ServerConfigFile, true, password, false)
-	if err != nil {
-		return nil, err
-	}
-	server := NewServer(config)
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	s.servers[server.Onion()] = server
-	return server, nil
-}
-
-// GetServer returns a server interface for the supplied onion
-func (s *servers) GetServer(onion string) Server {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	return s.servers[onion]
-}
-
-// ListServers returns a list of server onion identifies this servers struct is managing
-func (s *servers) ListServers() []string {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	list := []string{}
-	for onion := range s.servers {
-		list = append(list, onion)
-	}
-	return list
-}
-
-// DeleteServer delete's the requested server (assuming the passwords match
-func (s *servers) DeleteServer(onion string, password string) error {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	server := s.servers[onion]
-	if server != nil {
-		err := server.Delete(password)
-		if err == nil {
-			delete(s.servers, onion)
-		}
-		return err
-	}
-	return errors.New("server not found")
-}
-
-// LaunchServer Run() the specified server
-func (s *servers) LaunchServer(onion string) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if server, exists := s.servers[onion]; exists {
-		server.Run(s.acn)
-	}
-}
-
-// StopServer stops the specified server
-func (s *servers) StopServer(onion string) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if server, exists := s.servers[onion]; exists {
-		server.Stop()
-	}
-}
-
-// Stop stops all the servers
-func (s *servers) Stop() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	for _, server := range s.servers {
-		server.Stop()
-	}
-}
-
-// Destroy destroys all the servers
-func (s *servers) Destroy() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	for _, server := range s.servers {
-		server.Destroy()
-	}
-}

servers_test.go

@@ -1,59 +0,0 @@
-package server
-
-import (
-	"git.openprivacy.ca/openprivacy/connectivity"
-	"git.openprivacy.ca/openprivacy/log"
-	"os"
-	"testing"
-)
-
-const TestDir = "./serversTest"
-const DefaultPassword = "be gay do crime"
-
-const TestServerDesc = "a test Server"
-
-func TestServers(t *testing.T) {
-	log.SetLevel(log.LevelDebug)
-	log.Infof("clean up / setup...")
-	os.RemoveAll(TestDir)
-	os.Mkdir(TestDir, 0700)
-
-	acn := connectivity.NewLocalACN()
-	log.Infof("NewServers()...")
-	servers := NewServers(acn, TestDir)
-	s, err := servers.CreateServer(DefaultPassword)
-	if err != nil {
-		t.Errorf("could not create server: %s", err)
-		return
-	}
-	s.SetAttribute(AttrDescription, TestServerDesc)
-	serverOnion := s.Onion()
-
-	s.Destroy()
-
-	log.Infof("NewServers()...")
-	servers2 := NewServers(acn, TestDir)
-	log.Infof("LoadServers()...")
-	list, err := servers2.LoadServers(DefaultPassword)
-	log.Infof("Loaded!")
-	if err != nil {
-		t.Errorf("clould not load server: %s", err)
-		return
-	}
-	if len(list) != 1 {
-		t.Errorf("expected to load 1 server, got %d", len(list))
-		return
-	}
-
-	if list[0] != serverOnion {
-		t.Errorf("expected loaded server to have onion: %s but got %s", serverOnion, list[0])
-	}
-
-	s1 := servers.GetServer(list[0])
-	if s1.GetAttribute(AttrDescription) != TestServerDesc {
-		t.Errorf("expected server description of '%s' but got '%s'", TestServerDesc, s1.GetAttribute(AttrDescription))
-	}
-
-	servers2.Destroy()
-	os.RemoveAll(TestDir)
-}

storage/file_store.go

@@ -1,156 +0,0 @@
-package storage
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"errors"
-	"git.openprivacy.ca/openprivacy/log"
-	"golang.org/x/crypto/nacl/secretbox"
-	"golang.org/x/crypto/pbkdf2"
-	"golang.org/x/crypto/sha3"
-	"io"
-	"os"
-	"path"
-)
-
-// SaltFile is the standard filename to store an encrypted config's SALT under beside it
-const SaltFile = "SALT"
-
-const version = "1"
-const versionFile = "VERSION"
-
-// GenerateRandomID generates a random 16 byte hex id code
-func GenerateRandomID() string {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	return hex.EncodeToString(randBytes)
-}
-
-// InitV1Directory generates a key and salt from a password, writes a SALT and VERSION file and returns the key and salt
-func InitV1Directory(directory, password string) ([32]byte, [128]byte, error) {
-	os.Mkdir(directory, 0700)
-
-	key, salt, err := CreateKeySalt(password)
-	if err != nil {
-		log.Errorf("Could not create key for profile store from password: %v\n", err)
-		return [32]byte{}, [128]byte{}, err
-	}
-
-	if err = os.WriteFile(path.Join(directory, versionFile), []byte(version), 0600); err != nil {
-		log.Errorf("Could not write version file: %v", err)
-		return [32]byte{}, [128]byte{}, err
-	}
-
-	if err = os.WriteFile(path.Join(directory, SaltFile), salt[:], 0600); err != nil {
-		log.Errorf("Could not write salt file: %v", err)
-		return [32]byte{}, [128]byte{}, err
-	}
-
-	return key, salt, nil
-}
-
-// fileStore stores a cwtchPeer in an encrypted file
-type fileStore struct {
-	directory string
-	filename  string
-	key       [32]byte
-}
-
-// FileStore is a primitive around storing encrypted files
-type FileStore interface {
-	Write([]byte) error
-	Read() ([]byte, error)
-	Delete()
-	ChangeKey(newkey [32]byte)
-}
-
-// NewFileStore instantiates a fileStore given a filename and a password
-func NewFileStore(directory string, filename string, key [32]byte) FileStore {
-	filestore := new(fileStore)
-	filestore.key = key
-	filestore.filename = filename
-	filestore.directory = directory
-	return filestore
-}
-
-// CreateKeySalt derives a key and salt from a password: returns key, salt, err
-func CreateKeySalt(password string) ([32]byte, [128]byte, error) {
-	var salt [128]byte
-	if _, err := io.ReadFull(rand.Reader, salt[:]); err != nil {
-		log.Errorf("Cannot read from random: %v\n", err)
-		return [32]byte{}, salt, err
-	}
-	dk := pbkdf2.Key([]byte(password), salt[:], 4096, 32, sha3.New512)
-
-	var dkr [32]byte
-	copy(dkr[:], dk)
-	return dkr, salt, nil
-}
-
-// CreateKey derives a key from a password and salt
-func CreateKey(password string, salt []byte) [32]byte {
-	dk := pbkdf2.Key([]byte(password), salt, 4096, 32, sha3.New512)
-
-	var dkr [32]byte
-	copy(dkr[:], dk)
-	return dkr
-}
-
-// EncryptFileData encrypts the data with the supplied key
-func EncryptFileData(data []byte, key [32]byte) ([]byte, error) {
-	var nonce [24]byte
-
-	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
-		log.Errorf("Cannot read from random: %v\n", err)
-		return nil, err
-	}
-
-	encrypted := secretbox.Seal(nonce[:], data, &nonce, &key)
-	return encrypted, nil
-}
-
-// DecryptFile decrypts the passed ciphertext with the supplied key.
-func DecryptFile(ciphertext []byte, key [32]byte) ([]byte, error) {
-	var decryptNonce [24]byte
-	copy(decryptNonce[:], ciphertext[:24])
-	decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &key)
-	if ok {
-		return decrypted, nil
-	}
-	return nil, errors.New("failed to decrypt")
-}
-
-// ReadEncryptedFile reads data from an encrypted file in directory with key
-func ReadEncryptedFile(directory, filename string, key [32]byte) ([]byte, error) {
-	encryptedbytes, err := os.ReadFile(path.Join(directory, filename))
-	if err == nil {
-		return DecryptFile(encryptedbytes, key)
-	}
-	return nil, err
-}
-
-// write serializes a cwtchPeer to a file
-func (fps *fileStore) Write(data []byte) error {
-	encryptedbytes, err := EncryptFileData(data, fps.key)
-	if err != nil {
-		return err
-	}
-
-	err = os.WriteFile(path.Join(fps.directory, fps.filename), encryptedbytes, 0600)
-	return err
-}
-
-func (fps *fileStore) Read() ([]byte, error) {
-	return ReadEncryptedFile(fps.directory, fps.filename, fps.key)
-}
-
-func (fps *fileStore) Delete() {
-	err := os.Remove(path.Join(fps.directory, fps.filename))
-	if err != nil {
-		log.Errorf("Deleting file %v\n", err)
-	}
-}
-
-func (fps *fileStore) ChangeKey(newkey [32]byte) {
-	fps.key = newkey
-}

storage/message_store.go

@@ -2,39 +2,29 @@ package storage
 
 import (
 	"cwtch.im/cwtch/protocol/groups"
+	"git.openprivacy.ca/cwtch.im/server/metrics"
 	"database/sql"
 	"encoding/base64"
 	"fmt"
 	"git.openprivacy.ca/openprivacy/log"
-	"sync"
+	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
 )
 
 // MessageStoreInterface defines an interface to interact with a store of cwtch messages.
 type MessageStoreInterface interface {
 	AddMessage(groups.EncryptedGroupMessage)
 	FetchMessages() []*groups.EncryptedGroupMessage
-	MessagesCount() int
 	FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage
-	SetMessageCap(newcap int)
-	Close()
 }
 
 // SqliteMessageStore is an sqlite3 backed message store
 type SqliteMessageStore struct {
-	incMessageCounterFn func()
+	messageCounter metrics.Counter
-	messageCap          int
+	database       *sql.DB
-
-	messageCount int
-	countLock    sync.Mutex
-
-	database *sql.DB
 
 	// Some prepared queries...
 	preparedInsertStatement *sql.Stmt // A Stmt is safe for concurrent use by multiple goroutines.
 	preparedFetchFromQuery  *sql.Stmt
-	preparedFetchQuery      *sql.Stmt
-	preparedCountQuery      *sql.Stmt
-	preparedPruneStatement  *sql.Stmt
 }
 
 // Close closes the underlying sqlite3 database to further changes
@@ -44,18 +34,9 @@ func (s *SqliteMessageStore) Close() {
 	s.database.Close()
 }
 
-func (s *SqliteMessageStore) SetMessageCap(newcap int) {
-	s.countLock.Lock()
-	defer s.countLock.Unlock()
-	s.messageCap = newcap
-	s.checkPruneMessages()
-}
-
 // AddMessage implements the MessageStoreInterface AddMessage for sqlite message store
 func (s *SqliteMessageStore) AddMessage(message groups.EncryptedGroupMessage) {
-	if s.incMessageCounterFn != nil {
+	s.messageCounter.Add(1)
-		s.incMessageCounterFn()
-	}
 	// ignore this clearly invalid message...
 	if len(message.Signature) == 0 {
 		return
@@ -66,53 +47,11 @@ func (s *SqliteMessageStore) AddMessage(message groups.EncryptedGroupMessage) {
 		log.Errorf("%v %q", stmt, err)
 		return
 	}
-
-	s.countLock.Lock()
-	defer s.countLock.Unlock()
-	s.messageCount++
-	s.checkPruneMessages()
-}
-
-func (s *SqliteMessageStore) checkPruneMessages() {
-	if s.messageCap != -1 && s.messageCount > s.messageCap {
-		log.Debugf("Message Count: %d / Message Cap: %d, message cap exceeded, pruning oldest 10%...", s.messageCount, s.messageCap)
-		// Delete 10% of messages (and any overage if the cap was adjusted lower)
-		delCount := (s.messageCount - s.messageCap) + s.messageCap/10
-		stmt, err := s.preparedPruneStatement.Exec(delCount)
-		if err != nil {
-			log.Errorf("%v %q", stmt, err)
-		}
-		s.messageCount -= delCount
-	}
-}
-
-func (s *SqliteMessageStore) MessagesCount() int {
-	rows, err := s.preparedCountQuery.Query()
-
-	if err != nil {
-		log.Errorf("%v", err)
-		return -1
-	}
-	defer rows.Close()
-
-	result := rows.Next()
-	if !result {
-		return -1
-	}
-
-	var rownum int
-	err = rows.Scan(&rownum)
-	if err != nil {
-		log.Errorf("error fetching rows: %v", err)
-		return -1
-	}
-
-	return rownum
 }
 
 // FetchMessages implements the MessageStoreInterface FetchMessages for sqlite message store
-func (s *SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
+func (s SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
-	rows, err := s.preparedFetchQuery.Query()
+	rows, err := s.database.Query("SELECT id, signature,ciphertext from messages")
 	if err != nil {
 		log.Errorf("%v", err)
 		return nil
@@ -122,10 +61,10 @@ func (s *SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
 }
 
 // FetchMessagesFrom implements the MessageStoreInterface FetchMessagesFrom for sqlite message store
-func (s *SqliteMessageStore) FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage {
+func (s SqliteMessageStore) FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage {
 
 	// If signature is empty then treat this as a complete sync request
-	if len(signature) == 0 {
+	if signature == nil || len(signature) == 0 {
 		return s.FetchMessages()
 	}
 
@@ -168,7 +107,7 @@ func (s *SqliteMessageStore) compileRows(rows *sql.Rows) []*groups.EncryptedGrou
 
 // InitializeSqliteMessageStore creates a database `dbfile` with the necessary tables (if it doesn't already exist)
 // and returns an open database
-func InitializeSqliteMessageStore(dbfile string, messageCap int, incMessageCounterFn func()) (*SqliteMessageStore, error) {
+func InitializeSqliteMessageStore(dbfile string, messageCounter metrics.Counter) (*SqliteMessageStore, error) {
 	db, err := sql.Open("sqlite3", dbfile)
 	if err != nil {
 		log.Errorf("database %v cannot be created or opened %v", dbfile, err)
@@ -184,8 +123,7 @@ func InitializeSqliteMessageStore(dbfile string, messageCap int, incMessageCount
 	log.Infof("Database Initialized")
 	slms := new(SqliteMessageStore)
 	slms.database = db
-	slms.incMessageCounterFn = incMessageCounterFn
+	slms.messageCounter = messageCounter
-	slms.messageCap = messageCap
 
 	sqlStmt = `INSERT INTO messages(signature, ciphertext) values (?,?);`
 	stmt, err := slms.database.Prepare(sqlStmt)
@@ -195,41 +133,12 @@ func InitializeSqliteMessageStore(dbfile string, messageCap int, incMessageCount
 	}
 	slms.preparedInsertStatement = stmt
 
-	sqlStmt = "SELECT id, signature,ciphertext from messages"
+	query, err := slms.database.Prepare("SELECT id, signature,ciphertext FROM messages WHERE id>=(SELECT id FROM messages WHERE signature=(?));")
-	query, err := slms.database.Prepare(sqlStmt)
 	if err != nil {
-		log.Errorf("%q: %s", err, sqlStmt)
+		log.Errorf("%v", err)
-		return nil, fmt.Errorf("%s: %q", sqlStmt, err)
-	}
-	slms.preparedFetchQuery = query
-
-	sqlStmt = "SELECT id, signature,ciphertext FROM messages WHERE id>=(SELECT id FROM messages WHERE signature=(?));"
-	query, err = slms.database.Prepare(sqlStmt)
-	if err != nil {
-		log.Errorf("%q: %s", err, sqlStmt)
 		return nil, fmt.Errorf("%s: %q", sqlStmt, err)
 	}
 	slms.preparedFetchFromQuery = query
 
-	sqlStmt = "SELECT COUNT(*) from messages"
-	stmt, err = slms.database.Prepare(sqlStmt)
-	if err != nil {
-		log.Errorf("%q: %s", err, sqlStmt)
-		return nil, fmt.Errorf("%s: %q", sqlStmt, err)
-	}
-	slms.preparedCountQuery = stmt
-
-	sqlStmt = "DELETE FROM messages WHERE id IN (SELECT id FROM messages ORDER BY id ASC LIMIT (?))"
-	stmt, err = slms.database.Prepare(sqlStmt)
-	if err != nil {
-		log.Errorf("%q: %s", err, sqlStmt)
-		return nil, fmt.Errorf("%s: %q", sqlStmt, err)
-	}
-	slms.preparedPruneStatement = stmt
-
-	slms.messageCount = slms.MessagesCount()
-
-	slms.checkPruneMessages()
-
 	return slms, nil
 }

storage/message_store_test.go

@@ -2,10 +2,9 @@ package storage
 
 import (
 	"cwtch.im/cwtch/protocol/groups"
-	"encoding/binary"
 	"git.openprivacy.ca/cwtch.im/server/metrics"
+	"encoding/binary"
 	"git.openprivacy.ca/openprivacy/log"
-	_ "github.com/mattn/go-sqlite3" // sqlite3 driver
 	"os"
 	"testing"
 	"time"
@@ -16,7 +15,7 @@ func TestMessageStore(t *testing.T) {
 	os.Remove(filename)
 	log.SetLevel(log.LevelDebug)
 	counter := metrics.NewCounter()
-	db, err := InitializeSqliteMessageStore(filename, -1, func() { counter.Add(1) })
+	db, err := InitializeSqliteMessageStore(filename, counter)
 	if err != nil {
 		t.Fatalf("Error: %v", err)
 	}

testing/quality.sh

@@ -9,15 +9,16 @@ go list ./... | xargs go vet
 echo ""
 echo "Linting:"
 
-go list ./... | staticcheck ./...
+go list ./... | xargs golint
-
-echo "Running nilaway..."
-nilaway -include-pkgs="cwtch.im/server,cwtch.im/cwtch,cwtch.im/tapir,git.openprivacy.ca/openprivacy/connectivity" -exclude-file-docstrings="nolint:nilaway" ./...
 
 
 echo "Time to format"
 gofmt -l -s -w .
 
+# ineffassign (https://github.com/gordonklaus/ineffassign)
+echo "Checking for ineffectual assignment of errors (unchecked errors...)"
+ineffassign .
+
 # misspell (https://github.com/client9/misspell/cmd/misspell)
 echo "Checking for misspelled words..."
 misspell . | grep -v "vendor/" | grep -v "go.sum" | grep -v ".idea"

testing/tests.sh

@@ -4,8 +4,7 @@ set -e
 pwd
 GORACE="haltonerror=1"
 go test -race ${1} -coverprofile=server.metrics.cover.out -v ./metrics
-go test -race ${1} -coverprofile=server.storage.cover.out -v ./storage
+go test -race ${1} -coverprofile=server.metrics.cover.out -v ./storage
-go test -race ${1} -coverprofile=server.cover.out -v ./
 echo "mode: set" > coverage.out && cat *.cover.out | grep -v mode: | sort -r | \
 awk '{if($1 != last) {print $0;last=$1}}' >> coverage.out
 rm -rf *.cover.out