Compare commits

..

40 Commits

Author SHA1 Message Date
Dan Ballard 5accebc5e2 fix prune logic
continuous-integration/drone/pr Build is pending Details
continuous-integration/drone/push Build is pending Details
2024-03-10 11:39:23 -07:00
Dan Ballard aaee1d7ee6 call prune on staorage start, prune account for extra messages if cap has been adjusted; remove unused maxBuffLines
continuous-integration/drone/pr Build is passing Details
continuous-integration/drone/push Build is pending Details
2024-03-09 16:35:01 -08:00
Sarah Jamie Lewis 5f7fdd66d5 Remove MaxBufferLines Config
continuous-integration/drone/pr Build is passing Details
continuous-integration/drone/push Build is pending Details
2024-03-09 16:24:15 -08:00
Sarah Jamie Lewis 6599832012 Upgrade Server to Newest Cwtch 2024-03-09 16:20:46 -08:00
Dan Ballard 635080755d staticcheck and nilaway
continuous-integration/drone/pr Build is passing Details
continuous-integration/drone/push Build is pending Details
2024-03-09 14:17:03 -08:00
Dan Ballard 4fc3121d23 update automated testing
continuous-integration/drone/pr Build is failing Details
2024-03-09 13:54:22 -08:00
Sarah Jamie Lewis ae0aa7a5b5 Merge pull request 'dep upgrades; server app wait for ACN' (#34) from networkAfterOnline into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #34
2022-09-08 22:38:14 +00:00
Dan Ballard a7186604f7 dep upgrades; server app wait for ACN
continuous-integration/drone/pr Build was killed Details
2022-09-08 14:55:25 -07:00
Dan Ballard d748752648 Merge pull request 'add config and storage support for a message cap' (#32) from msgCap into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #32
2022-04-26 01:47:30 +00:00
Dan Ballard eba86577c8 spelling
continuous-integration/drone/pr Build is pending Details
2022-04-25 18:47:24 -07:00
Dan Ballard a8ef9fe001 deps version bump
continuous-integration/drone/pr Build is passing Details
2022-04-25 18:34:57 -07:00
Dan Ballard b144bc34cc add config and storage support for a message cap
continuous-integration/drone/pr Build is passing Details
2022-04-25 18:22:50 -07:00
Sarah Jamie Lewis 622a3f8db8 Add 'LICENSE'
continuous-integration/drone/push Build is passing Details
2022-01-29 19:11:47 +00:00
Sarah Jamie Lewis 9eb766732c Merge pull request 'minor fixes' (#29) from minorFixes into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #29
2022-01-05 17:06:15 +00:00
Dan Ballard cbef826bf0 Merge branch 'trunk' into minorFixes
continuous-integration/drone/pr Build is passing Details
2022-01-04 21:46:43 +00:00
Dan Ballard c75d25f1c0 document mathiasme fix
continuous-integration/drone/push Build is passing Details
2022-01-04 11:43:51 -05:00
Dan Ballard 07dfbcb7a2 Merge pull request 'Patched atomic calculation issue with 32 bits ARM architectures' (#30) from mathiasme/server:trunk into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #30

oph wow that's bad. thanks so much for the catch and patch
2022-01-04 16:40:47 +00:00
mathiasme 2622e64213 Patched atomic calculation issue with 32 bits ARM architectures
continuous-integration/drone/pr Build is passing Details
Server was not working on my raspberry Pi. Appears the issue is related to the Go compiler (https://github.com/golang/go/issues/599). As read here https://github.com/census-instrumentation/opencensus-go/issues/587 , I switched these two lines in the counter struct and it now works.
2022-01-01 16:40:56 +00:00
Dan Ballard 20de58bb56 fix tofubundle dump from app, fix READAME docker lines to be correct; drone more specific to avoid container checks on latest
continuous-integration/drone/pr Build is passing Details
2021-12-22 08:42:24 -05:00
Sarah Jamie Lewis b9ca357dfd Merge pull request 'cwtchDeps' (#28) from cwtchDeps into trunk
continuous-integration/drone/push Build is passing Details
continuous-integration/drone/pr Build is passing Details
Reviewed-on: #28
2021-12-20 23:09:20 +00:00
Dan Ballard 8eafa8f98a latest cwtch
continuous-integration/drone/pr Build is passing Details
2021-12-20 14:08:04 -05:00
Dan Ballard 9305e9164d move storage and GenerateRandomID cwtch deps into server to minimize parts of cwtch depended on. Now it's just model and protocol/groups 2021-12-20 13:31:09 -05:00
Dan Ballard 91bb48e82c dont access storage on non running servers for metrics (segfault) 2021-12-20 13:26:38 -05:00
Dan Ballard 92c4b25898 Merge pull request 'Move InitV1Directory to Server' (#27) from server-storage into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #27
2021-12-08 02:05:38 +00:00
Sarah Jamie Lewis c53010d2e1 Move InitV1Directory to Server
continuous-integration/drone/pr Build is passing Details
2021-12-07 17:41:38 -08:00
Sarah Jamie Lewis 819c766aa2 Merge pull request 'add flags: dir, disableMetrics and env var DISABLE_METRICS to the app' (#26) from appFlags into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #26
2021-11-29 21:09:15 +00:00
Sarah Jamie Lewis 0deb64ee6f Merge branch 'trunk' into appFlags
continuous-integration/drone/pr Build is passing Details
2021-11-29 21:09:09 +00:00
Dan Ballard b4bb859cdb add flags: dir, disableMetrics and env var DISABLE_METRICS to the app
continuous-integration/drone/pr Build is passing Details
2021-11-26 16:56:58 -08:00
Sarah Jamie Lewis 49a69bdfec Merge pull request 'refactor monitors, remove pidusage, fix logging optionality, improve fetch stats, improve reporting' (#25) from monitorsSql into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #25
2021-11-27 00:11:41 +00:00
Dan Ballard 872bf30b1b refactor monitors, remove pidusage, fix logging optionality, improve fetch stats, improve reporting
continuous-integration/drone/pr Build is passing Details
2021-11-25 18:22:22 -08:00
Sarah Jamie Lewis e6c184df4e Merge pull request 'Remove windows pidusage monitoring' (#24) from rmWinMonitor into trunk
continuous-integration/drone/push Build is pending Details
Reviewed-on: #24
2021-11-09 22:47:17 +00:00
Sarah Jamie Lewis 42383257b9 Merge branch 'trunk' into rmWinMonitor
continuous-integration/drone/pr Build is passing Details
2021-11-09 20:59:33 +00:00
Dan Ballard 851b6000d6 add monitors test
continuous-integration/drone/pr Build is passing Details
2021-11-08 17:41:31 -08:00
Dan Ballard 92c6979b2b README update for windows build instructions 2021-11-08 17:18:36 -08:00
Dan Ballard 52f7733a59 stub out pidusage calls for windows, and also check errors 2021-11-08 16:38:31 -08:00
Sarah Jamie Lewis 227a169f67 Merge pull request 'initial test build drone script' (#22) from drone into trunk
continuous-integration/drone/push Build is passing Details
Reviewed-on: #22
2021-11-09 00:36:03 +00:00
Dan Ballard 7fbe110361 golint -> staticcheck and required changes
continuous-integration/drone/pr Build is passing Details
2021-11-08 15:15:00 -08:00
Dan Ballard c725470ea2 remove windows from drone for now
continuous-integration/drone/pr Build is passing Details
2021-11-08 14:45:35 -08:00
Dan Ballard 6bff3cd74d drone + windows + macos
continuous-integration/drone/pr Build is running Details
2021-11-08 10:33:19 -08:00
Dan Ballard 883a535a7d initial test build drone script
continuous-integration/drone/pr Build is passing Details
2021-11-08 09:49:31 -08:00
16 changed files with 740 additions and 259 deletions

120
.drone.yml Normal file
View File

@ -0,0 +1,120 @@
---
kind: pipeline
type: docker
name: linux-test
steps:
- name: fetch
image: golang:1.21.5
volumes:
- name: deps
path: /go
commands:
- go install honnef.co/go/tools/cmd/staticcheck@latest
- go install go.uber.org/nilaway/cmd/nilaway@latest
- git fetch --tags
- go get
- echo `git describe --tags` > VERSION
- echo `date +%G-%m-%d-%H-%M` > BUILDDATE
- name: quality
image: golang:1.21.5
volumes:
- name: deps
path: /go
commands:
- go list ./... | xargs go vet
- staticcheck ./...
- name: units-tests
image: golang:1.21.5
volumes:
- name: deps
path: /go
commands:
- sh testing/tests.sh
- name: test-builda-app
image: golang:1.21.5
volumes:
- name: deps
path: /go
commands:
- cd app
- go build
- name: notify-gogs
pull: if-not-exists
image: openpriv/drone-gogs
when:
event: pull_request
status: [success, changed, failure]
environment:
GOGS_ACCOUNT_TOKEN:
from_secret: gogs_account_token
settings:
gogs_url: https://git.openprivacy.ca
volumes:
# gopath where bin and pkg lives to persist across steps
- name: deps
temp: {}
trigger:
repo: cwtch.im/server
branch: trunk
event:
- push
- pull_request
- tag
---
kind: pipeline
type: exec
name: mac-test
platform:
os: darwin
arch: amd64
steps:
- name: fetch
volumes:
- name: deps
path: /go
commands:
- export PATH=$PATH:/usr/local/go/bin
- go get
- name: units-tests
volumes:
- name: deps
path: /go
commands:
- export PATH=$PATH:/usr/local/go/bin
- sh testing/tests.sh
- name: test-builda-app
volumes:
- name: deps
path: /go
commands:
- export PATH=$PATH:/usr/local/go/bin
- cd app
- go build
volumes:
# gopath where bin and pkg lives to persist across steps
- name: deps
temp: {}
trigger:
repo: cwtch.im/server
branch: trunk
event:
- push
- pull_request
- tag
# TODO: windows: but need windows docker containers with
# go + gcc to compile sqlite. Will prolly come from likewise work for Cwtch

9
LICENSE Normal file
View File

@ -0,0 +1,9 @@
MIT License
Copyright (c) 2021 Open Privacy Research Society
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,5 +1,15 @@
# Cwtch Server
## Building
Pretty straight forward:
- build the app in `app/` with `go build`
- build the docker container in `docker/` with `docker build . -t openpriv/server`
### Windows
The server package relies on sqlite which in turn requires the use of CGO. As per [this issue](https://github.com/golang/go/issues/12029) that means [TDM-GCC](https://jmeubank.github.io/tdm-gcc/download/) is required to be installed and used to compile on Windows
## Running
- cd app
@ -9,16 +19,18 @@
The app takes the following arguments
- -debug: enabled debug logging
- -exportServerBundle: Export the server bundle to a file called serverbundle
- -disableMetrics: Disable metrics reporting to serverMonitor.txt and associated tracking routines
- -dir [directory]: specify a directory to store server files (default is current directory)
The app takes the following environment variables
- CWTCH_HOME: sets the config dir for the app
- DISABLE_METRICS: if set to any value ('1') it disables metrics reporting to serverMonitor.txt and associated tracking routines
`env CONFIG_HOME=./conf ./app`
## Using the Server
When run the app will output standard log lines, one of which will contain the `tofubundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
When run the app will output standard log lines, one of which will contain the `serverbundle` in purple. This is the part you need to capture and import into a Cwtch client app so you can use the server for hosting groups
## Docker
@ -28,4 +40,10 @@ or run our prebuild ones with
`pull openpriv/cwtch-server`
and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch`
and run it. It stores all Cwtch data in a Volume at `/var/lib/cwtch` so if you want the server data to persist you would run
`docker run -v /var/lib/cwtch/server01:/var/lib/cwtch openpriv/cwtch-server`
to create a persistent container you might try a command like:
`docker run --name cwtch -v /var/lib/cwtch/server01:/var/lib/cwtch --restart always openpriv/cwtch-server`

View File

@ -8,7 +8,7 @@ import (
"git.openprivacy.ca/cwtch.im/tapir/primitives"
"git.openprivacy.ca/openprivacy/connectivity/tor"
"git.openprivacy.ca/openprivacy/log"
"io/ioutil"
_ "github.com/mattn/go-sqlite3" // sqlite3 driver
mrand "math/rand"
"os"
"os/signal"
@ -20,6 +20,8 @@ import (
func main() {
flagDebug := flag.Bool("debug", false, "Enable debug logging")
flagExportServer := flag.Bool("exportServerBundle", false, "Export the server bundle to a file called serverbundle")
flagDir := flag.String("dir", ".", "Directory to store server files in (config, encrypted messages, metrics)")
flagDisableMetrics := flag.Bool("disableMetrics", false, "Disable metrics reporting")
flag.Parse()
log.AddEverythingFromPattern("server/app/main")
@ -31,7 +33,9 @@ func main() {
log.SetLevel(log.LevelDebug)
}
configDir := os.Getenv("CWTCH_HOME")
if configDir == "" {
configDir = *flagDir
}
if len(os.Args) == 2 && os.Args[1] == "gen1" {
config := new(cwtchserver.Config)
id, pk := primitives.InitializeEphemeralIdentity()
@ -40,11 +44,8 @@ func main() {
config.PublicKey = id.PublicKey()
config.TokenServerPrivateKey = tpk
config.TokenServerPublicKey = tid.PublicKey()
config.MaxBufferLines = 100000
config.ServerReporting = cwtchserver.Reporting{
LogMetricsToFile: true,
ReportingGroupID: "",
ReportingServerAddr: "",
LogMetricsToFile: true,
}
config.ConfigDir = "."
config.FilePath = cwtchserver.ServerConfigFile
@ -53,14 +54,19 @@ func main() {
return
}
serverConfig, err := cwtchserver.LoadCreateDefaultConfigFile(configDir, cwtchserver.ServerConfigFile, false, "")
disableMetrics := *flagDisableMetrics
if os.Getenv("DISABLE_METRICS") != "" {
disableMetrics = true
}
serverConfig, err := cwtchserver.LoadCreateDefaultConfigFile(configDir, cwtchserver.ServerConfigFile, false, "", !disableMetrics)
if err != nil {
log.Errorf("Could not load/create config file: %s\n", err)
return
}
serverConfig.ServerReporting.LogMetricsToFile = !disableMetrics
// we don't need real randomness for the port, just to avoid a possible conflict...
mrand.Seed(int64(time.Now().Nanosecond()))
controlPort := mrand.Intn(1000) + 9052
r := mrand.New(mrand.NewSource(int64(time.Now().Nanosecond())))
controlPort := r.Intn(1000) + 9052
// generate a random password
key := make([]byte, 64)
@ -70,9 +76,8 @@ func main() {
}
os.MkdirAll("tordir/tor", 0700)
tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).Build("./tordir/tor/torrc")
acn, err := tor.NewTorACNWithAuth("tordir", "", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).WithSocksPort(controlPort + 1).Build("./tordir/tor/torrc")
acn, err := tor.NewTorACNWithAuth("tordir", "", "tordir/tor", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
if err != nil {
log.Errorf("\nError connecting to Tor: %v\n", err)
os.Exit(1)
@ -87,7 +92,7 @@ func main() {
if *flagExportServer {
// Todo: change all to server export
ioutil.WriteFile(path.Join(serverConfig.ConfigDir, "serverbundle"), []byte(server.TofuBundle()), 0600)
os.WriteFile(path.Join(serverConfig.ConfigDir, "serverbundle"), []byte(server.ServerBundle()), 0600)
}
// Graceful Stop
@ -100,8 +105,27 @@ func main() {
os.Exit(1)
}()
server.Run(acn)
running := false
lastStatus := -2
for {
status, msg := acn.GetBootstrapStatus()
if status == 100 && !running {
log.Infoln("ACN is online, Running Server")
server.Run(acn)
running = true
}
if status != 100 {
if running {
log.Infoln("ACN is offline, Stopping Server")
server.Stop()
running = false
} else {
if lastStatus != status {
log.Infof("ACN booting... Status %v%%: %v\n", status, msg)
lastStatus = status
}
}
}
time.Sleep(time.Second)
}
}

25
go.mod
View File

@ -1,14 +1,23 @@
module git.openprivacy.ca/cwtch.im/server
go 1.14
go 1.20
require (
cwtch.im/cwtch v0.12.2
git.openprivacy.ca/cwtch.im/tapir v0.4.9
git.openprivacy.ca/openprivacy/connectivity v1.5.0
cwtch.im/cwtch v0.27.0
git.openprivacy.ca/cwtch.im/tapir v0.6.0
git.openprivacy.ca/openprivacy/connectivity v1.11.0
git.openprivacy.ca/openprivacy/log v1.0.3
github.com/gtank/ristretto255 v0.1.2
github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c
github.com/mattn/go-sqlite3 v1.14.7
github.com/struCoder/pidusage v0.2.1
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
)
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d
)
require (
filippo.io/edwards25519 v1.0.0 // indirect
git.openprivacy.ca/openprivacy/bine v0.0.5 // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect
)

113
go.sum
View File

@ -1,114 +1,47 @@
cwtch.im/cwtch v0.7.3 h1:3f2Q2XFgNtq8hVbEz7K++exS1gDsO+zmSO7X7Fqz8Mo=
cwtch.im/cwtch v0.7.3/go.mod h1:S0JRLSTwFM+kRrpOPKgUQn/loKe/fc6lLDnOFopRKq8=
cwtch.im/cwtch v0.8.0 h1:QDRaDBTXefFRPZPqUMtxoNhOcgXv0rl0bGjysSOmJX0=
cwtch.im/cwtch v0.8.0/go.mod h1:+SY/4ueF1U7mK+CX8hZFbtd+GC1lx/cReo110KgtQAw=
cwtch.im/cwtch v0.8.5 h1:W67jAF2oRwqWytbZEv1UeCqW0cU2x69tgUw8iy27xFA=
cwtch.im/cwtch v0.8.5/go.mod h1:5GHxaaeVnKeXSU64IvtCKzkqhU8DRiLoVM+tiBT8kkc=
cwtch.im/cwtch v0.12.2 h1:I+ndKadCRCITw4SPbd+1cpRv+z/7iHjjTUv8OzRwTrE=
cwtch.im/cwtch v0.12.2/go.mod h1:QpTkQK7MqNt0dQK9/pBk5VpkvFhy6xuoxJIn401B8fM=
filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU=
filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
git.openprivacy.ca/cwtch.im/tapir v0.4.0 h1:clG8uORt0NKEhT4P+Dpw1pzyUuYzYBMevGqn2pciKk8=
git.openprivacy.ca/cwtch.im/tapir v0.4.0/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
git.openprivacy.ca/cwtch.im/tapir v0.4.1 h1:9LMpQX41IzecNNlRc1FZKXHg6wlFss679tFsa3vzb3Y=
git.openprivacy.ca/cwtch.im/tapir v0.4.1/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
git.openprivacy.ca/cwtch.im/tapir v0.4.2 h1:bxMWZnVJXX4dqqOFS7ELW4iFkVL4GS8wiRkjRv5rJe8=
git.openprivacy.ca/cwtch.im/tapir v0.4.2/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
git.openprivacy.ca/cwtch.im/tapir v0.4.4 h1:KyuTVmr9GYptTCeR7JDODjmhBBbnIBf9V3NSC4+6bHc=
git.openprivacy.ca/cwtch.im/tapir v0.4.4/go.mod h1:qMFTdmDZITc1BLP1jSW0gVpLmvpg+Zjsh5ek8StwbFE=
git.openprivacy.ca/cwtch.im/tapir v0.4.9 h1:LXonlztwvI1F1++0IyomIcDH1/Bxzo+oN8YjGonNvjM=
git.openprivacy.ca/cwtch.im/tapir v0.4.9/go.mod h1:p4bHo3DAO8wwimU6JAeZXbfPQ4jnoA2bV+4YvknWTNQ=
git.openprivacy.ca/openprivacy/bine v0.0.4 h1:CO7EkGyz+jegZ4ap8g5NWRuDHA/56KKvGySR6OBPW+c=
git.openprivacy.ca/openprivacy/bine v0.0.4/go.mod h1:13ZqhKyqakDsN/ZkQkIGNULsmLyqtXc46XBcnuXm/mU=
git.openprivacy.ca/openprivacy/connectivity v1.4.3 h1:i2Ad/U9FlL9dKr2bhRck7lJ8NoWyGtoEfUwoCyMT0fU=
git.openprivacy.ca/openprivacy/connectivity v1.4.3/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
git.openprivacy.ca/openprivacy/connectivity v1.4.5 h1:UYMdCWPzEAP7LbqdMXGNXmfKjWlvfnKdmewBtnbgQRI=
git.openprivacy.ca/openprivacy/connectivity v1.4.5/go.mod h1:JVRCIdL+lAG6ohBFWiKeC/MN42nnC0sfFszR9XG6vPQ=
git.openprivacy.ca/openprivacy/connectivity v1.5.0 h1:ZxsR/ZaVKXIkD2x6FlajZn62ciNQjamrI4i/5xIpdoQ=
git.openprivacy.ca/openprivacy/connectivity v1.5.0/go.mod h1:UjQiGBnWbotmBzIw59B8H6efwDadjkKzm3RPT1UaIRw=
git.openprivacy.ca/openprivacy/log v1.0.1/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
git.openprivacy.ca/openprivacy/log v1.0.2 h1:HLP4wsw4ljczFAelYnbObIs821z+jgMPCe8uODPnGQM=
git.openprivacy.ca/openprivacy/log v1.0.2/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
cwtch.im/cwtch v0.27.0 h1:MkNIZ+pT5ZwiGlKHk20GMUTlzBsmCWJEibbj6hr+WHw=
cwtch.im/cwtch v0.27.0/go.mod h1:A3i92aFuhyHI2DYO2Qnvl5iqEw0Cox22pdiypHnMOy4=
filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
git.openprivacy.ca/cwtch.im/tapir v0.6.0 h1:TtnKjxitkIDMM7Qn0n/u+mOHRLJzuQUYjYRu5n0/QFY=
git.openprivacy.ca/cwtch.im/tapir v0.6.0/go.mod h1:iQIq4y7N+DuP3CxyG66WNEC/d6vzh+wXvvOmelB+KoY=
git.openprivacy.ca/openprivacy/bine v0.0.5 h1:DJs5gqw3SkvLSgRDvroqJxZ7F+YsbxbBRg5t0rU5gYE=
git.openprivacy.ca/openprivacy/bine v0.0.5/go.mod h1:fwdeq6RO08WDkV0k7HfArsjRvurVULoUQmT//iaABZM=
git.openprivacy.ca/openprivacy/connectivity v1.11.0 h1:roASjaFtQLu+HdH5fa2wx6F00NL3YsUTlmXBJh8aLZk=
git.openprivacy.ca/openprivacy/connectivity v1.11.0/go.mod h1:OQO1+7OIz/jLxDrorEMzvZA6SEbpbDyLGpjoFqT3z1Y=
git.openprivacy.ca/openprivacy/log v1.0.3 h1:E/PMm4LY+Q9s3aDpfySfEDq/vYQontlvNj/scrPaga0=
git.openprivacy.ca/openprivacy/log v1.0.3/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is=
github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc=
github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c h1:gkfmnY4Rlt3VINCo4uKdpvngiibQyoENVj5Q88sxXhE=
github.com/gtank/ristretto255 v0.1.3-0.20210930101514-6bb39798585c/go.mod h1:tDPFhGdt3hJWqtKwx57i9baiB1Cj0yAg22VOPUqm5vY=
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0=
github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b h1:QrHweqAtyJ9EwCaGHBu1fghwxIPiopAHV06JlXrMHjk=
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b/go.mod h1:xxLb2ip6sSUts3g1irPVHyk/DGslwQsNOo9I7smJfNU=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/struCoder/pidusage v0.1.3 h1:pZcSa6asBE38TJtW0Nui6GeCjLTpaT/jAnNP7dUTLSQ=
github.com/struCoder/pidusage v0.1.3/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI=
github.com/struCoder/pidusage v0.2.1 h1:dFiEgUDkubeIj0XA1NpQ6+8LQmKrLi7NiIQl86E6BoY=
github.com/struCoder/pidusage v0.2.1/go.mod h1:bewtP2KUA1TBUyza5+/PCpSQ6sc/H6jJbIKAzqW86BA=
github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI=
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d h1:3qF+Z8Hkrw9sOhrFHti9TlB1Hkac1x+DNRkv0XQiFjo=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 h1:UiNENfZ8gDvpiWw7IpOMQ27spWmThO1RwwdQVbJahJM=
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -9,9 +9,14 @@ import (
"time"
)
// Fields must be in this order because go compiler has problems with 64bit fields on 32 bit arches (arm32 raspberry pi):
//
// https://git.openprivacy.ca/cwtch.im/server/pulls/30
// https://github.com/golang/go/issues/599
// https://github.com/census-instrumentation/opencensus-go/issues/587
type counter struct {
startTime time.Time
count uint64
startTime time.Time
}
// Counter providers a threadsafe counter to use for storing long running counts
@ -71,6 +76,7 @@ const (
Average
)
// TODO port to SQLite for persistence between runs?
type monitorHistory struct {
monitorType MonitorType
monitorAccumulation MonitorAccumulation
@ -150,13 +156,25 @@ func (mh *monitorHistory) Months() []float64 {
return mh.returnCopy(mh.perMonthForYear[:])
}
const timeDay = time.Hour * 24
const timeWeek = timeDay * 7
const timeMonth = timeDay * 28
func (mh *monitorHistory) Report(w *bufio.Writer) {
mh.lock.Lock()
fmt.Fprintln(w, "Minutes:", reportLine(mh.monitorType, mh.perMinutePerHour[:]))
fmt.Fprintln(w, "Hours: ", reportLine(mh.monitorType, mh.perHourForDay[:]))
fmt.Fprintln(w, "Days: ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
fmt.Fprintln(w, "Weeks: ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
if time.Since(mh.starttime) >= time.Hour {
fmt.Fprintln(w, "Hours: ", reportLine(mh.monitorType, mh.perHourForDay[:]))
}
if time.Since(mh.starttime) >= timeDay {
fmt.Fprintln(w, "Days: ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
}
if time.Since(mh.starttime) >= timeWeek {
fmt.Fprintln(w, "Weeks: ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
}
if time.Since(mh.starttime) >= timeMonth {
fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
}
mh.lock.Unlock()
}
@ -179,9 +197,7 @@ func reportLine(t MonitorType, array []float64) string {
func (mh *monitorHistory) returnCopy(slice []float64) []float64 {
retSlice := make([]float64, len(slice))
mh.lock.Lock()
for i, v := range slice {
retSlice[i] = v
}
copy(retSlice, slice)
mh.lock.Unlock()
return retSlice
}
@ -217,24 +233,24 @@ func (mh *monitorHistory) monitorThread() {
case <-time.After(time.Minute):
mh.lock.Lock()
minuteAvg := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
minuteAcc := rotateAndAccumulate(mh.perMinutePerHour[:], mh.monitor(), mh.monitorAccumulation)
if time.Now().Sub(mh.timeLastHourRotate) > time.Hour {
rotateAndAccumulate(mh.perHourForDay[:], minuteAvg, mh.monitorAccumulation)
if time.Since(mh.timeLastHourRotate) > time.Hour {
rotateAndAccumulate(mh.perHourForDay[:], minuteAcc, mh.monitorAccumulation)
mh.timeLastHourRotate = time.Now()
}
if time.Now().Sub(mh.timeLastDayRotate) > time.Hour*24 {
if time.Since(mh.timeLastDayRotate) > time.Hour*24 {
rotateAndAccumulate(mh.perDayForWeek[:], accumulate(mh.perHourForDay[:], mh.monitorAccumulation), mh.monitorAccumulation)
mh.timeLastDayRotate = time.Now()
}
if time.Now().Sub(mh.timeLastWeekRotate) > time.Hour*24*7 {
if time.Since(mh.timeLastWeekRotate) > time.Hour*24*7 {
rotateAndAccumulate(mh.perWeekForMonth[:], accumulate(mh.perDayForWeek[:], mh.monitorAccumulation), mh.monitorAccumulation)
mh.timeLastWeekRotate = time.Now()
}
if time.Now().Sub(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
if time.Since(mh.timeLastMonthRotate) > time.Hour*24*7*4 {
rotateAndAccumulate(mh.perMonthForYear[:], accumulate(mh.perWeekForMonth[:], mh.monitorAccumulation), mh.monitorAccumulation)
mh.timeLastMonthRotate = time.Now()
}

View File

@ -5,9 +5,9 @@ import (
"fmt"
"git.openprivacy.ca/cwtch.im/tapir"
"git.openprivacy.ca/openprivacy/log"
"github.com/struCoder/pidusage"
"os"
"path"
"runtime"
"sync"
"time"
)
@ -16,52 +16,48 @@ const (
reportFile = "serverMonitorReport.txt"
)
type MessageCountFn func() int
// Monitors is a package of metrics for a Cwtch Server including message count, CPU, Mem, and conns
type Monitors struct {
MessageCounter Counter
TotalMessageCounter Counter
Messages MonitorHistory
CPU MonitorHistory
Memory MonitorHistory
ClientConns MonitorHistory
starttime time.Time
breakChannel chan bool
log bool
configDir string
MessageCounter Counter
Messages MonitorHistory
Memory MonitorHistory
ClientConns MonitorHistory
messageCountFn MessageCountFn
starttime time.Time
breakChannel chan bool
log bool
configDir string
running bool
lock sync.Mutex
}
func bToMb(b uint64) uint64 {
return b / 1024 / 1024
}
// Start initializes a Monitors's monitors
func (mp *Monitors) Start(ts tapir.Service, configDir string, log bool) {
mp.log = log
func (mp *Monitors) Start(ts tapir.Service, mcfn MessageCountFn, configDir string, doLogging bool) {
mp.log = doLogging
mp.configDir = configDir
mp.starttime = time.Now()
mp.breakChannel = make(chan bool)
mp.MessageCounter = NewCounter()
mp.messageCountFn = mcfn
// Maintain a count of total messages
mp.TotalMessageCounter = NewCounter()
mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) {
c = float64(mp.MessageCounter.Count())
mp.TotalMessageCounter.Add(int(c))
mp.MessageCounter.Reset()
return
})
var pidUsageLock sync.Mutex
mp.CPU = NewMonitorHistory(Percent, Average, func() float64 {
pidUsageLock.Lock()
defer pidUsageLock.Unlock()
sysInfo, _ := pidusage.GetStat(os.Getpid())
return float64(sysInfo.CPU)
})
mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
pidUsageLock.Lock()
defer pidUsageLock.Unlock()
sysInfo, _ := pidusage.GetStat(os.Getpid())
return float64(sysInfo.Memory)
var m runtime.MemStats
runtime.ReadMemStats(&m)
return float64(bToMb(m.Sys))
})
// TODO: replace with ts.
mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ts.Metrics().ConnectionCount) })
if mp.log {
@ -70,16 +66,34 @@ func (mp *Monitors) Start(ts tapir.Service, configDir string, log bool) {
}
func (mp *Monitors) run() {
mp.running = true
for {
select {
case <-time.After(time.Minute):
mp.lock.Lock()
mp.report()
mp.lock.Unlock()
case <-mp.breakChannel:
mp.lock.Lock()
mp.running = false
mp.lock.Unlock()
return
}
}
}
func FormatDuration(ts time.Duration) string {
const (
Day = 24 * time.Hour
)
d := ts / Day
ts = ts % Day
h := ts / time.Hour
ts = ts % time.Hour
m := ts / time.Minute
return fmt.Sprintf("%dd%dh%dm", d, h, m)
}
func (mp *Monitors) report() {
f, err := os.Create(path.Join(mp.configDir, reportFile))
if err != nil {
@ -90,18 +104,16 @@ func (mp *Monitors) report() {
w := bufio.NewWriter(f)
fmt.Fprintf(w, "Uptime: %v\n\n", time.Now().Sub(mp.starttime))
fmt.Fprintf(w, "Uptime: %v \n", FormatDuration(time.Since(mp.starttime)))
fmt.Fprintf(w, "Total Messages: %v \n\n", mp.messageCountFn())
fmt.Fprintln(w, "messages:")
fmt.Fprintln(w, "Messages:")
mp.Messages.Report(w)
fmt.Fprintln(w, "\nClient Connections:")
mp.ClientConns.Report(w)
fmt.Fprintln(w, "\nCPU:")
mp.CPU.Report(w)
fmt.Fprintln(w, "\nMemory:")
fmt.Fprintln(w, "\nSys Memory:")
mp.Memory.Report(w)
w.Flush()
@ -109,11 +121,15 @@ func (mp *Monitors) report() {
// Stop stops all the monitors in a Monitors
func (mp *Monitors) Stop() {
if mp.log {
mp.breakChannel <- true
mp.lock.Lock()
running := mp.running
mp.lock.Unlock()
if running {
if mp.log {
mp.breakChannel <- true
}
mp.Messages.Stop()
mp.Memory.Stop()
mp.ClientConns.Stop()
}
mp.Messages.Stop()
mp.CPU.Stop()
mp.Memory.Stop()
mp.ClientConns.Stop()
}

31
metrics/monitors_test.go Normal file
View File

@ -0,0 +1,31 @@
package metrics
import (
tor2 "git.openprivacy.ca/cwtch.im/tapir/networks/tor"
"git.openprivacy.ca/openprivacy/log"
"os"
"path/filepath"
"testing"
"time"
)
func TestMonitors(t *testing.T) {
log.SetLevel(log.LevelInfo)
os.RemoveAll("testLog")
os.Mkdir("testLog", 0700)
service := new(tor2.BaseOnionService)
mp := Monitors{}
mp.Start(service, func() int { return 1 }, "testLog", true)
mp.MessageCounter.Add(1)
log.Infof("sleeping for minute to give to for monitors to trigger...")
// wait a minute for it to trigger
time.Sleep(62 * time.Second)
// it didn't segfault? that's good, did it create a log file?
if _, err := os.Stat(filepath.Join("testLog", "serverMonitorReport.txt")); err != nil {
t.Errorf("serverMonitorReport.txt not generated")
}
mp.Stop()
os.RemoveAll("testLog")
}

105
server.go
View File

@ -42,21 +42,22 @@ type Server interface {
TofuBundle() string
GetAttribute(string) string
SetAttribute(string, string)
SetMonitorLogging(bool)
}
type server struct {
service tapir.Service
config *Config
metricsPack metrics.Monitors
tokenTapirService tapir.Service
tokenServer *privacypass.TokenServer
tokenService primitives.Identity
tokenServicePrivKey ed25519.PrivateKey
tokenServiceStopped bool
onionServiceStopped bool
running bool
existingMessageCount int
lock sync.RWMutex
config *Config
service tapir.Service
messageStore storage.MessageStoreInterface
metricsPack metrics.Monitors
tokenTapirService tapir.Service
tokenServer *privacypass.TokenServer
tokenService primitives.Identity
tokenServicePrivKey ed25519.PrivateKey
tokenServiceStopped bool
onionServiceStopped bool
running bool
lock sync.RWMutex
}
// NewServer creates and configures a new server based on the supplied configuration
@ -78,6 +79,21 @@ func (s *server) Identity() primitives.Identity {
return s.config.Identity()
}
// helper fn to pass to metrics
func (s *server) getStorageTotalMessageCount() int {
if s.messageStore != nil {
return s.messageStore.MessagesCount()
}
return 0
}
// helper fn to pass to storage
func (s *server) incMessageCount() {
if s.metricsPack.MessageCounter != nil {
s.metricsPack.MessageCounter.Add(1)
}
}
// Run starts a server with the given privateKey
func (s *server) Run(acn connectivity.ACN) error {
s.lock.Lock()
@ -87,22 +103,21 @@ func (s *server) Run(acn connectivity.ACN) error {
}
identity := primitives.InitializeIdentity("", &s.config.PrivateKey, &s.config.PublicKey)
var service tapir.Service
service = new(tor2.BaseOnionService)
service := new(tor2.BaseOnionService)
service.Init(acn, s.config.PrivateKey, &identity)
s.service = service
log.Infof("cwtch server running on cwtch:%s\n", s.Onion())
s.metricsPack.Start(service, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
ms, err := storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.metricsPack.MessageCounter)
if s.config.ServerReporting.LogMetricsToFile {
s.metricsPack.Start(service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
}
var err error
s.messageStore, err = storage.InitializeSqliteMessageStore(path.Join(s.config.ConfigDir, "cwtch.messages"), s.config.GetMaxMessages(), s.incMessageCount)
if err != nil {
return fmt.Errorf("could not open database: %v", err)
}
// Needed because we only collect metrics on a per-session basis
// TODO fix metrics so they persist across sessions?
s.existingMessageCount = len(ms.FetchMessages())
s.tokenTapirService = new(tor2.BaseOnionService)
s.tokenTapirService.Init(acn, s.tokenServicePrivKey, &s.tokenService)
tokenApplication := new(applications.TokenApplication)
@ -115,7 +130,7 @@ func (s *server) Run(acn connectivity.ACN) error {
s.tokenServiceStopped = true
}()
go func() {
s.service.Listen(NewTokenBoardServer(ms, s.tokenServer))
s.service.Listen(NewTokenBoardServer(s.messageStore, s.tokenServer))
s.onionServiceStopped = true
}()
@ -138,20 +153,21 @@ func (s *server) KeyBundle() *model.KeyBundle {
func (s *server) CheckStatus() (bool, error) {
s.lock.RLock()
defer s.lock.RUnlock()
if s.onionServiceStopped == true || s.tokenServiceStopped == true {
if s.onionServiceStopped || s.tokenServiceStopped {
return s.running, fmt.Errorf("one of more server components are down: onion:%v token service: %v", s.onionServiceStopped, s.tokenServiceStopped)
}
return s.running, nil
}
// Stop turns off the server so it cannot receive connections and frees most resourses.
// The server is still in a reRunable state and tokenServer still has an active persistance
// The server is still in a reRunable state and tokenServer still has an active persistence
func (s *server) Stop() {
log.Infof("Shutting down server")
s.lock.Lock()
defer s.lock.Unlock()
if s.running {
s.service.Shutdown()
s.messageStore.Close()
s.tokenTapirService.Shutdown()
log.Infof("Closing Token server Database...")
@ -160,7 +176,7 @@ func (s *server) Stop() {
}
}
// Destroy frees the last of the resources the server has active (toklenServer persistance) leaving it un-re-runable and completely shutdown
// Destroy frees the last of the resources the server has active (tokenServer persistence) leaving it un-re-runable and completely shutdown
func (s *server) Destroy() {
s.Stop()
s.lock.Lock()
@ -170,21 +186,20 @@ func (s *server) Destroy() {
// Statistics is an encapsulation of information about the server that an operator might want to know at a glance.
type Statistics struct {
TotalMessages int
TotalMessages int
TotalConnections int
}
// GetStatistics is a stub method for providing some high level information about
// the server operation to bundling applications (e.g. the UI)
func (s *server) GetStatistics() Statistics {
// TODO Statistics from Metrics is very awkward. Metrics needs an overhaul to make safe
total := s.existingMessageCount
if s.metricsPack.TotalMessageCounter != nil {
total += s.metricsPack.TotalMessageCounter.Count()
}
return Statistics{
TotalMessages: total,
if s.running {
return Statistics{
TotalMessages: s.messageStore.MessagesCount(),
TotalConnections: s.service.Metrics().ConnectionCount,
}
}
return Statistics{}
}
func (s *server) Delete(password string) error {
@ -192,7 +207,7 @@ func (s *server) Delete(password string) error {
if s.config.Encrypted && !s.config.CheckPassword(password) {
s.lock.Unlock()
log.Errorf("encryped and checkpassword failed")
return errors.New("Cannot delete server, passwords do not match")
return errors.New("cannot delete server, passwords do not match")
}
s.lock.Unlock()
s.Destroy()
@ -230,3 +245,25 @@ func (s *server) GetAttribute(key string) string {
func (s *server) SetAttribute(key, val string) {
s.config.SetAttribute(key, val)
}
// GetMessageCap gets a server's MaxStorageMBs value
func (s *server) GetMaxStorageMBs() int {
return s.config.GetMaxMessageMBs()
}
// SetMaxStorageMBs sets a server's MaxStorageMBs and sets MaxMessages for storage (which can trigger a prune)
func (s *server) SetMaxStorageMBs(val int) {
s.config.SetMaxMessageMBs(val)
s.messageStore.SetMessageCap(s.config.GetMaxMessages())
}
// SetMonitorLogging turns on or off the monitor logging suite, and logging to a file in the server dir
func (s *server) SetMonitorLogging(do bool) {
s.config.ServerReporting.LogMetricsToFile = do
s.config.Save()
if do {
s.metricsPack.Start(s.service, s.getStorageTotalMessageCount, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
} else {
s.metricsPack.Stop()
}
}

View File

@ -2,23 +2,19 @@ package server
import (
"crypto/rand"
v1 "cwtch.im/cwtch/storage/v1"
"encoding/json"
"git.openprivacy.ca/cwtch.im/server/storage"
"git.openprivacy.ca/cwtch.im/tapir/primitives"
"git.openprivacy.ca/openprivacy/connectivity/tor"
"git.openprivacy.ca/openprivacy/log"
"github.com/gtank/ristretto255"
"golang.org/x/crypto/ed25519"
"io/ioutil"
"os"
"path"
"sync"
)
const (
// SaltFile is the standard filename to store an encrypted config's SALT under beside it
SaltFile = "SALT"
// AttrAutostart is the attribute key for autostart setting
AttrAutostart = "autostart"
@ -42,18 +38,18 @@ const (
// Reporting is a struct for storing a the config a server needs to be a peer, and connect to a group to report
type Reporting struct {
LogMetricsToFile bool `json:"logMetricsToFile"`
ReportingGroupID string `json:"reportingGroupId"`
ReportingServerAddr string `json:"reportingServerAddr"`
LogMetricsToFile bool `json:"logMetricsToFile"`
}
// messages are ~4kb of storage
const MessagesPerMB = 250
// Config is a struct for storing basic server configuration
type Config struct {
ConfigDir string `json:"-"`
FilePath string `json:"-"`
Encrypted bool `json:"-"`
key [32]byte
MaxBufferLines int `json:"maxBufferLines"`
ConfigDir string `json:"-"`
FilePath string `json:"-"`
Encrypted bool `json:"-"`
key [32]byte
PublicKey ed25519.PublicKey `json:"publicKey"`
PrivateKey ed25519.PrivateKey `json:"privateKey"`
@ -67,8 +63,12 @@ type Config struct {
Attributes map[string]string `json:"attributes"`
// messages are ~4kb of storage
// -1 == infinite
MaxStorageMBs int `json:"maxStorageMBs"`
lock sync.Mutex
encFileStore v1.FileStore
encFileStore storage.FileStore
}
// Identity returns an encapsulation of the servers keys
@ -90,13 +90,11 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
config.PublicKey = id.PublicKey()
config.TokenServerPrivateKey = tpk
config.TokenServerPublicKey = tid.PublicKey()
config.MaxBufferLines = 100000
config.ServerReporting = Reporting{
LogMetricsToFile: true,
ReportingGroupID: "",
ReportingServerAddr: "",
LogMetricsToFile: false,
}
config.Attributes[AttrAutostart] = "false"
config.MaxStorageMBs = -1
k := new(ristretto255.Scalar)
b := make([]byte, 64)
@ -105,34 +103,35 @@ func initDefaultConfig(configDir, filename string, encrypted bool) *Config {
// unable to generate secure random numbers
panic("unable to generate secure random numbers")
}
k.FromUniformBytes(b)
k.SetUniformBytes(b)
config.TokenServiceK = *k
return config
}
// LoadCreateDefaultConfigFile loads a Config from or creates a default config and saves it to a json file specified by filename
// if the encrypted flag is true the config is store encrypted by password
func LoadCreateDefaultConfigFile(configDir, filename string, encrypted bool, password string) (*Config, error) {
func LoadCreateDefaultConfigFile(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
if _, err := os.Stat(path.Join(configDir, filename)); os.IsNotExist(err) {
return CreateConfig(configDir, filename, encrypted, password)
return CreateConfig(configDir, filename, encrypted, password, defaultLogToFile)
}
return LoadConfig(configDir, filename, encrypted, password)
}
// CreateConfig creates a default config and saves it to a json file specified by filename
// if the encrypted flag is true the config is store encrypted by password
func CreateConfig(configDir, filename string, encrypted bool, password string) (*Config, error) {
func CreateConfig(configDir, filename string, encrypted bool, password string, defaultLogToFile bool) (*Config, error) {
log.Debugf("CreateConfig for server with configDir: %s\n", configDir)
os.MkdirAll(configDir, 0700)
config := initDefaultConfig(configDir, filename, encrypted)
config.ServerReporting.LogMetricsToFile = defaultLogToFile
if encrypted {
key, _, err := v1.InitV1Directory(configDir, password)
key, _, err := storage.InitV1Directory(configDir, password)
if err != nil {
log.Errorf("could not create server directory: %s", err)
return nil, err
}
config.key = key
config.encFileStore = v1.NewFileStore(configDir, ServerConfigFile, key)
config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, key)
}
config.Save()
@ -145,12 +144,12 @@ func LoadConfig(configDir, filename string, encrypted bool, password string) (*C
var raw []byte
var err error
if encrypted {
salt, err := ioutil.ReadFile(path.Join(configDir, SaltFile))
salt, err := os.ReadFile(path.Join(configDir, storage.SaltFile))
if err != nil {
return nil, err
}
config.key = v1.CreateKey(password, salt)
config.encFileStore = v1.NewFileStore(configDir, ServerConfigFile, config.key)
config.key = storage.CreateKey(password, salt)
config.encFileStore = storage.NewFileStore(configDir, ServerConfigFile, config.key)
raw, err = config.encFileStore.Read()
if err != nil {
// Not an error to log as load config is called blindly across all dirs with a password to see what it applies to
@ -158,7 +157,7 @@ func LoadConfig(configDir, filename string, encrypted bool, password string) (*C
return nil, err
}
} else {
raw, err = ioutil.ReadFile(path.Join(configDir, filename))
raw, err = os.ReadFile(path.Join(configDir, filename))
if err != nil {
return nil, err
}
@ -182,18 +181,18 @@ func (config *Config) Save() error {
if config.Encrypted {
return config.encFileStore.Write(bytes)
}
return ioutil.WriteFile(path.Join(config.ConfigDir, config.FilePath), bytes, 0600)
return os.WriteFile(path.Join(config.ConfigDir, config.FilePath), bytes, 0600)
}
// CheckPassword returns true if the given password produces the same key as the current stored key, otherwise false.
func (config *Config) CheckPassword(checkpass string) bool {
config.lock.Lock()
defer config.lock.Unlock()
salt, err := ioutil.ReadFile(path.Join(config.ConfigDir, SaltFile))
salt, err := os.ReadFile(path.Join(config.ConfigDir, storage.SaltFile))
if err != nil {
return false
}
oldkey := v1.CreateKey(checkpass, salt[:])
oldkey := storage.CreateKey(checkpass, salt[:])
return oldkey == config.key
}
@ -218,3 +217,26 @@ func (config *Config) GetAttribute(key string) string {
defer config.lock.Unlock()
return config.Attributes[key]
}
// GetMaxMessages returns the config setting for Max messages converting from MaxMB to messages
// or -1 for infinite
func (config *Config) GetMaxMessages() int {
config.lock.Lock()
defer config.lock.Unlock()
if config.MaxStorageMBs == -1 {
return -1
}
return config.MaxStorageMBs * MessagesPerMB
}
func (config *Config) GetMaxMessageMBs() int {
config.lock.Lock()
defer config.lock.Unlock()
return config.MaxStorageMBs
}
func (config *Config) SetMaxMessageMBs(newval int) {
config.lock.Lock()
defer config.lock.Unlock()
config.MaxStorageMBs = newval
}

View File

@ -1,12 +1,12 @@
package server
import (
"cwtch.im/cwtch/model"
"errors"
"fmt"
"git.openprivacy.ca/cwtch.im/server/storage"
"git.openprivacy.ca/openprivacy/connectivity"
"git.openprivacy.ca/openprivacy/log"
"io/ioutil"
"os"
"path"
"sync"
)
@ -47,7 +47,7 @@ func NewServers(acn connectivity.ACN, directory string) Servers {
func (s *servers) LoadServers(password string) ([]string, error) {
s.lock.Lock()
defer s.lock.Unlock()
dirs, err := ioutil.ReadDir(s.directory)
dirs, err := os.ReadDir(s.directory)
if err != nil {
return nil, fmt.Errorf("error: cannot read server directory: %v", err)
}
@ -63,15 +63,14 @@ func (s *servers) LoadServers(password string) ([]string, error) {
}
}
}
log.Infof("LoadServers returning: %s\n", loadedServers)
return loadedServers, nil
}
// CreateServer creates a new server and stores it, also returns an interface to it
func (s *servers) CreateServer(password string) (Server, error) {
newLocalID := model.GenerateRandomID()
newLocalID := storage.GenerateRandomID()
directory := path.Join(s.directory, newLocalID)
config, err := CreateConfig(directory, ServerConfigFile, true, password)
config, err := CreateConfig(directory, ServerConfigFile, true, password, false)
if err != nil {
return nil, err
}

156
storage/file_store.go Normal file
View File

@ -0,0 +1,156 @@
package storage
import (
"crypto/rand"
"encoding/hex"
"errors"
"git.openprivacy.ca/openprivacy/log"
"golang.org/x/crypto/nacl/secretbox"
"golang.org/x/crypto/pbkdf2"
"golang.org/x/crypto/sha3"
"io"
"os"
"path"
)
// SaltFile is the standard filename to store an encrypted config's SALT under beside it
const SaltFile = "SALT"
const version = "1"
const versionFile = "VERSION"
// GenerateRandomID generates a random 16 byte hex id code
func GenerateRandomID() string {
randBytes := make([]byte, 16)
rand.Read(randBytes)
return hex.EncodeToString(randBytes)
}
// InitV1Directory generates a key and salt from a password, writes a SALT and VERSION file and returns the key and salt
func InitV1Directory(directory, password string) ([32]byte, [128]byte, error) {
os.Mkdir(directory, 0700)
key, salt, err := CreateKeySalt(password)
if err != nil {
log.Errorf("Could not create key for profile store from password: %v\n", err)
return [32]byte{}, [128]byte{}, err
}
if err = os.WriteFile(path.Join(directory, versionFile), []byte(version), 0600); err != nil {
log.Errorf("Could not write version file: %v", err)
return [32]byte{}, [128]byte{}, err
}
if err = os.WriteFile(path.Join(directory, SaltFile), salt[:], 0600); err != nil {
log.Errorf("Could not write salt file: %v", err)
return [32]byte{}, [128]byte{}, err
}
return key, salt, nil
}
// fileStore stores a cwtchPeer in an encrypted file
type fileStore struct {
directory string
filename string
key [32]byte
}
// FileStore is a primitive around storing encrypted files
type FileStore interface {
Write([]byte) error
Read() ([]byte, error)
Delete()
ChangeKey(newkey [32]byte)
}
// NewFileStore instantiates a fileStore given a filename and a password
func NewFileStore(directory string, filename string, key [32]byte) FileStore {
filestore := new(fileStore)
filestore.key = key
filestore.filename = filename
filestore.directory = directory
return filestore
}
// CreateKeySalt derives a key and salt from a password: returns key, salt, err
func CreateKeySalt(password string) ([32]byte, [128]byte, error) {
var salt [128]byte
if _, err := io.ReadFull(rand.Reader, salt[:]); err != nil {
log.Errorf("Cannot read from random: %v\n", err)
return [32]byte{}, salt, err
}
dk := pbkdf2.Key([]byte(password), salt[:], 4096, 32, sha3.New512)
var dkr [32]byte
copy(dkr[:], dk)
return dkr, salt, nil
}
// CreateKey derives a key from a password and salt
func CreateKey(password string, salt []byte) [32]byte {
dk := pbkdf2.Key([]byte(password), salt, 4096, 32, sha3.New512)
var dkr [32]byte
copy(dkr[:], dk)
return dkr
}
// EncryptFileData encrypts the data with the supplied key
func EncryptFileData(data []byte, key [32]byte) ([]byte, error) {
var nonce [24]byte
if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
log.Errorf("Cannot read from random: %v\n", err)
return nil, err
}
encrypted := secretbox.Seal(nonce[:], data, &nonce, &key)
return encrypted, nil
}
// DecryptFile decrypts the passed ciphertext with the supplied key.
func DecryptFile(ciphertext []byte, key [32]byte) ([]byte, error) {
var decryptNonce [24]byte
copy(decryptNonce[:], ciphertext[:24])
decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &key)
if ok {
return decrypted, nil
}
return nil, errors.New("failed to decrypt")
}
// ReadEncryptedFile reads data from an encrypted file in directory with key
func ReadEncryptedFile(directory, filename string, key [32]byte) ([]byte, error) {
encryptedbytes, err := os.ReadFile(path.Join(directory, filename))
if err == nil {
return DecryptFile(encryptedbytes, key)
}
return nil, err
}
// write serializes a cwtchPeer to a file
func (fps *fileStore) Write(data []byte) error {
encryptedbytes, err := EncryptFileData(data, fps.key)
if err != nil {
return err
}
err = os.WriteFile(path.Join(fps.directory, fps.filename), encryptedbytes, 0600)
return err
}
func (fps *fileStore) Read() ([]byte, error) {
return ReadEncryptedFile(fps.directory, fps.filename, fps.key)
}
func (fps *fileStore) Delete() {
err := os.Remove(path.Join(fps.directory, fps.filename))
if err != nil {
log.Errorf("Deleting file %v\n", err)
}
}
func (fps *fileStore) ChangeKey(newkey [32]byte) {
fps.key = newkey
}

View File

@ -5,26 +5,36 @@ import (
"database/sql"
"encoding/base64"
"fmt"
"git.openprivacy.ca/cwtch.im/server/metrics"
"git.openprivacy.ca/openprivacy/log"
_ "github.com/mattn/go-sqlite3" // sqlite3 driver
"sync"
)
// MessageStoreInterface defines an interface to interact with a store of cwtch messages.
type MessageStoreInterface interface {
AddMessage(groups.EncryptedGroupMessage)
FetchMessages() []*groups.EncryptedGroupMessage
MessagesCount() int
FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage
SetMessageCap(newcap int)
Close()
}
// SqliteMessageStore is an sqlite3 backed message store
type SqliteMessageStore struct {
messageCounter metrics.Counter
database *sql.DB
incMessageCounterFn func()
messageCap int
messageCount int
countLock sync.Mutex
database *sql.DB
// Some prepared queries...
preparedInsertStatement *sql.Stmt // A Stmt is safe for concurrent use by multiple goroutines.
preparedFetchFromQuery *sql.Stmt
preparedFetchQuery *sql.Stmt
preparedCountQuery *sql.Stmt
preparedPruneStatement *sql.Stmt
}
// Close closes the underlying sqlite3 database to further changes
@ -34,9 +44,18 @@ func (s *SqliteMessageStore) Close() {
s.database.Close()
}
func (s *SqliteMessageStore) SetMessageCap(newcap int) {
s.countLock.Lock()
defer s.countLock.Unlock()
s.messageCap = newcap
s.checkPruneMessages()
}
// AddMessage implements the MessageStoreInterface AddMessage for sqlite message store
func (s *SqliteMessageStore) AddMessage(message groups.EncryptedGroupMessage) {
s.messageCounter.Add(1)
if s.incMessageCounterFn != nil {
s.incMessageCounterFn()
}
// ignore this clearly invalid message...
if len(message.Signature) == 0 {
return
@ -47,11 +66,53 @@ func (s *SqliteMessageStore) AddMessage(message groups.EncryptedGroupMessage) {
log.Errorf("%v %q", stmt, err)
return
}
s.countLock.Lock()
defer s.countLock.Unlock()
s.messageCount++
s.checkPruneMessages()
}
func (s *SqliteMessageStore) checkPruneMessages() {
if s.messageCap != -1 && s.messageCount > s.messageCap {
log.Debugf("Message Count: %d / Message Cap: %d, message cap exceeded, pruning oldest 10%...", s.messageCount, s.messageCap)
// Delete 10% of messages (and any overage if the cap was adjusted lower)
delCount := (s.messageCount - s.messageCap) + s.messageCap/10
stmt, err := s.preparedPruneStatement.Exec(delCount)
if err != nil {
log.Errorf("%v %q", stmt, err)
}
s.messageCount -= delCount
}
}
func (s *SqliteMessageStore) MessagesCount() int {
rows, err := s.preparedCountQuery.Query()
if err != nil {
log.Errorf("%v", err)
return -1
}
defer rows.Close()
result := rows.Next()
if !result {
return -1
}
var rownum int
err = rows.Scan(&rownum)
if err != nil {
log.Errorf("error fetching rows: %v", err)
return -1
}
return rownum
}
// FetchMessages implements the MessageStoreInterface FetchMessages for sqlite message store
func (s SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
rows, err := s.database.Query("SELECT id, signature,ciphertext from messages")
func (s *SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
rows, err := s.preparedFetchQuery.Query()
if err != nil {
log.Errorf("%v", err)
return nil
@ -61,10 +122,10 @@ func (s SqliteMessageStore) FetchMessages() []*groups.EncryptedGroupMessage {
}
// FetchMessagesFrom implements the MessageStoreInterface FetchMessagesFrom for sqlite message store
func (s SqliteMessageStore) FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage {
func (s *SqliteMessageStore) FetchMessagesFrom(signature []byte) []*groups.EncryptedGroupMessage {
// If signature is empty then treat this as a complete sync request
if signature == nil || len(signature) == 0 {
if len(signature) == 0 {
return s.FetchMessages()
}
@ -107,7 +168,7 @@ func (s *SqliteMessageStore) compileRows(rows *sql.Rows) []*groups.EncryptedGrou
// InitializeSqliteMessageStore creates a database `dbfile` with the necessary tables (if it doesn't already exist)
// and returns an open database
func InitializeSqliteMessageStore(dbfile string, messageCounter metrics.Counter) (*SqliteMessageStore, error) {
func InitializeSqliteMessageStore(dbfile string, messageCap int, incMessageCounterFn func()) (*SqliteMessageStore, error) {
db, err := sql.Open("sqlite3", dbfile)
if err != nil {
log.Errorf("database %v cannot be created or opened %v", dbfile, err)
@ -123,7 +184,8 @@ func InitializeSqliteMessageStore(dbfile string, messageCounter metrics.Counter)
log.Infof("Database Initialized")
slms := new(SqliteMessageStore)
slms.database = db
slms.messageCounter = messageCounter
slms.incMessageCounterFn = incMessageCounterFn
slms.messageCap = messageCap
sqlStmt = `INSERT INTO messages(signature, ciphertext) values (?,?);`
stmt, err := slms.database.Prepare(sqlStmt)
@ -133,12 +195,41 @@ func InitializeSqliteMessageStore(dbfile string, messageCounter metrics.Counter)
}
slms.preparedInsertStatement = stmt
query, err := slms.database.Prepare("SELECT id, signature,ciphertext FROM messages WHERE id>=(SELECT id FROM messages WHERE signature=(?));")
sqlStmt = "SELECT id, signature,ciphertext from messages"
query, err := slms.database.Prepare(sqlStmt)
if err != nil {
log.Errorf("%v", err)
log.Errorf("%q: %s", err, sqlStmt)
return nil, fmt.Errorf("%s: %q", sqlStmt, err)
}
slms.preparedFetchQuery = query
sqlStmt = "SELECT id, signature,ciphertext FROM messages WHERE id>=(SELECT id FROM messages WHERE signature=(?));"
query, err = slms.database.Prepare(sqlStmt)
if err != nil {
log.Errorf("%q: %s", err, sqlStmt)
return nil, fmt.Errorf("%s: %q", sqlStmt, err)
}
slms.preparedFetchFromQuery = query
sqlStmt = "SELECT COUNT(*) from messages"
stmt, err = slms.database.Prepare(sqlStmt)
if err != nil {
log.Errorf("%q: %s", err, sqlStmt)
return nil, fmt.Errorf("%s: %q", sqlStmt, err)
}
slms.preparedCountQuery = stmt
sqlStmt = "DELETE FROM messages WHERE id IN (SELECT id FROM messages ORDER BY id ASC LIMIT (?))"
stmt, err = slms.database.Prepare(sqlStmt)
if err != nil {
log.Errorf("%q: %s", err, sqlStmt)
return nil, fmt.Errorf("%s: %q", sqlStmt, err)
}
slms.preparedPruneStatement = stmt
slms.messageCount = slms.MessagesCount()
slms.checkPruneMessages()
return slms, nil
}

View File

@ -5,6 +5,7 @@ import (
"encoding/binary"
"git.openprivacy.ca/cwtch.im/server/metrics"
"git.openprivacy.ca/openprivacy/log"
_ "github.com/mattn/go-sqlite3" // sqlite3 driver
"os"
"testing"
"time"
@ -15,7 +16,7 @@ func TestMessageStore(t *testing.T) {
os.Remove(filename)
log.SetLevel(log.LevelDebug)
counter := metrics.NewCounter()
db, err := InitializeSqliteMessageStore(filename, counter)
db, err := InitializeSqliteMessageStore(filename, -1, func() { counter.Add(1) })
if err != nil {
t.Fatalf("Error: %v", err)
}

View File

@ -9,16 +9,15 @@ go list ./... | xargs go vet
echo ""
echo "Linting:"
go list ./... | xargs golint
go list ./... | staticcheck ./...
echo "Running nilaway..."
nilaway -include-pkgs="cwtch.im/server,cwtch.im/cwtch,cwtch.im/tapir,git.openprivacy.ca/openprivacy/connectivity" -exclude-file-docstrings="nolint:nilaway" ./...
echo "Time to format"
gofmt -l -s -w .
# ineffassign (https://github.com/gordonklaus/ineffassign)
echo "Checking for ineffectual assignment of errors (unchecked errors...)"
ineffassign .
# misspell (https://github.com/client9/misspell/cmd/misspell)
echo "Checking for misspelled words..."
misspell . | grep -v "vendor/" | grep -v "go.sum" | grep -v ".idea"