Use a constant-time compare

Reviewed-on: cwtch.im/cwtch#364

.drone.yml

@@ -5,28 +5,43 @@ workspace:
 pipeline:
   fetch:
     image: golang
+    when:
+      repo: cwtch.im/cwtch
+      branch: master
+      event: [ push, pull_request ]
     commands:
       - wget https://git.openprivacy.ca/openprivacy/buildfiles/raw/master/tor/tor
       - wget https://git.openprivacy.ca/openprivacy/buildfiles/raw/master/tor/torrc
       - chmod a+x tor
-      - go list ./... | xargs go get
       - go get -u golang.org/x/lint/golint
+      - export GO111MODULE=on
+      - go mod vendor
   quality:
     image: golang
+    when:
+      repo: cwtch.im/cwtch
+      branch: master
+      event: [ push, pull_request ]
     commands:
       - go list ./... | xargs go vet
       - go list ./... | xargs golint -set_exit_status
   units-tests:
     image: golang
+    when:
+      repo: cwtch.im/cwtch
+      branch: master
+      event: [ push, pull_request ]
     commands:
       - export PATH=$PATH:/go/src/cwtch.im/cwtch
       - sh testing/tests.sh
   integ-test:
     image: golang
+    when:
+      repo: cwtch.im/cwtch
+      branch: master
+      event: [ push, pull_request ]
     commands:
-      - ./tor -f ./torrc
-      - sleep 15
-      - go test -v cwtch.im/cwtch/testing
+      - go test -race -v cwtch.im/cwtch/testing/
   notify-email:
       image: drillster/drone-email
       host: build.openprivacy.ca
@@ -34,10 +49,15 @@ pipeline:
       skip_verify: true
       from: drone@openprivacy.ca
       when:
+        repo: cwtch.im/cwtch
+        branch: master
+        event: [ push, pull_request ]
         status: [ failure ]
   notify-gogs:
     image: openpriv/drone-gogs
     when:
+      repo: cwtch.im/cwtch
+      branch: master
       event: pull_request
       status: [ success, changed, failure ]
     secrets: [gogs_account_token]

.gitignore

@@ -7,3 +7,8 @@
 */messages/*
 server/app/messages
 .reviewboardrc
+/vendor/
+/testing/tor/
+/storage/*/testing/
+/storage/testing/
+/testing/storage/

README.md

@@ -19,7 +19,7 @@ We seek to protect the following communication contexts:
 Beyond individual conversations, we also seek to defend against context correlation attacks, whereby multiple conversations are analyzed to derive higher level information:
 
 * **Relationships** - Discovering social relationships between parties by analyzing the frequency and length of their communications over a period of time. (Carol and Eve call each other every single day for multiple hours at a time.)
-* **Cliques** - Discovering social relationships between multiple parties by deriving casual communication chains from their communication metadata (e.g. everytime Alice talks to Bob she talks to Carol almost immediately after.) 
+* **Cliques** - Discovering social relationships between multiple parties by deriving casual communication chains from their communication metadata (e.g. everytime Alice talks to Bob she talks to Carol almost immediately after.)
 * **Pattern of Life** - Discovering which communications are cyclical and predictable. (e.g. Alice calls Eve every Monday evening for around an hour.)
 
 
@@ -32,7 +32,7 @@ Development and Contributing information in [CONTRIBUTING.md](https://git.openpr
 #### Docker
 This repository contains a `Dockerfile` allowing you to build and run the server as a [docker](https://www.docker.com/) container.
 
-To get started issue `docker build -t openpriv/cwtch-server:latest`, this will create 2 temporary docker containers, one to build the Tor daemon and one to build Cwtch. The compiled binaries will then be bundled into a new image and tagged as `openpriv/cwtch-server:latest`.
+To get started issue `docker build -t openpriv/cwtch-server:latest .`, this will create 2 temporary docker containers, one to build the Tor daemon and one to build Cwtch. The compiled binaries will then be bundled into a new image and tagged as `openpriv/cwtch-server:latest`.
 
 To run Cwtch in the foreground execute `docker run openpriv/cwtch-server:latest`, you will see a small amount of output from Tor and then Cwtch will output your server address. When you `Ctrl + C` the container will terminate. To run Cwtch in the background execute `docker run --name my-cwtch-server -d openpriv/cwtch-server:latest`. To get your Cwtch server address issue `docker logs my-cwtch-server`.
 

app/app.go

@@ -1,170 +1,294 @@
 package app
 
 import (
-	"crypto/rand"
+	"cwtch.im/cwtch/app/plugins"
 	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
 	"cwtch.im/cwtch/peer"
+	"cwtch.im/cwtch/protocol/connections"
 	"cwtch.im/cwtch/storage"
-	"encoding/hex"
 	"fmt"
-
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/log"
 	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
+	"strconv"
 	"sync"
 )
 
+// AttributeTag is a const name for a peer attribute that can be set at creation time, for example for versioning info
+const AttributeTag = "tag"
+
+type applicationCore struct {
+	eventBuses map[string]event.Manager
+
+	directory string
+	coremutex sync.Mutex
+}
+
 type application struct {
-	peers        map[string]peer.CwtchPeer
-	acn          connectivity.ACN
-	directory    string
-	mutex        sync.Mutex
-	primaryonion string
-	storage      map[string]storage.ProfileStore
-	eventBus     *event.Manager
+	applicationCore
+	appletPeers
+	appletACN
+	appletPlugins
+	storage  map[string]storage.ProfileStore
+	engines  map[string]connections.Engine
+	appBus   event.Manager
+	appmutex sync.Mutex
 }
 
 // Application is a full cwtch peer application. It allows management, usage and storage of multiple peers
 type Application interface {
-	SaveProfile(cwtchPeer peer.CwtchPeer)
-	LoadProfiles(password string) error
-	CreatePeer(name string, password string) (peer.CwtchPeer, error)
-
-	PrimaryIdentity() peer.CwtchPeer
-	GetPeer(onion string) peer.CwtchPeer
-	ListPeers() map[string]string
+	LoadProfiles(password string)
+	CreatePeer(name string, password string)
+	CreateTaggedPeer(name string, password string, tag string)
+	DeletePeer(onion string)
+	AddPeerPlugin(onion string, pluginID plugins.PluginID)
+	ChangePeerPassword(onion, oldpass, newpass string)
 	LaunchPeers()
 
-	//GetTorStatus() (map[string]string, error)
+	GetPrimaryBus() event.Manager
+	GetEventBus(onion string) event.Manager
+	QueryACNStatus()
+	QueryACNVersion()
 
+	ShutdownPeer(string)
 	Shutdown()
+
+	GetPeer(onion string) peer.CwtchPeer
+	ListPeers() map[string]string
+}
+
+// LoadProfileFn is the function signature for a function in an app that loads a profile
+type LoadProfileFn func(profile *model.Profile, store storage.ProfileStore)
+
+func newAppCore(appDirectory string) *applicationCore {
+	appCore := &applicationCore{eventBuses: make(map[string]event.Manager), directory: appDirectory}
+	os.MkdirAll(path.Join(appCore.directory, "profiles"), 0700)
+	return appCore
 }
 
 // NewApp creates a new app with some environment awareness and initializes a Tor Manager
 func NewApp(acn connectivity.ACN, appDirectory string) Application {
 	log.Debugf("NewApp(%v)\n", appDirectory)
-	app := &application{peers: make(map[string]peer.CwtchPeer), storage: make(map[string]storage.ProfileStore), directory: appDirectory, acn: acn}
-	os.Mkdir(path.Join(app.directory, "profiles"), 0700)
-	app.eventBus = new(event.Manager)
-	app.eventBus.Initialize()
+	app := &application{storage: make(map[string]storage.ProfileStore), engines: make(map[string]connections.Engine), applicationCore: *newAppCore(appDirectory), appBus: event.NewEventManager()}
+	app.appletPeers.init()
+
+	app.appletACN.init(acn, app.getACNStatusHandler())
 	return app
 }
 
-func generateRandomFilename() string {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	return filepath.Join(hex.EncodeToString(randBytes))
-}
-
-func (app *application) SaveProfile(p peer.CwtchPeer) {
-	app.mutex.Lock()
-	defer app.mutex.Unlock()
-	app.peers[p.GetProfile().Onion] = p
-	app.storage[p.GetProfile().Onion].Save(p)
-}
-
-// NewProfile creates a new cwtchPeer with a given name.
-func (app *application) CreatePeer(name string, password string) (peer.CwtchPeer, error) {
+// CreatePeer creates a new Peer with a given name and core required accessories (eventbus)
+func (ac *applicationCore) CreatePeer(name string) (*model.Profile, error) {
 	log.Debugf("CreatePeer(%v)\n", name)
 
-	randomFileName := generateRandomFilename()
-	fileStore := storage.CreateFileProfileStore(path.Join(app.directory, "profiles", randomFileName), password)
-	p := peer.NewCwtchPeer(name)
-	err := fileStore.Save(p)
-	if err != nil {
-		return nil, err
-	}
-	p.Init(app.acn, app.eventBus)
-	_, exists := app.peers[p.GetProfile().Onion]
-	if exists {
-		p.Shutdown()
-		return nil, fmt.Errorf("Error: profile for onion %v already exists", p.GetProfile().Onion)
-	}
-	app.mutex.Lock()
-	app.peers[p.GetProfile().Onion] = p
-	app.storage[p.GetProfile().Onion] = fileStore
-	app.mutex.Unlock()
+	profile := storage.NewProfile(name)
 
-	return p, nil
+	ac.coremutex.Lock()
+	defer ac.coremutex.Unlock()
+
+	_, exists := ac.eventBuses[profile.Onion]
+	if exists {
+		return nil, fmt.Errorf("error: profile for onion %v already exists", profile.Onion)
+	}
+
+	eventBus := event.NewEventManager()
+	ac.eventBuses[profile.Onion] = eventBus
+
+	return profile, nil
 }
 
-func (app *application) LoadProfiles(password string) error {
-	files, err := ioutil.ReadDir(path.Join(app.directory, "profiles"))
+func (ac *applicationCore) DeletePeer(onion string) {
+	ac.coremutex.Lock()
+	defer ac.coremutex.Unlock()
+
+	ac.eventBuses[onion].Shutdown()
+	delete(ac.eventBuses, onion)
+}
+
+func (app *application) CreateTaggedPeer(name string, password string, tag string) {
+	profile, err := app.applicationCore.CreatePeer(name)
 	if err != nil {
-		return fmt.Errorf("Error: cannot read profiles directory: %v", err)
+		app.appBus.Publish(event.NewEventList(event.PeerError, event.Error, err.Error()))
+		return
+	}
+
+	profileStore := storage.CreateProfileWriterStore(app.eventBuses[profile.Onion], path.Join(app.directory, "profiles", profile.LocalID), password, profile)
+	app.storage[profile.Onion] = profileStore
+
+	pc := app.storage[profile.Onion].GetProfileCopy(true)
+	p := peer.FromProfile(pc)
+	p.Init(app.eventBuses[profile.Onion])
+
+	peerAuthorizations := profile.ContactsAuthorizations()
+	// TODO: Would be nice if ProtocolEngine did not need to explicitly be given the Private Key.
+	identity := primitives.InitializeIdentity(profile.Name, &profile.Ed25519PrivateKey, &profile.Ed25519PublicKey)
+	engine := connections.NewProtocolEngine(identity, profile.Ed25519PrivateKey, app.acn, app.eventBuses[profile.Onion], peerAuthorizations)
+
+	app.peers[profile.Onion] = p
+	app.engines[profile.Onion] = engine
+
+	if tag != "" {
+		p.SetAttribute(AttributeTag, tag)
+	}
+
+	app.appBus.Publish(event.NewEvent(event.NewPeer, map[event.Field]string{event.Identity: profile.Onion, event.Created: event.True}))
+}
+
+// CreatePeer creates a new Peer with the given name and required accessories (eventbus, storage, protocol engine)
+func (app *application) CreatePeer(name string, password string) {
+	app.CreateTaggedPeer(name, password, "")
+}
+
+func (app *application) DeletePeer(onion string) {
+	log.Infof("DeletePeer called on %v\n", onion)
+	app.appmutex.Lock()
+	defer app.appmutex.Unlock()
+
+	app.appletPlugins.ShutdownPeer(onion)
+	app.plugins.Delete(onion)
+
+	app.peers[onion].Shutdown()
+	delete(app.peers, onion)
+
+	app.engines[onion].Shutdown()
+	delete(app.engines, onion)
+
+	app.storage[onion].Shutdown()
+	app.storage[onion].Delete()
+	delete(app.storage, onion)
+
+	app.eventBuses[onion].Publish(event.NewEventList(event.ShutdownPeer, event.Identity, onion))
+
+	app.applicationCore.DeletePeer(onion)
+	log.Debugf("Delete peer for %v Done\n", onion)
+}
+
+func (app *application) ChangePeerPassword(onion, oldpass, newpass string) {
+	app.eventBuses[onion].Publish(event.NewEventList(event.ChangePassword, event.Password, oldpass, event.NewPassword, newpass))
+}
+
+func (app *application) AddPeerPlugin(onion string, pluginID plugins.PluginID) {
+	app.AddPlugin(onion, pluginID, app.eventBuses[onion], app.acn)
+}
+
+// LoadProfiles takes a password and attempts to load any profiles it can from storage with it and create Peers for them
+func (ac *applicationCore) LoadProfiles(password string, timeline bool, loadProfileFn LoadProfileFn) error {
+	files, err := ioutil.ReadDir(path.Join(ac.directory, "profiles"))
+	if err != nil {
+		return fmt.Errorf("error: cannot read profiles directory: %v", err)
 	}
 
 	for _, file := range files {
-
-		fileStore := storage.CreateFileProfileStore(path.Join(app.directory, "profiles", file.Name()), password)
-
-		p, err := fileStore.Load()
+		eventBus := event.NewEventManager()
+		profileStore, err := storage.LoadProfileWriterStore(eventBus, path.Join(ac.directory, "profiles", file.Name()), password)
 		if err != nil {
 			continue
 		}
 
-		_, exists := app.peers[p.GetProfile().Onion]
+		profile := profileStore.GetProfileCopy(timeline)
+
+		_, exists := ac.eventBuses[profile.Onion]
 		if exists {
-			p.Shutdown()
-			log.Errorf("profile for onion %v already exists", p.GetProfile().Onion)
+			profileStore.Shutdown()
+			eventBus.Shutdown()
+			log.Errorf("profile for onion %v already exists", profile.Onion)
 			continue
 		}
 
-		p.Init(app.acn, app.eventBus)
+		ac.coremutex.Lock()
+		ac.eventBuses[profile.Onion] = eventBus
+		ac.coremutex.Unlock()
 
-		app.mutex.Lock()
-		app.peers[p.GetProfile().Onion] = p
-		app.storage[p.GetProfile().Onion] = fileStore
-		if app.primaryonion == "" {
-			app.primaryonion = p.GetProfile().Onion
-		}
-		app.mutex.Unlock()
+		loadProfileFn(profile, profileStore)
 	}
 	return nil
 }
 
-func (app *application) LaunchPeers() {
-	for _, p := range app.peers {
-		if !p.IsStarted() {
-			p.Listen()
-		}
+// LoadProfiles takes a password and attempts to load any profiles it can from storage with it and create Peers for them
+func (app *application) LoadProfiles(password string) {
+	count := 0
+	app.applicationCore.LoadProfiles(password, true, func(profile *model.Profile, profileStore storage.ProfileStore) {
+		peer := peer.FromProfile(profile)
+		peer.Init(app.eventBuses[profile.Onion])
+
+		peerAuthorizations := profile.ContactsAuthorizations()
+		identity := primitives.InitializeIdentity(profile.Name, &profile.Ed25519PrivateKey, &profile.Ed25519PublicKey)
+		engine := connections.NewProtocolEngine(identity, profile.Ed25519PrivateKey, app.acn, app.eventBuses[profile.Onion], peerAuthorizations)
+		app.appmutex.Lock()
+		app.peers[profile.Onion] = peer
+		app.storage[profile.Onion] = profileStore
+		app.engines[profile.Onion] = engine
+		app.appmutex.Unlock()
+		app.appBus.Publish(event.NewEvent(event.NewPeer, map[event.Field]string{event.Identity: profile.Onion, event.Created: event.False}))
+		count++
+	})
+	if count == 0 {
+		message := event.NewEventList(event.AppError, event.Error, event.AppErrLoaded0)
+		app.appBus.Publish(message)
 	}
 }
 
-// ListPeers returns a map of onions to their profile's Name
-func (app *application) ListPeers() map[string]string {
-	keys := map[string]string{}
-	for k, p := range app.peers {
-		keys[k] = p.GetProfile().Name
-	}
-	return keys
+// GetPrimaryBus returns the bus the Application uses for events that aren't peer specific
+func (app *application) GetPrimaryBus() event.Manager {
+	return app.appBus
 }
 
-// PrimaryIdentity returns a cwtchPeer for a given onion address
-func (app *application) PrimaryIdentity() peer.CwtchPeer {
-	return app.peers[app.primaryonion]
-}
-
-// GetPeer returns a cwtchPeer for a given onion address
-func (app *application) GetPeer(onion string) peer.CwtchPeer {
-	if peer, ok := app.peers[onion]; ok {
-		return peer
+// GetEventBus returns a cwtchPeer's event bus
+func (ac *applicationCore) GetEventBus(onion string) event.Manager {
+	if manager, ok := ac.eventBuses[onion]; ok {
+		return manager
 	}
 	return nil
 }
 
-/*
-// GetTorStatus returns tor control port bootstrap-phase status info in a map
-func (app *application) GetTorStatus() (map[string]string, error) {
-	return app.torManager.GetStatus()
-}*/
+func (app *application) getACNStatusHandler() func(int, string) {
+	return func(progress int, status string) {
+		progStr := strconv.Itoa(progress)
+		app.peerLock.Lock()
+		app.appBus.Publish(event.NewEventList(event.ACNStatus, event.Progress, progStr, event.Status, status))
+		for _, bus := range app.eventBuses {
+			bus.Publish(event.NewEventList(event.ACNStatus, event.Progress, progStr, event.Status, status))
+		}
+		app.peerLock.Unlock()
+	}
+}
+
+func (app *application) QueryACNStatus() {
+	prog, status := app.acn.GetBootstrapStatus()
+	app.getACNStatusHandler()(prog, status)
+}
+
+func (app *application) QueryACNVersion() {
+	version := app.acn.GetVersion()
+	app.appBus.Publish(event.NewEventList(event.ACNVersion, event.Data, version))
+}
+
+// ShutdownPeer shuts down a peer and removes it from the app's management
+func (app *application) ShutdownPeer(onion string) {
+	app.appmutex.Lock()
+	defer app.appmutex.Unlock()
+	app.eventBuses[onion].Shutdown()
+	delete(app.eventBuses, onion)
+	app.peers[onion].Shutdown()
+	delete(app.peers, onion)
+	app.engines[onion].Shutdown()
+	delete(app.engines, onion)
+	app.storage[onion].Shutdown()
+	delete(app.storage, onion)
+	app.appletPlugins.Shutdown()
+}
 
 // Shutdown shutsdown all peers of an app and then the tormanager
 func (app *application) Shutdown() {
-	for _, peer := range app.peers {
+	for id, peer := range app.peers {
 		peer.Shutdown()
+		app.appletPlugins.ShutdownPeer(id)
+		app.engines[id].Shutdown()
+		app.storage[id].Shutdown()
+		app.eventBuses[id].Shutdown()
 	}
+	app.appBus.Shutdown()
 }

app/appBridge.go

@@ -0,0 +1,39 @@
+package app
+
+import "cwtch.im/cwtch/event"
+import "git.openprivacy.ca/openprivacy/log"
+
+const (
+	// DestApp should be used as a destination for IPC messages that are for the application itself an not a peer
+	DestApp = "app"
+)
+
+type applicationBridge struct {
+	applicationCore
+
+	bridge event.IPCBridge
+	handle func(*event.Event)
+}
+
+func (ab *applicationBridge) listen() {
+	log.Infoln("ab.listen()")
+	for {
+		ipcMessage, ok := ab.bridge.Read()
+		log.Debugf("listen() got %v for %v\n", ipcMessage.Message.EventType, ipcMessage.Dest)
+		if !ok {
+			log.Debugln("exiting appBridge.listen()")
+			return
+		}
+
+		if ipcMessage.Dest == DestApp {
+			ab.handle(&ipcMessage.Message)
+		} else {
+			if eventBus, exists := ab.eventBuses[ipcMessage.Dest]; exists {
+				eventBus.PublishLocal(ipcMessage.Message)
+			}
+		}
+	}
+}
+
+func (ab *applicationBridge) Shutdown() {
+}

app/appClient.go

@@ -0,0 +1,176 @@
+package app
+
+import (
+	"cwtch.im/cwtch/app/plugins"
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/peer"
+	"cwtch.im/cwtch/storage"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/log"
+	"path"
+	"strconv"
+	"sync"
+)
+
+type applicationClient struct {
+	applicationBridge
+	appletPeers
+
+	appBus  event.Manager
+	acmutex sync.Mutex
+}
+
+// NewAppClient returns an Application that acts as a client to a AppService, connected by the IPCBridge supplied
+func NewAppClient(appDirectory string, bridge event.IPCBridge) Application {
+	appClient := &applicationClient{appletPeers: appletPeers{peers: make(map[string]peer.CwtchPeer)}, applicationBridge: applicationBridge{applicationCore: *newAppCore(appDirectory), bridge: bridge}, appBus: event.NewEventManager()}
+	appClient.handle = appClient.handleEvent
+
+	go appClient.listen()
+
+	appClient.bridge.Write(&event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.ReloadClient)})
+
+	log.Infoln("Created new App Client")
+	return appClient
+}
+
+// GetPrimaryBus returns the bus the Application uses for events that aren't peer specific
+func (ac *applicationClient) GetPrimaryBus() event.Manager {
+	return ac.appBus
+}
+
+func (ac *applicationClient) handleEvent(ev *event.Event) {
+	switch ev.EventType {
+	case event.NewPeer:
+		localID := ev.Data[event.Identity]
+		key := ev.Data[event.Key]
+		salt := ev.Data[event.Salt]
+		reload := ev.Data[event.Status] == event.StorageRunning
+		created := ev.Data[event.Created]
+		ac.newPeer(localID, key, salt, reload, created)
+	case event.DeletePeer:
+		onion := ev.Data[event.Identity]
+		ac.handleDeletedPeer(onion)
+	case event.PeerError:
+		ac.appBus.Publish(*ev)
+	case event.AppError:
+		ac.appBus.Publish(*ev)
+	case event.ACNStatus:
+		ac.appBus.Publish(*ev)
+	case event.ACNVersion:
+		ac.appBus.Publish(*ev)
+	case event.ReloadDone:
+		ac.appBus.Publish(*ev)
+	}
+}
+
+func (ac *applicationClient) newPeer(localID, key, salt string, reload bool, created string) {
+	var keyBytes [32]byte
+	var saltBytes [128]byte
+	copy(keyBytes[:], key)
+	copy(saltBytes[:], salt)
+	profile, err := storage.ReadProfile(path.Join(ac.directory, "profiles", localID), keyBytes, saltBytes)
+	if err != nil {
+		log.Errorf("Could not read profile for NewPeer event: %v\n", err)
+		ac.appBus.Publish(event.NewEventList(event.PeerError, event.Error, fmt.Sprintf("Could not read profile for NewPeer event: %v\n", err)))
+		return
+	}
+
+	_, exists := ac.peers[profile.Onion]
+	if exists {
+		log.Errorf("profile for onion %v already exists", profile.Onion)
+		ac.appBus.Publish(event.NewEventList(event.PeerError, event.Error, fmt.Sprintf("profile for onion %v already exists", profile.Onion)))
+		return
+	}
+
+	eventBus := event.NewIPCEventManager(ac.bridge, profile.Onion)
+	peer := peer.FromProfile(profile)
+	peer.Init(eventBus)
+
+	ac.peerLock.Lock()
+	defer ac.peerLock.Unlock()
+	ac.peers[profile.Onion] = peer
+	ac.eventBuses[profile.Onion] = eventBus
+	npEvent := event.NewEvent(event.NewPeer, map[event.Field]string{event.Identity: profile.Onion, event.Created: created})
+	if reload {
+		npEvent.Data[event.Status] = event.StorageRunning
+	}
+	ac.appBus.Publish(npEvent)
+
+	if reload {
+		ac.bridge.Write(&event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.ReloadPeer, event.Identity, profile.Onion)})
+	}
+}
+
+// CreatePeer messages the service to create a new Peer with the given name
+func (ac *applicationClient) CreatePeer(name string, password string) {
+	ac.CreateTaggedPeer(name, password, "")
+}
+
+func (ac *applicationClient) CreateTaggedPeer(name, password, tag string) {
+	log.Infof("appClient CreatePeer %v\n", name)
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.CreatePeer, map[event.Field]string{event.ProfileName: name, event.Password: password, event.Data: tag})}
+	ac.bridge.Write(&message)
+}
+
+// DeletePeer messages tehe service to delete a peer
+func (ac *applicationClient) DeletePeer(onion string) {
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.DeletePeer, map[event.Field]string{event.Identity: onion})}
+	ac.bridge.Write(&message)
+}
+
+func (ac *applicationClient) ChangePeerPassword(onion, oldpass, newpass string) {
+	message := event.IPCMessage{Dest: onion, Message: event.NewEventList(event.ChangePassword, event.Password, oldpass, event.NewPassword, newpass)}
+	ac.bridge.Write(&message)
+}
+
+func (ac *applicationClient) handleDeletedPeer(onion string) {
+	ac.acmutex.Lock()
+	defer ac.acmutex.Unlock()
+	ac.peers[onion].Shutdown()
+	delete(ac.peers, onion)
+	ac.eventBuses[onion].Publish(event.NewEventList(event.ShutdownPeer, event.Identity, onion))
+
+	ac.applicationCore.DeletePeer(onion)
+}
+
+func (ac *applicationClient) AddPeerPlugin(onion string, pluginID plugins.PluginID) {
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.AddPeerPlugin, map[event.Field]string{event.Identity: onion, event.Data: strconv.Itoa(int(pluginID))})}
+	ac.bridge.Write(&message)
+}
+
+// LoadProfiles messages the service to load any profiles for the given password
+func (ac *applicationClient) LoadProfiles(password string) {
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.LoadProfiles, map[event.Field]string{event.Password: password})}
+	ac.bridge.Write(&message)
+}
+
+func (ac *applicationClient) QueryACNStatus() {
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.GetACNStatus, map[event.Field]string{})}
+	ac.bridge.Write(&message)
+}
+
+func (ac *applicationClient) QueryACNVersion() {
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.GetACNVersion, map[event.Field]string{})}
+	ac.bridge.Write(&message)
+}
+
+// ShutdownPeer shuts down a peer and removes it from the app's management
+func (ac *applicationClient) ShutdownPeer(onion string) {
+	ac.acmutex.Lock()
+	defer ac.acmutex.Unlock()
+	ac.eventBuses[onion].Shutdown()
+	delete(ac.eventBuses, onion)
+	ac.peers[onion].Shutdown()
+	delete(ac.peers, onion)
+	message := event.IPCMessage{Dest: DestApp, Message: event.NewEvent(event.ShutdownPeer, map[event.Field]string{event.Identity: onion})}
+	ac.bridge.Write(&message)
+}
+
+// Shutdown shuts down the application client and all front end peer components
+func (ac *applicationClient) Shutdown() {
+	for id := range ac.peers {
+		ac.ShutdownPeer(id)
+	}
+	ac.applicationBridge.Shutdown()
+	ac.appBus.Shutdown()
+}

app/appService.go

@@ -0,0 +1,201 @@
+package app
+
+import (
+	"cwtch.im/cwtch/app/plugins"
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
+	"cwtch.im/cwtch/protocol/connections"
+	"cwtch.im/cwtch/storage"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/log"
+	"path"
+	"strconv"
+	"sync"
+)
+
+type applicationService struct {
+	applicationBridge
+	appletACN
+	appletPlugins
+
+	storage map[string]storage.ProfileStore
+	engines map[string]connections.Engine
+	asmutex sync.Mutex
+}
+
+// ApplicationService is the back end of an application that manages engines and writing storage and communicates to an ApplicationClient by an IPCBridge
+type ApplicationService interface {
+	Shutdown()
+}
+
+// NewAppService returns an ApplicationService that runs the backend of an app and communicates with a client by the supplied IPCBridge
+func NewAppService(acn connectivity.ACN, appDirectory string, bridge event.IPCBridge) ApplicationService {
+	appService := &applicationService{storage: make(map[string]storage.ProfileStore), engines: make(map[string]connections.Engine), applicationBridge: applicationBridge{applicationCore: *newAppCore(appDirectory), bridge: bridge}}
+
+	appService.appletACN.init(acn, appService.getACNStatusHandler())
+	appService.handle = appService.handleEvent
+
+	go appService.listen()
+
+	log.Infoln("Created new App Service")
+	return appService
+}
+
+func (as *applicationService) handleEvent(ev *event.Event) {
+	log.Infof("app Service handleEvent %v\n", ev.EventType)
+	switch ev.EventType {
+	case event.CreatePeer:
+		profileName := ev.Data[event.ProfileName]
+		password := ev.Data[event.Password]
+		tag := ev.Data[event.Data]
+		as.createPeer(profileName, password, tag)
+	case event.DeletePeer:
+		onion := ev.Data[event.Identity]
+		as.deletePeer(onion)
+
+		message := event.IPCMessage{Dest: DestApp, Message: *ev}
+		as.bridge.Write(&message)
+	case event.AddPeerPlugin:
+		onion := ev.Data[event.Identity]
+		pluginID, _ := strconv.Atoi(ev.Data[event.Data])
+		as.AddPlugin(onion, plugins.PluginID(pluginID), as.eventBuses[onion], as.acn)
+	case event.LoadProfiles:
+		password := ev.Data[event.Password]
+		as.loadProfiles(password)
+	case event.ReloadClient:
+		for _, storage := range as.storage {
+			peerMsg := *storage.GetNewPeerMessage()
+			peerMsg.Data[event.Status] = event.StorageRunning
+			peerMsg.Data[event.Created] = event.False
+			message := event.IPCMessage{Dest: DestApp, Message: peerMsg}
+			as.bridge.Write(&message)
+		}
+
+		message := event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.ReloadDone)}
+		as.bridge.Write(&message)
+	case event.ReloadPeer:
+		onion := ev.Data[event.Identity]
+		events := as.storage[onion].GetStatusMessages()
+
+		for _, ev := range events {
+			message := event.IPCMessage{Dest: onion, Message: *ev}
+			as.bridge.Write(&message)
+		}
+	case event.GetACNStatus:
+		prog, status := as.acn.GetBootstrapStatus()
+		as.getACNStatusHandler()(prog, status)
+	case event.GetACNVersion:
+		version := as.acn.GetVersion()
+		as.bridge.Write(&event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.ACNVersion, event.Data, version)})
+	case event.ShutdownPeer:
+		onion := ev.Data[event.Identity]
+		as.ShutdownPeer(onion)
+	}
+}
+
+func (as *applicationService) createPeer(name, password, tag string) {
+	log.Infof("app Service create peer %v %v\n", name, password)
+	profile, err := as.applicationCore.CreatePeer(name)
+	as.eventBuses[profile.Onion] = event.IPCEventManagerFrom(as.bridge, profile.Onion, as.eventBuses[profile.Onion])
+	if err != nil {
+		log.Errorf("Could not create Peer: %v\n", err)
+		message := event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.PeerError, event.Error, err.Error())}
+		as.bridge.Write(&message)
+		return
+	}
+
+	if tag != "" {
+		profile.SetAttribute(AttributeTag, tag)
+	}
+
+	profileStore := storage.CreateProfileWriterStore(as.eventBuses[profile.Onion], path.Join(as.directory, "profiles", profile.LocalID), password, profile)
+
+	peerAuthorizations := profile.ContactsAuthorizations()
+	// TODO: Would be nice if ProtocolEngine did not need to explicitly be given the Private Key.
+	identity := primitives.InitializeIdentity(profile.Name, &profile.Ed25519PrivateKey, &profile.Ed25519PublicKey)
+	engine := connections.NewProtocolEngine(identity, profile.Ed25519PrivateKey, as.acn, as.eventBuses[profile.Onion], peerAuthorizations)
+
+	as.storage[profile.Onion] = profileStore
+	as.engines[profile.Onion] = engine
+
+	peerMsg := *profileStore.GetNewPeerMessage()
+	peerMsg.Data[event.Created] = event.True
+	peerMsg.Data[event.Status] = event.StorageNew
+	message := event.IPCMessage{Dest: DestApp, Message: peerMsg}
+	as.bridge.Write(&message)
+}
+
+func (as *applicationService) loadProfiles(password string) {
+	count := 0
+	as.applicationCore.LoadProfiles(password, false, func(profile *model.Profile, profileStore storage.ProfileStore) {
+		as.eventBuses[profile.Onion] = event.IPCEventManagerFrom(as.bridge, profile.Onion, as.eventBuses[profile.Onion])
+
+		peerAuthorizations := profile.ContactsAuthorizations()
+		identity := primitives.InitializeIdentity(profile.Name, &profile.Ed25519PrivateKey, &profile.Ed25519PublicKey)
+		engine := connections.NewProtocolEngine(identity, profile.Ed25519PrivateKey, as.acn, as.eventBuses[profile.Onion], peerAuthorizations)
+		as.asmutex.Lock()
+		as.storage[profile.Onion] = profileStore
+		as.engines[profile.Onion] = engine
+		as.asmutex.Unlock()
+
+		peerMsg := *profileStore.GetNewPeerMessage()
+		peerMsg.Data[event.Created] = event.False
+		peerMsg.Data[event.Status] = event.StorageNew
+		message := event.IPCMessage{Dest: DestApp, Message: peerMsg}
+		as.bridge.Write(&message)
+		count++
+	})
+	if count == 0 {
+		message := event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.AppError, event.Error, event.AppErrLoaded0)}
+		as.bridge.Write(&message)
+	}
+}
+
+func (as *applicationService) getACNStatusHandler() func(int, string) {
+	return func(progress int, status string) {
+		progStr := strconv.Itoa(progress)
+		as.bridge.Write(&event.IPCMessage{Dest: DestApp, Message: event.NewEventList(event.ACNStatus, event.Progress, progStr, event.Status, status)})
+		as.applicationCore.coremutex.Lock()
+		defer as.applicationCore.coremutex.Unlock()
+		for _, bus := range as.eventBuses {
+			bus.Publish(event.NewEventList(event.ACNStatus, event.Progress, progStr, event.Status, status))
+		}
+	}
+}
+
+func (as *applicationService) deletePeer(onion string) {
+	as.asmutex.Lock()
+	defer as.asmutex.Unlock()
+
+	as.appletPlugins.ShutdownPeer(onion)
+	as.plugins.Delete(onion)
+
+	as.engines[onion].Shutdown()
+	delete(as.engines, onion)
+
+	as.storage[onion].Shutdown()
+	as.storage[onion].Delete()
+	delete(as.storage, onion)
+
+	as.applicationCore.DeletePeer(onion)
+}
+
+func (as *applicationService) ShutdownPeer(onion string) {
+	as.engines[onion].Shutdown()
+	delete(as.engines, onion)
+	as.storage[onion].Shutdown()
+	delete(as.storage, onion)
+	as.eventBuses[onion].Shutdown()
+	delete(as.eventBuses, onion)
+}
+
+// Shutdown shuts down the application Service and all peer related backend parts
+func (as *applicationService) Shutdown() {
+	log.Debugf("shutting down application service...")
+	as.appletPlugins.Shutdown()
+	for id := range as.engines {
+		log.Debugf("shutting down application service peer engine %v", id)
+		as.ShutdownPeer(id)
+	}
+}

app/applets.go

@@ -0,0 +1,121 @@
+package app
+
+import (
+	"cwtch.im/cwtch/event"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/log"
+	"sync"
+
+	"cwtch.im/cwtch/app/plugins"
+	"cwtch.im/cwtch/peer"
+)
+
+type appletPeers struct {
+	peerLock sync.Mutex
+	peers    map[string]peer.CwtchPeer
+	launched bool // bit hacky, place holder while we transition to full multi peer support and a better api
+}
+
+type appletACN struct {
+	acn connectivity.ACN
+}
+
+type appletPlugins struct {
+	plugins sync.Map //map[string] []plugins.Plugin
+}
+
+// ***** applet ACN
+
+func (a *appletACN) init(acn connectivity.ACN, publish func(int, string)) {
+	a.acn = acn
+	acn.SetStatusCallback(publish)
+	prog, status := acn.GetBootstrapStatus()
+	publish(prog, status)
+}
+
+func (a *appletACN) Shutdown() {
+	a.acn.Close()
+}
+
+// ***** appletPeers
+
+func (ap *appletPeers) init() {
+	ap.peers = make(map[string]peer.CwtchPeer)
+	ap.launched = false
+}
+
+// LaunchPeers starts each peer Listening and connecting to peers and groups
+func (ap *appletPeers) LaunchPeers() {
+	log.Debugf("appletPeers LaunchPeers\n")
+	ap.peerLock.Lock()
+	defer ap.peerLock.Unlock()
+	if ap.launched {
+		return
+	}
+	for pid, p := range ap.peers {
+		log.Debugf("Launching %v\n", pid)
+		p.Listen()
+		log.Debugf("done Listen() for %v\n", pid)
+		p.StartPeersConnections()
+		log.Debugf("done StartPeersConnections() for %v\n", pid)
+	}
+	ap.launched = true
+}
+
+// ListPeers returns a map of onions to their profile's Name
+func (ap *appletPeers) ListPeers() map[string]string {
+	keys := map[string]string{}
+
+	ap.peerLock.Lock()
+	defer ap.peerLock.Unlock()
+	for k, p := range ap.peers {
+		keys[k] = p.GetOnion()
+	}
+	return keys
+}
+
+// GetPeer returns a cwtchPeer for a given onion address
+func (ap *appletPeers) GetPeer(onion string) peer.CwtchPeer {
+	if peer, ok := ap.peers[onion]; ok {
+		return peer
+	}
+	return nil
+}
+
+// ***** applet Plugins
+
+func (ap *appletPlugins) Shutdown() {
+	log.Debugf("shutting down applet plugins...")
+	ap.plugins.Range(func(k, v interface{}) bool {
+		log.Debugf("shutting down plugins for %v", k)
+		ap.ShutdownPeer(k.(string))
+		return true
+	})
+}
+
+func (ap *appletPlugins) ShutdownPeer(peerid string) {
+	log.Debugf("shutting down plugins for %v", peerid)
+	pluginsI, ok := ap.plugins.Load(peerid)
+	if ok {
+		plugins := pluginsI.([]plugins.Plugin)
+		for _, plugin := range plugins {
+			log.Debugf("shutting down plugin: %v", plugin)
+			plugin.Shutdown()
+		}
+	}
+}
+
+func (ap *appletPlugins) AddPlugin(peerid string, id plugins.PluginID, bus event.Manager, acn connectivity.ACN) {
+	if _, exists := ap.plugins.Load(peerid); !exists {
+		ap.plugins.Store(peerid, []plugins.Plugin{})
+	}
+
+	pluginsinf, _ := ap.plugins.Load(peerid)
+	peerPlugins := pluginsinf.([]plugins.Plugin)
+
+	newp := plugins.Get(id, bus, acn, peerid)
+	newp.Start()
+	peerPlugins = append(peerPlugins, newp)
+	log.Debugf("storing plugin for %v %v", peerid, peerPlugins)
+	ap.plugins.Store(peerid, peerPlugins)
+}

app/bots/servermon/main.go

@@ -1,26 +1,27 @@
 package main
 
 import (
-	"cwtch.im/cwtch/event"
+	app2 "cwtch.im/cwtch/app"
+	"cwtch.im/cwtch/app/utils"
 	"cwtch.im/cwtch/peer"
 	"cwtch.im/cwtch/protocol/connections"
 	"errors"
 	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"os"
 	"time"
 )
 
-func waitForPeerServerConnection(peer peer.CwtchPeer, server string) error {
+func waitForPeerGroupConnection(peer peer.CwtchPeer, groupID string) error {
 	for {
-		servers := peer.GetServers()
-		state, ok := servers[server]
-		if ok {
+		group := peer.GetGroup(groupID)
+		if group != nil {
+			state, _ := peer.GetGroupState(groupID)
 			if state == connections.FAILED {
-				return errors.New("Connection to server " + server + " failed!")
+				return errors.New("Connection to group " + groupID + " failed!")
 			}
 			if state != connections.AUTHENTICATED {
-				fmt.Printf("peer %v waiting to authenticate with server %v, current state: %v\n", peer.GetProfile().Onion, server, connections.ConnectionStateName[state])
+				fmt.Printf("peer %v waiting to authenticate with group %v 's server, current state: %v\n", peer.GetOnion(), groupID, connections.ConnectionStateName[state])
 				time.Sleep(time.Second * 10)
 				continue
 			}
@@ -40,14 +41,17 @@ func main() {
 
 	serverAddr := os.Args[1]
 
-	acn, err := connectivity.StartTor(".", "")
+	acn, err := tor.NewTorACN(".", "")
 	if err != nil {
 		fmt.Printf("Could not start tor: %v\n", err)
 		os.Exit(1)
 	}
 
-	botPeer := peer.NewCwtchPeer("servermon")
-	botPeer.Init(acn, new(event.Manager))
+	app := app2.NewApp(acn, ".")
+
+	app.CreatePeer("servermon", "be gay, do crimes")
+
+	botPeer := utils.WaitGetPeer(app, "servermon")
 
 	fmt.Printf("Connecting to %v...\n", serverAddr)
 	botPeer.JoinServer(serverAddr)
@@ -57,7 +61,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = waitForPeerServerConnection(botPeer, serverAddr)
+	err = waitForPeerGroupConnection(botPeer, groupID)
 	if err != nil {
 		fmt.Printf("Could not connect to server %v: %v\n", serverAddr, err)
 		os.Exit(1)
@@ -66,7 +70,7 @@ func main() {
 	timeout := 1 * time.Second
 	timeElapsed := 0 * time.Second
 	for {
-		err := botPeer.SendMessageToGroup(groupID, timeout.String())
+		_, err := botPeer.SendMessageToGroupTracked(groupID, timeout.String())
 		if err != nil {
 			fmt.Printf("Sent to group on server %v failed at interval %v of total %v with: %v\n", serverAddr, timeout, timeElapsed, err)
 			os.Exit(1)

app/cli/main.go

@@ -1,614 +0,0 @@
-package main
-
-import (
-	app2 "cwtch.im/cwtch/app"
-	peer2 "cwtch.im/cwtch/peer"
-
-	"bytes"
-	"cwtch.im/cwtch/model"
-	"cwtch.im/cwtch/protocol/connections"
-	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"github.com/c-bata/go-prompt"
-	"golang.org/x/crypto/ssh/terminal"
-	"os"
-	"os/user"
-	"path"
-	"strings"
-	"syscall"
-	"time"
-)
-
-var app app2.Application
-var peer peer2.CwtchPeer
-var group *model.Group
-var groupFollowBreakChan chan bool
-var prmpt string
-
-var suggestionsBase = []prompt.Suggest{
-	{Text: "/new-profile", Description: "create a new profile"},
-	{Text: "/load-profiles", Description: "loads profiles with a password"},
-	{Text: "/list-profiles", Description: "list active profiles"},
-	{Text: "/select-profile", Description: "selects an active profile to use"},
-	{Text: "/help", Description: "print list of commands"},
-	{Text: "/quit", Description: "quit cwtch"},
-}
-
-var suggestionsSelectedProfile = []prompt.Suggest{
-	{Text: "/info", Description: "show user info"},
-	{Text: "/list-contacts", Description: "retrieve a list of contacts"},
-	{Text: "/list-groups", Description: "retrieve a list of groups"},
-	{Text: "/new-group", Description: "create a new group on a server"},
-	{Text: "/select-group", Description: "selects a group to follow"},
-	{Text: "/unselect-group", Description: "stop following the current group"},
-	{Text: "/invite", Description: "invite a new contact"},
-	{Text: "/invite-to-group", Description: "invite an existing contact to join an existing group"},
-	{Text: "/accept-invite", Description: "accept the invite of a group"},
-	{Text: "/list-servers", Description: "retrieve a list of servers and their connection status"},
-	{Text: "/list-peers", Description: "retrieve a list of peers and their connection status"},
-	{Text: "/export-group", Description: "export a group invite: prints as a string"},
-	{Text: "/trust", Description: "trust a peer"},
-	{Text: "/block", Description: "block a peer - you will no longer see messages or connect to this peer"},
-}
-
-var suggestions = suggestionsBase
-
-var usages = map[string]string{
-	"/new-profile":     "/new-profile [name]",
-	"/load-profiles":   "/load-profiles",
-	"/list-profiles":   "",
-	"/select-profile":  "/select-profile [onion]",
-	"/quit":            "",
-	"/list-servers":    "",
-	"/list-peers":      "",
-	"/list-contacts":   "",
-	"/list-groups":     "",
-	"/select-group":    "/select-group [groupid]",
-	"/unselect-group":  "",
-	"/export-group":    "/export-group [groupid]",
-	"/info":            "",
-	"/send":            "/send [groupid] [message]",
-	"/timeline":        "/timeline [groupid]",
-	"/accept-invite":   "/accept-invite [groupid]",
-	"/invite":          "/invite [peerid]",
-	"/invite-to-group": "/invite-to-group [groupid] [peerid]",
-	"/new-group":       "/new-group [server]",
-	"/help":            "",
-	"/trust":           "/trust [peerid]",
-	"/block":           "/block [peerid]",
-}
-
-func printMessage(m model.Message) {
-	p := peer.GetContact(m.PeerID)
-	name := "unknown"
-	if p != nil {
-		name = p.Name
-	} else if peer.GetProfile().Onion == m.PeerID {
-		name = peer.GetProfile().Name
-	}
-
-	fmt.Printf("%v %v (%v): %v\n", m.Timestamp, name, m.PeerID, m.Message)
-}
-
-func startGroupFollow() {
-	groupFollowBreakChan = make(chan bool)
-	go func() {
-		for {
-			l := len(group.Timeline.GetMessages())
-			select {
-			case <-time.After(1 * time.Second):
-				if group == nil {
-					return
-				}
-				gms := group.Timeline.GetMessages()
-				if len(gms) != l {
-					fmt.Printf("\n")
-					for ; l < len(gms); l++ {
-						printMessage(gms[l])
-					}
-					fmt.Printf(prmpt)
-				}
-			case <-groupFollowBreakChan:
-				return
-			}
-		}
-	}()
-}
-
-func stopGroupFollow() {
-	if group != nil {
-		groupFollowBreakChan <- true
-		group = nil
-	}
-}
-
-func completer(d prompt.Document) []prompt.Suggest {
-
-	var s []prompt.Suggest
-
-	if d.FindStartOfPreviousWord() == 0 {
-		return prompt.FilterHasPrefix(suggestions, d.GetWordBeforeCursor(), true)
-	}
-
-	w := d.CurrentLine()
-
-	// Suggest a profile id
-	if strings.HasPrefix(w, "/select-profile") {
-		s = []prompt.Suggest{}
-		peerlist := app.ListPeers()
-		for onion, peername := range peerlist {
-			s = append(s, prompt.Suggest{Text: onion, Description: peername})
-		}
-	}
-
-	if peer == nil {
-		return s
-	}
-
-	// Suggest groupid
-	if /*strings.HasPrefix(w, "send") || strings.HasPrefix(w, "timeline") ||*/ strings.HasPrefix(w, "/export-group") || strings.HasPrefix(w, "/select-group") {
-		s = []prompt.Suggest{}
-		groups := peer.GetGroups()
-		for _, groupID := range groups {
-			group := peer.GetGroup(groupID)
-			s = append(s, prompt.Suggest{Text: group.GroupID, Description: "Group owned by " + group.Owner + " on " + group.GroupServer})
-		}
-		return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true)
-	}
-
-	// Suggest unaccepted group
-	if strings.HasPrefix(w, "/accept-invite") {
-		s = []prompt.Suggest{}
-		groups := peer.GetGroups()
-		for _, groupID := range groups {
-			group := peer.GetGroup(groupID)
-			if group.Accepted == false {
-				s = append(s, prompt.Suggest{Text: group.GroupID, Description: "Group owned by " + group.Owner + " on " + group.GroupServer})
-			}
-		}
-		return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true)
-	}
-
-	// suggest groupid AND peerid
-	if strings.HasPrefix(w, "/invite-to-group") {
-
-		if d.FindStartOfPreviousWordWithSpace() == 0 {
-			s = []prompt.Suggest{}
-			groups := peer.GetGroups()
-			for _, groupID := range groups {
-				group := peer.GetGroup(groupID)
-				if group.Owner == "self" {
-					s = append(s, prompt.Suggest{Text: group.GroupID, Description: "Group owned by " + group.Owner + " on " + group.GroupServer})
-				}
-			}
-			return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true)
-		}
-
-		s = []prompt.Suggest{}
-		contacts := peer.GetContacts()
-		for _, onion := range contacts {
-			contact := peer.GetContact(onion)
-			s = append(s, prompt.Suggest{Text: contact.Onion, Description: contact.Name})
-		}
-		return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true)
-	}
-
-	// Suggest contact onion / peerid
-	if strings.HasPrefix(w, "/block") || strings.HasPrefix(w, "/trust") || strings.HasPrefix(w, "/invite") {
-		s = []prompt.Suggest{}
-		contacts := peer.GetContacts()
-		for _, onion := range contacts {
-			contact := peer.GetContact(onion)
-			s = append(s, prompt.Suggest{Text: contact.Onion, Description: contact.Name})
-		}
-		return prompt.FilterHasPrefix(s, d.GetWordBeforeCursor(), true)
-	}
-
-	return s
-}
-
-func main() {
-
-	cwtch :=
-		`
-                                                   #, #'
-                                                  @@@@@@:
-                                                  @@@@@@.
-                             @'@@+#'               @@@@+
-                             ''''''@               #+@ :
-                             @''''+;+'             .   '
-                              @''@'    :+'     ,  ;     ##, +'
-                               ,@@         ;' #'#@''.   #''@''#
-                                  #           ''''''#:,,#'''''@
-                                   :          @''''@    :+'''@
-                                    '         @;+'@       @'#
-                                    .:# '#..#   '#         @
-                                     @@@@@@
-                                     @@@@@@
-                                     '@@@@
-                                       @#                             .  .
-                                                +++,                 #'@+'@
-                                                ''',                ''''''#
-                             .#+#               ''',                 @'''+,
-                             @''#               ''',                  .#@
-     :;                     '@''#       .;.     ''',  '                :          ;.     ,
-   @+'''@  '+'+   @++   @+'@+''''+@   #+'''#:   ''';#''+@             @@@@  @@@@@@@@@ :@@@@#
-  #''''''#  +''.  +'':  +'''''''''+  @'''''''#  '''+'''''@            @@@@  @@@@@@@@@@@@@@@@:
- @'''@@'''@ @''# ,'''@  ''+ @@''+#+ :'''@@+'''  ''''@@''''            @@@@  @@@@@@@@@@@@@@@@@
- '''#  @''# +''@ @'''# ;''@  +''+   @''@  ,+'', '''@  #'''.           @@@@  @@@@  '@@@#  @@@@
-;'''    @@; '''# #'@'' @''@  @''+   +''#   .@@  ''',   '''.           @@@@  @@@    @@@   .@@@
-@''#         #'' ''#''#@''.  #''#   '''.        '''.   +'',           @@@@  @@@    @@@    @@@
-@''#         @''@'' #'@+'+   #''#   '''.        ''',   +'', +@@@.@@@  @@@@  @@@,   @@@   ,@@@
-;''+     @,  +''@'# @'+''@   @''#   +'';    '+  ''',   +'', @@@@@@@@# @@@@  @@@.  .@@@   .@@@
- '''#  ++'+   ''''@ ,''''#   #'''   @''@  '@''+ ''',   ''', @@@@@@@@: @@@@  @@@;  .@@@'  ;@@@
- @'''@@'''@   #'''.  +''''   ;'''#@ :'''#@+''+  ''',   ''',  @@@@@@#  @@@@  @@@+  ,@@@.  @@@@
-  #''''''#    @''+   @''+     +''''  @'''''''#  ''',   ''',   #@@@.   @@@@  @@@+   @@@   @@@@
-   @+''+@     '++@   ;++@     '#''@   ##'''@:   +++,   +++,    :@     @@@@  @@@'   @@@   '@@@
-                                :'                              '                          '`
-	fmt.Printf("%v\n\n", cwtch)
-	quit := false
-
-	usr, err := user.Current()
-	if err != nil {
-		log.Errorf("\nError: could not load current user: %v\n", err)
-		os.Exit(1)
-	}
-
-	acn, err := connectivity.StartTor(path.Join(usr.HomeDir, ".cwtch"), "")
-	if err != nil {
-		log.Errorf("\nError connecting to Tor: %v\n", err)
-		os.Exit(1)
-	}
-	app = app2.NewApp(acn, path.Join(usr.HomeDir, ".cwtch"))
-	if err != nil {
-		log.Errorf("Error initializing application: %v", err)
-		os.Exit(1)
-	}
-	log.SetLevel(log.LevelDebug)
-	fmt.Printf("\nWelcome to Cwtch!\n")
-	fmt.Printf("If this if your first time you should create a profile by running `/new-profile`\n")
-	fmt.Printf("`/load-profiles` will prompt you for a password and load profiles from storage\n")
-	fmt.Printf("`/help` will show you other available commands\n")
-	fmt.Printf("There is full [TAB] completion support\n\n")
-
-	var history []string
-	for !quit {
-
-		prmpt = "cwtch> "
-		if group != nil {
-			prmpt = fmt.Sprintf("cwtch %v (%v) [%v] say> ", peer.GetProfile().Name, peer.GetProfile().Onion, group.GroupID)
-		} else if peer != nil {
-			prmpt = fmt.Sprintf("cwtch %v (%v)> ", peer.GetProfile().Name, peer.GetProfile().Onion)
-		}
-
-		text := prompt.Input(prmpt, completer, prompt.OptionSuggestionBGColor(prompt.Purple),
-			prompt.OptionDescriptionBGColor(prompt.White),
-			prompt.OptionPrefixTextColor(prompt.White),
-			prompt.OptionInputTextColor(prompt.Purple),
-			prompt.OptionHistory(history))
-
-		commands := strings.Split(text[0:], " ")
-		history = append(history, text)
-
-		if peer == nil {
-			if commands[0] != "/help" && commands[0] != "/quit" && commands[0] != "/new-profile" && commands[0] != "/load-profiles" && commands[0] != "/select-profile" && commands[0] != "/list-profiles" {
-				fmt.Printf("Profile needs to be set\n")
-				continue
-			}
-		}
-
-		// Send
-		if group != nil && !strings.HasPrefix(commands[0], "/") {
-			err := peer.SendMessageToGroup(group.GroupID, text)
-			if err != nil {
-				fmt.Printf("Error sending message: %v\n", err)
-			}
-		}
-
-		switch commands[0] {
-		case "/quit":
-			quit = true
-		case "/new-profile":
-			if len(commands) == 2 {
-				name := strings.Trim(commands[1], " ")
-				if name == "" {
-					fmt.Printf("Error creating profile, usage: %v\n", usages[commands[0]])
-					break
-				}
-
-				fmt.Print("** WARNING: PASSWORDS CANNOT BE RECOVERED! **\n")
-
-				password := ""
-				failcount := 0
-				for ; failcount < 3; failcount++ {
-					fmt.Print("Enter a password to encrypt the profile: ")
-					bytePassword, _ := terminal.ReadPassword(int(syscall.Stdin))
-					if string(bytePassword) == "" {
-						fmt.Print("\nBlank password not allowed.")
-						continue
-					}
-					fmt.Print("\nRe-enter password: ")
-					bytePassword2, _ := terminal.ReadPassword(int(syscall.Stdin))
-					if bytes.Equal(bytePassword, bytePassword2) {
-						password = string(bytePassword)
-						break
-					} else {
-						fmt.Print("\nPASSWORDS DIDN'T MATCH! Try again.\n")
-					}
-				}
-
-				if failcount >= 3 {
-					fmt.Printf("Error creating profile for %v: Your password entries must match!\n", name)
-				} else {
-					p, err := app.CreatePeer(name, password)
-					app.LaunchPeers()
-					if err == nil {
-						stopGroupFollow()
-						fmt.Printf("\nNew profile created for %v\n", name)
-						peer = p
-						suggestions = append(suggestionsBase, suggestionsSelectedProfile...)
-
-					} else {
-						fmt.Printf("\nError creating profile for %v: %v\n", name, err)
-					}
-				}
-			} else {
-				fmt.Printf("Error creating New Profile, usage: %s\n", usages[commands[0]])
-			}
-		case "/load-profiles":
-			fmt.Print("Enter a password to decrypt the profile: ")
-			bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
-
-			err = app.LoadProfiles(string(bytePassword))
-			if err == nil {
-				app.LaunchPeers()
-				profiles := app.ListPeers()
-				fmt.Printf("\n%v profiles active now\n", len(profiles))
-				fmt.Printf("You should run `select-profile` to use a profile or `list-profiles` to view loaded profiles\n")
-			} else {
-				fmt.Printf("\nError loading profiles: %v\n", err)
-			}
-
-		case "/list-profiles":
-			peerlist := app.ListPeers()
-			for onion, peername := range peerlist {
-				fmt.Printf("  %v\t%v\n", onion, peername)
-			}
-		case "/select-profile":
-			if len(commands) == 2 {
-				p := app.GetPeer(commands[1])
-				if p == nil {
-					fmt.Printf("Error: profile '%v' does not exist\n", commands[1])
-				} else {
-					stopGroupFollow()
-					peer = p
-					suggestions = append(suggestionsBase, suggestionsSelectedProfile...)
-				}
-
-				// Auto cwtchPeer / Join Server
-				// TODO There are some privacy implications with this that we should
-				// think over.
-				for _, name := range p.GetProfile().GetContacts() {
-					profile := p.GetContact(name)
-					if profile.Trusted && !profile.Blocked {
-						p.PeerWithOnion(profile.Onion)
-					}
-				}
-
-				for _, groupid := range p.GetGroups() {
-					group := p.GetGroup(groupid)
-					if group.Accepted || group.Owner == "self" {
-						p.JoinServer(group.GroupServer)
-					}
-				}
-
-			} else {
-				fmt.Printf("Error selecting profile, usage: %s\n", usages[commands[0]])
-			}
-		case "/info":
-			if peer != nil {
-				fmt.Printf("Address cwtch:%v\n", peer.GetProfile().Onion)
-			} else {
-				fmt.Printf("Profile needs to be set\n")
-			}
-		case "/invite":
-			if len(commands) == 2 {
-				fmt.Printf("Inviting cwtch:%v\n", commands[1])
-				peer.PeerWithOnion(commands[1])
-			} else {
-				fmt.Printf("Error inviting peer, usage: %s\n", usages[commands[0]])
-			}
-		case "/list-peers":
-			peers := peer.GetPeers()
-			for p, s := range peers {
-				fmt.Printf("Name: %v Status: %v\n", p, connections.ConnectionStateName[s])
-			}
-		case "/list-servers":
-			servers := peer.GetServers()
-			for s, st := range servers {
-				fmt.Printf("Name: %v Status: %v\n", s, connections.ConnectionStateName[st])
-			}
-		case "/list-contacts":
-			contacts := peer.GetContacts()
-			for _, onion := range contacts {
-				c := peer.GetContact(onion)
-				fmt.Printf("Name: %v Onion: %v Trusted: %v\n", c.Name, c.Onion, c.Trusted)
-			}
-		case "/list-groups":
-			for _, gid := range peer.GetGroups() {
-				g := peer.GetGroup(gid)
-				fmt.Printf("Group Id: %v Owner: %v Accepted:%v\n", gid, g.Owner, g.Accepted)
-			}
-		case "/trust":
-			if len(commands) == 2 {
-				peer.TrustPeer(commands[1])
-			} else {
-				fmt.Printf("Error trusting peer, usage: %s\n", usages[commands[0]])
-			}
-		case "/block":
-			if len(commands) == 2 {
-				peer.BlockPeer(commands[1])
-			} else {
-				fmt.Printf("Error blocking peer, usage: %s\n", usages[commands[0]])
-			}
-		case "/accept-invite":
-			if len(commands) == 2 {
-				groupID := commands[1]
-				err := peer.AcceptInvite(groupID)
-				if err != nil {
-					fmt.Printf("Error: %v\n", err)
-				} else {
-					group := peer.GetGroup(groupID)
-					if group == nil {
-						fmt.Printf("Error: group does not exist\n")
-					} else {
-						peer.JoinServer(group.GroupServer)
-					}
-				}
-			} else {
-				fmt.Printf("Error accepting invite, usage: %s\n", usages[commands[0]])
-			}
-		case "/invite-to-group":
-			if len(commands) == 3 {
-				fmt.Printf("Inviting %v to %v\n", commands[1], commands[2])
-				err := peer.InviteOnionToGroup(commands[2], commands[1])
-				if err != nil {
-					fmt.Printf("Error: %v\n", err)
-				}
-			} else {
-				fmt.Printf("Error inviting peer to group, usage: %s\n", usages[commands[0]])
-			}
-		case "/new-group":
-			if len(commands) == 2 && commands[1] != "" {
-				fmt.Printf("Setting up a new group on server:%v\n", commands[1])
-				id, _, err := peer.StartGroup(commands[1])
-				if err == nil {
-					fmt.Printf("New Group [%v] created for server %v\n", id, commands[1])
-					group := peer.GetGroup(id)
-					if group == nil {
-						fmt.Printf("Error: group does not exist\n")
-					} else {
-						peer.JoinServer(group.GroupServer)
-					}
-				} else {
-					fmt.Printf("Error creating new group: %v", err)
-				}
-			} else {
-				fmt.Printf("Error creating a new group, usage: %s\n", usages[commands[0]])
-			}
-		case "/select-group":
-			if len(commands) == 2 {
-				g := peer.GetGroup(commands[1])
-				if g == nil {
-					fmt.Printf("Error: group %s not found!\n", commands[1])
-				} else {
-					stopGroupFollow()
-					group = g
-
-					fmt.Printf("--------------- %v ---------------\n", group.GroupID)
-					gms := group.Timeline.Messages
-					max := 20
-					if len(gms) < max {
-						max = len(gms)
-					}
-					for i := len(gms) - max; i < len(gms); i++ {
-						printMessage(gms[i])
-					}
-					fmt.Printf("------------------------------\n")
-
-					startGroupFollow()
-				}
-			} else {
-				fmt.Printf("Error selecting a group, usage: %s\n", usages[commands[0]])
-			}
-		case "/unselect-group":
-			stopGroupFollow()
-		case "/export-group":
-			if len(commands) == 2 {
-				group := peer.GetGroup(commands[1])
-				if group == nil {
-					fmt.Printf("Error: group does not exist\n")
-				} else {
-					invite, _ := peer.ExportGroup(commands[1])
-					fmt.Printf("Invite: %v\n", invite)
-				}
-			} else {
-				fmt.Printf("Error exporting group, usage: %s\n", usages[commands[0]])
-			}
-		case "/import-group":
-			if len(commands) == 2 {
-				groupID, err := peer.ImportGroup(commands[1])
-				if err != nil {
-					fmt.Printf("Error importing group: %v\n", err)
-				} else {
-					fmt.Printf("Imported group: %s\n", groupID)
-				}
-			} else {
-				fmt.Printf("%v", commands)
-				fmt.Printf("Error importing group, usage: %s\n", usages[commands[0]])
-			}
-		case "/save":
-			app.SaveProfile(peer)
-		case "/help":
-			for _, command := range suggestions {
-				fmt.Printf("%-18s%-56s%s\n", command.Text, command.Description, usages[command.Text])
-			}
-		case "/sendlots":
-			if len(commands) == 2 {
-				group := peer.GetGroup(commands[1])
-				if group == nil {
-					fmt.Printf("Error: group does not exist\n")
-				} else {
-					for i := 0; i < 100; i++ {
-						fmt.Printf("Sending message: %v\n", i)
-						err := peer.SendMessageToGroup(commands[1], fmt.Sprintf("this is message %v", i))
-						if err != nil {
-							fmt.Printf("could not send message %v because %v\n", i, err)
-						}
-					}
-					fmt.Printf("Waiting 5 seconds for message to process...\n")
-					time.Sleep(time.Second * 5)
-					timeline := group.GetTimeline()
-					totalLatency := time.Duration(0)
-					maxLatency := time.Duration(0)
-					totalMessages := 0
-					for i := 0; i < 100; i++ {
-						found := false
-						for _, m := range timeline {
-							if m.Message == fmt.Sprintf("this is message %v", i) && m.PeerID == peer.GetProfile().Onion {
-								found = true
-								latency := m.Received.Sub(m.Timestamp)
-								fmt.Printf("Latency for Message %v was %v\n", i, latency)
-								totalLatency = totalLatency + latency
-								if maxLatency < latency {
-									maxLatency = latency
-								}
-								totalMessages++
-							}
-						}
-
-						if !found {
-							fmt.Printf("message %v was never received\n", i)
-						}
-					}
-
-					fmt.Printf("Average Latency for %v messages was: %vms\n", totalMessages, time.Duration(int64(totalLatency)/int64(totalMessages)))
-					fmt.Printf("Max Latency for %v messages was: %vms\n", totalMessages, maxLatency)
-
-				}
-			}
-		}
-		if peer != nil {
-			app.SaveProfile(peer)
-		}
-	}
-
-	if peer != nil {
-		app.SaveProfile(peer)
-	}
-
-	app.Shutdown()
-	acn.Close()
-	os.Exit(0)
-}

app/cwtchutil/main.go

@@ -1,54 +1,14 @@
 package main
 
 import (
-	"bufio"
-	"crypto/rand"
-	libpeer "cwtch.im/cwtch/peer"
-	storage2 "cwtch.im/cwtch/storage"
-	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-
 	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"golang.org/x/crypto/ed25519"
-	"io/ioutil"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/log"
 	"os"
-	"strconv"
-	"strings"
-	"time"
+	//"bufio"
+	//"cwtch.im/cwtch/storage"
 )
 
-func convertCwtchFile(filename string, password string) error {
-	fileStore := storage2.CreateFileProfileStore(filename, password)
-	peer, err := fileStore.Load()
-	if err != nil {
-		return err
-	}
-
-	b := []byte("== ed25519v1-secret: type0 ==")
-	b = append(b, peer.GetProfile().Ed25519PrivateKey...)
-	err = ioutil.WriteFile("hs_ed25519_secret_key", b, 0600)
-	if err != nil {
-		return err
-	}
-
-	b = []byte("== ed25519v1-public: type0 ==")
-	b = append(b, peer.GetProfile().Ed25519PublicKey...)
-	err = ioutil.WriteFile("hs_ed25519_public_key", b, 0600)
-	if err != nil {
-		return err
-	}
-
-	b = []byte(peer.GetProfile().Onion + ".onion\n")
-	err = ioutil.WriteFile("hostname", b, 0600)
-	if err != nil {
-		return err
-	}
-
-	log.Infoln("success!")
-	return nil
-}
-
 func convertTorFile(filename string, password string) error {
 	return errors.New("this code doesn't work and can never work :( it's a math thing")
 
@@ -77,8 +37,8 @@ func convertTorFile(filename string, password string) error {
 	peer.GetProfile().Ed25519PrivateKey = sk
 	peer.GetProfile().Ed25519PublicKey = pk
 	peer.GetProfile().Onion = string(onion)
-	fileStore := storage2.CreateFileProfileStore(filename, password)
-	err = fileStore.Save(peer)
+	fileStore := storage2.NewFileStore(filename, password)
+	err = fileStore.save(peer)
 	if err != nil {
 		return err
 	}
@@ -87,6 +47,7 @@ func convertTorFile(filename string, password string) error {
 	return nil*/
 }
 
+/*
 func vanity() error {
 	for {
 		pk, sk, err := ed25519.GenerateKey(rand.Reader)
@@ -100,16 +61,14 @@ func vanity() error {
 				peer.GetProfile().Ed25519PrivateKey = sk
 				peer.GetProfile().Ed25519PublicKey = pk
 				peer.GetProfile().Onion = onion
-				fileStore := storage2.CreateFileProfileStore(os.Args[3], onion+".cwtch")
-				err := fileStore.Save(peer)
-				if err != nil {
-					return err
-				}
+				profileStore, _ := storage2.NewProfileStore(nil, os.Args[3], onion+".cwtch")
+				profileStore.Init("")
+				// need to signal new onion? impossible
 				log.Infof("found %s.onion\n", onion)
 			}
 		}
 	}
-}
+}*/
 
 func printHelp() {
 	log.Infoln("usage: cwtchutil {help, convert-cwtch-file, convert-tor-file, changepw, vanity}")
@@ -127,15 +86,6 @@ func main() {
 		printHelp()
 	case "help":
 		printHelp()
-	case "convert-cwtch-file":
-		if len(os.Args) != 4 {
-			fmt.Println("example: cwtchutil convert-cwtch-file ~/.cwtch/profiles/11ddd78a9918c064e742d5e36a8b8fd4 passw0rd")
-			os.Exit(1)
-		}
-		err := convertCwtchFile(os.Args[2], os.Args[3])
-		if err != nil {
-			log.Errorln(err)
-		}
 	case "convert-tor-file":
 		if len(os.Args) != 4 {
 			fmt.Println("example: cwtchutil convert-tor-file /var/lib/tor/hs1 passw0rd")
@@ -145,7 +95,7 @@ func main() {
 		if err != nil {
 			log.Errorln(err)
 		}
-	case "vanity":
+		/*case "vanity":
 		if len(os.Args) < 5 {
 			fmt.Println("example: cwtchutil vanity 4 passw0rd erinn openpriv")
 			os.Exit(1)
@@ -163,8 +113,8 @@ func main() {
 
 		for { // run until ctrl+c
 			time.Sleep(time.Hour * 24)
-		}
-	case "changepw":
+		}*/
+		/*case "changepw":
 		if len(os.Args) != 3 {
 			fmt.Println("example: cwtch changepw ~/.cwtch/profiles/XXX")
 			os.Exit(1)
@@ -179,9 +129,9 @@ func main() {
 		}
 		pw = pw[:len(pw)-1]
 
-		fileStore := storage2.CreateFileProfileStore(os.Args[2], pw)
+		profileStore, _ := storage.NewProfileStore(nil, os.Args[2], pw)
 
-		peer, err := fileStore.Load()
+		err = profileStore.Read()
 		if err != nil {
 			log.Errorln(err)
 			os.Exit(1)
@@ -195,13 +145,15 @@ func main() {
 		}
 		newpw1 = newpw1[:len(newpw1)-1] // fuck go with this linebreak shit ^ea
 
-		fileStore2 := storage2.CreateFileProfileStore(os.Args[2], newpw1)
-		err = fileStore2.Save(peer)
+		fileStore2, _ := storage.NewProfileStore(nil, os.Args[2], newpw1)
+		// No way to copy, populate this method
+		err = fileStore2.save(peer)
 		if err != nil {
 			log.Errorln(err)
 			os.Exit(1)
 		}
 
 		log.Infoln("success!")
+		*/
 	}
 }

app/peer/alice/alice.go

@@ -1,10 +1,11 @@
 package main
 
 import (
+	app2 "cwtch.im/cwtch/app"
+	"cwtch.im/cwtch/app/utils"
 	"cwtch.im/cwtch/event"
-	"cwtch.im/cwtch/peer"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
 	"os"
 	"path"
 )
@@ -15,24 +16,21 @@ func main() {
 	log.AddEverythingFromPattern("peer/alice")
 	log.SetLevel(log.LevelDebug)
 
-	acn, err := connectivity.StartTor(path.Join(".", ".cwtch"), "")
+	acn, err := tor.NewTorACN(path.Join(".", ".cwtch"), "")
 	if err != nil {
 		log.Errorf("\nError connecting to Tor: %v\n", err)
 		os.Exit(1)
 	}
 
-	// Setup the Event Bus to Listen for Data Packets
-	eventBus := new(event.Manager)
-	eventBus.Initialize()
-	queue := event.NewEventQueue(100)
-	eventBus.Subscribe(event.NewMessageFromPeer, queue.EventChannel)
+	app := app2.NewApp(acn, ".")
+	app.CreatePeer("alice", "be gay, do crimes")
+	alice := utils.WaitGetPeer(app, "alice")
+	app.LaunchPeers()
+	eventBus := app.GetEventBus(alice.GetOnion())
+	queue := event.NewQueue()
+	eventBus.Subscribe(event.NewMessageFromPeer, queue)
 
-	// Setup Alice to Listen for new Events
-	alice := peer.NewCwtchPeer("alice")
-	alice.Init(acn, eventBus)
-	alice.Listen()
-
-	// For every new Data Packet Alice recieved she will Print it out.
+	// For every new Data Packet Alice received she will Print it out.
 	for {
 		event := queue.Next()
 		log.Printf(log.LevelInfo, "Received %v from %v: %s", event.EventType, event.Data["Onion"], event.Data["Data"])

app/peer/bob/bob.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"cwtch.im/cwtch/event"
-	"cwtch.im/cwtch/peer"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	app2 "cwtch.im/cwtch/app"
+	"cwtch.im/cwtch/app/utils"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
 	"os"
 	"path"
 	"time"
@@ -15,17 +15,15 @@ func main() {
 	// System Boilerplate, We need Tor Up and Running
 	log.AddEverythingFromPattern("peer/bob")
 	log.SetLevel(log.LevelDebug)
-	acn, err := connectivity.StartTor(path.Join(".", ".cwtch"), "")
+	acn, err := tor.NewTorACN(path.Join(".", ".cwtch"), "")
 	if err != nil {
 		log.Errorf("\nError connecting to Tor: %v\n", err)
 		os.Exit(1)
 	}
 
-	// Set up the Event Buss and Initialize the Peer
-	eventBus := new(event.Manager)
-	eventBus.Initialize()
-	bob := peer.NewCwtchPeer("bob")
-	bob.Init(acn, eventBus)
+	app := app2.NewApp(acn, ".")
+	app.CreatePeer("bob", "be gay, do crimes")
+	bob := utils.WaitGetPeer(app, "bob")
 
 	// Add Alice's Onion Here (It changes run to run)
 	bob.PeerWithOnion("upiztu7myymjf2dn4x4czhagp7axlnqjvf5zwfegbhtpkqb6v3vgu5yd")

app/plugins/contactRetry.go

@@ -0,0 +1,161 @@
+package plugins
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/protocol/connections"
+	"git.openprivacy.ca/openprivacy/log"
+	"sync"
+	"time"
+)
+
+const tickTime = 10 * time.Second
+const maxBakoff int = 32 // 320 seconds or ~5 min
+
+type connectionType int
+
+const (
+	peerConn connectionType = iota
+	serverConn
+)
+
+type contact struct {
+	id    string
+	state connections.ConnectionState
+	ctype connectionType
+
+	ticks   int
+	backoff int
+}
+
+type contactRetry struct {
+	bus       event.Manager
+	queue     event.Queue
+	networkUp bool
+	running   bool
+	breakChan chan bool
+	onion     string
+	lastCheck time.Time
+
+	connections sync.Map //[string]*contact
+}
+
+// NewConnectionRetry returns a Plugin that when started will retry connecting to contacts with a backoff timing
+func NewConnectionRetry(bus event.Manager, onion string) Plugin {
+	cr := &contactRetry{bus: bus, queue: event.NewQueue(), breakChan: make(chan bool), connections: sync.Map{}, networkUp: false, onion: onion}
+	return cr
+}
+
+func (cr *contactRetry) Start() {
+	if !cr.running {
+		go cr.run()
+	} else {
+		log.Errorf("Attempted to start Contact Retry plugin twice for %v", cr.onion)
+	}
+}
+
+func (cr *contactRetry) run() {
+	cr.running = true
+	cr.bus.Subscribe(event.PeerStateChange, cr.queue)
+	cr.bus.Subscribe(event.ACNStatus, cr.queue)
+	cr.bus.Subscribe(event.ServerStateChange, cr.queue)
+
+	for {
+		if time.Since(cr.lastCheck) > tickTime {
+			cr.retryDisconnected()
+			cr.lastCheck = time.Now()
+		}
+		select {
+		case e := <-cr.queue.OutChan():
+			switch e.EventType {
+			case event.PeerStateChange:
+				state := connections.ConnectionStateToType()[e.Data[event.ConnectionState]]
+				peer := e.Data[event.RemotePeer]
+				cr.handleEvent(peer, state, peerConn)
+
+			case event.ServerStateChange:
+				state := connections.ConnectionStateToType()[e.Data[event.ConnectionState]]
+				server := e.Data[event.GroupServer]
+				cr.handleEvent(server, state, serverConn)
+
+			case event.ACNStatus:
+				prog := e.Data[event.Progress]
+				if prog == "100" && !cr.networkUp {
+					cr.networkUp = true
+					cr.connections.Range(func(k, v interface{}) bool {
+						p := v.(*contact)
+						p.ticks = 0
+						p.backoff = 1
+						if p.ctype == peerConn {
+							cr.bus.Publish(event.NewEvent(event.RetryPeerRequest, map[event.Field]string{event.RemotePeer: p.id}))
+						}
+						if p.ctype == serverConn {
+							cr.bus.Publish(event.NewEvent(event.RetryServerRequest, map[event.Field]string{event.GroupServer: p.id}))
+						}
+						return true
+					})
+				} else if prog != "100" {
+					cr.networkUp = false
+				}
+			}
+
+		case <-time.After(tickTime):
+			continue
+
+		case <-cr.breakChan:
+			cr.running = false
+			return
+		}
+	}
+}
+
+func (cr *contactRetry) retryDisconnected() {
+	cr.connections.Range(func(k, v interface{}) bool {
+		p := v.(*contact)
+
+		if p.state == connections.DISCONNECTED {
+			p.ticks++
+			if p.ticks >= p.backoff {
+				p.ticks = 0
+				if cr.networkUp {
+					if p.ctype == peerConn {
+						cr.bus.Publish(event.NewEvent(event.RetryPeerRequest, map[event.Field]string{event.RemotePeer: p.id}))
+					}
+					if p.ctype == serverConn {
+						cr.bus.Publish(event.NewEvent(event.RetryServerRequest, map[event.Field]string{event.GroupServer: p.id}))
+					}
+				}
+			}
+		}
+		return true
+	})
+}
+
+func (cr *contactRetry) handleEvent(id string, state connections.ConnectionState, ctype connectionType) {
+	if _, exists := cr.connections.Load(id); !exists {
+		p := &contact{id: id, state: connections.DISCONNECTED, backoff: 0, ticks: 0, ctype: ctype}
+		cr.connections.Store(id, p)
+		return
+	}
+
+	pinf, _ := cr.connections.Load(id)
+	p := pinf.(*contact)
+	if state == connections.DISCONNECTED || state == connections.FAILED || state == connections.KILLED {
+		p.state = connections.DISCONNECTED
+		if p.backoff == 0 {
+			p.backoff = 1
+		} else if p.backoff < maxBakoff {
+			p.backoff *= 2
+		}
+		p.ticks = 0
+	} else if state == connections.CONNECTING || state == connections.CONNECTED {
+		p.state = state
+	} else if state == connections.AUTHENTICATED {
+		p.state = state
+		p.backoff = 0
+	}
+}
+
+func (cr *contactRetry) Shutdown() {
+	cr.breakChan <- true
+
+}

app/plugins/networkCheck.go

@@ -0,0 +1,175 @@
+package plugins
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/protocol/connections"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/log"
+	"sync"
+	"time"
+)
+
+// NetworkCheckError is a status for when the NetworkCheck Plugin has had an error making an out going connection indicating it may be offline
+const NetworkCheckError = "Error"
+
+// NetworkCheckSuccess is a status for when the NetworkCheck Plugin has had a successful message from a peer, indicating it is online right now
+const NetworkCheckSuccess = "Success"
+
+// networkCheck is a convenience plugin for testing high level availability of onion services
+type networkCheck struct {
+	bus           event.Manager
+	queue         event.Queue
+	acn           connectivity.ACN
+	onionsToCheck sync.Map // onion:string => true:bool
+	breakChan     chan bool
+	running       bool
+	offline       bool
+	offlineLock   sync.Mutex
+}
+
+// NewNetworkCheck returns a Plugin that when started will attempt various network tests
+func NewNetworkCheck(bus event.Manager, acn connectivity.ACN) Plugin {
+	nc := &networkCheck{bus: bus, acn: acn, queue: event.NewQueue(), breakChan: make(chan bool, 1)}
+	return nc
+}
+
+func (nc *networkCheck) Start() {
+	go nc.run()
+}
+
+func (nc *networkCheck) run() {
+	nc.running = true
+	nc.offline = true
+	nc.bus.Subscribe(event.ProtocolEngineStartListen, nc.queue)
+	nc.bus.Subscribe(event.NewMessageFromPeer, nc.queue)
+	nc.bus.Subscribe(event.PeerAcknowledgement, nc.queue)
+	nc.bus.Subscribe(event.EncryptedGroupMessage, nc.queue)
+	nc.bus.Subscribe(event.PeerStateChange, nc.queue)
+	nc.bus.Subscribe(event.ServerStateChange, nc.queue)
+	nc.bus.Subscribe(event.NewGetValMessageFromPeer, nc.queue)
+	nc.bus.Subscribe(event.NewRetValMessageFromPeer, nc.queue)
+	var lastMessageReceived time.Time
+	for {
+		select {
+		case <-nc.breakChan:
+			nc.running = false
+			return
+		case e := <-nc.queue.OutChan():
+			switch e.EventType {
+			// On receipt of a Listen request for an onion service we will add the onion to our list
+			// and then we will wait a minute and check the connection for the first time (the onion should be up)
+			// under normal operating circumstances
+			case event.ProtocolEngineStartListen:
+				if _, exists := nc.onionsToCheck.Load(e.Data[event.Onion]); !exists {
+					log.Debugf("initiating connection check for %v", e.Data[event.Onion])
+					nc.onionsToCheck.Store(e.Data[event.Onion], true)
+					if time.Since(lastMessageReceived) > time.Minute {
+						nc.selfTest()
+					}
+				}
+			case event.PeerStateChange:
+				fallthrough
+			case event.ServerStateChange:
+				// if we successfully connect / authenticated to a remote server / peer then we obviously have internet
+				connectionState := e.Data[event.ConnectionState]
+				nc.offlineLock.Lock()
+				if connectionState == connections.ConnectionStateName[connections.AUTHENTICATED] || connectionState == connections.ConnectionStateName[connections.CONNECTED] {
+					lastMessageReceived = time.Now()
+
+					if nc.offline {
+						nc.bus.Publish(event.NewEvent(event.NetworkStatus, map[event.Field]string{event.Error: "", event.Status: NetworkCheckSuccess}))
+						nc.offline = false
+					}
+				}
+				nc.offlineLock.Unlock()
+			default:
+				// if we receive either an encrypted group message or a peer acknowledgement we can assume the network
+				// is up and running (our onion service might still not be available, but we would aim to detect that
+				// through other actions
+				// we reset out timer
+				lastMessageReceived = time.Now()
+				nc.offlineLock.Lock()
+				if nc.offline {
+					nc.bus.Publish(event.NewEvent(event.NetworkStatus, map[event.Field]string{event.Error: "", event.Status: NetworkCheckSuccess}))
+					nc.offline = false
+				}
+				nc.offlineLock.Unlock()
+			}
+		case <-time.After(tickTime):
+			// if we haven't received an action in the last minute...kick off a set of testing
+			if time.Since(lastMessageReceived) > time.Minute {
+				nc.selfTest()
+			}
+		}
+	}
+}
+
+func (nc *networkCheck) Shutdown() {
+	if nc.running {
+		nc.queue.Shutdown()
+		log.Debugf("shutting down network status plugin")
+		nc.breakChan <- true
+	}
+}
+
+func (nc *networkCheck) selfTest() {
+	nc.onionsToCheck.Range(func(key, val interface{}) bool {
+		go nc.checkConnection(key.(string))
+		return true
+	})
+}
+
+//
+func (nc *networkCheck) checkConnection(onion string) {
+	prog, _ := nc.acn.GetBootstrapStatus()
+	if prog != 100 {
+		return
+	}
+
+	// we want to definitively time these actions out faster than tor will, because these onions should definitely be
+	// online
+	ClientTimeout := TimeoutPolicy(time.Second * 60)
+	err := ClientTimeout.ExecuteAction(func() error {
+		conn, _, err := nc.acn.Open(onion)
+		if err == nil {
+			conn.Close()
+		}
+		return err
+	})
+	nc.offlineLock.Lock()
+	defer nc.offlineLock.Unlock()
+	// regardless of the outcome we want to report a status to let anyone who might care know that we did do a check
+	if err != nil {
+		log.Debugf("publishing network error for %v -- %v\n", onion, err)
+		nc.bus.Publish(event.NewEvent(event.NetworkStatus, map[event.Field]string{event.Onion: onion, event.Error: err.Error(), event.Status: NetworkCheckError}))
+		nc.offline = true
+	} else {
+		log.Debugf("publishing network success for %v", onion)
+		nc.bus.Publish(event.NewEvent(event.NetworkStatus, map[event.Field]string{event.Onion: onion, event.Error: "", event.Status: NetworkCheckSuccess}))
+		nc.offline = false
+	}
+}
+
+// TODO we might want to reuse this, but for now it is only used by this plugin so it can live here
+
+// TimeoutPolicy is an interface for enforcing common timeout patterns
+type TimeoutPolicy time.Duration
+
+// ExecuteAction runs a function and returns an error if it hasn't returned
+// by the time specified by TimeoutPolicy
+func (tp *TimeoutPolicy) ExecuteAction(action func() error) error {
+
+	c := make(chan error)
+	go func() {
+		c <- action()
+	}()
+
+	tick := time.NewTicker(time.Duration(*tp))
+	select {
+	case <-tick.C:
+		return fmt.Errorf("ActionTimedOutError")
+	case err := <-c:
+		return err
+	}
+}

app/plugins/plugin.go

@@ -0,0 +1,33 @@
+package plugins
+
+import (
+	"cwtch.im/cwtch/event"
+	"git.openprivacy.ca/openprivacy/connectivity"
+)
+
+// PluginID is used as an ID for signaling plugin activities
+type PluginID int
+
+// These are the plugin IDs for the supplied plugins
+const (
+	CONNECTIONRETRY PluginID = iota
+	NETWORKCHECK
+)
+
+// Plugin is the interface for a plugin
+type Plugin interface {
+	Start()
+	Shutdown()
+}
+
+// Get is a plugin factory for the requested plugin
+func Get(id PluginID, bus event.Manager, acn connectivity.ACN, onion string) Plugin {
+	switch id {
+	case CONNECTIONRETRY:
+		return NewConnectionRetry(bus, onion)
+	case NETWORKCHECK:
+		return NewNetworkCheck(bus, acn)
+	}
+
+	return nil
+}

app/utils/utils.go

@@ -0,0 +1,25 @@
+package utils
+
+import (
+	app2 "cwtch.im/cwtch/app"
+	"cwtch.im/cwtch/model/attr"
+	"cwtch.im/cwtch/peer"
+	"time"
+)
+
+// WaitGetPeer is a helper function for utility apps not written using the event bus
+// Proper use of an App is to call CreatePeer and then process the NewPeer event
+// however for small utility use, this function which polls the app until the peer is created
+// may fill that usecase better
+func WaitGetPeer(app app2.Application, name string) peer.CwtchPeer {
+	for {
+		for id := range app.ListPeers() {
+			peer := app.GetPeer(id)
+			localName, _ := peer.GetAttribute(attr.GetLocalScope("name"))
+			if localName == name {
+				return peer
+			}
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}

event/EventQueue.go

@@ -1,31 +0,0 @@
-package event
-
-// Queue is a wrapper around a channel for handling Events in a consistent way across subsystems.
-// The expectation is that each subsystem in Cwtch will manage a given an event.Queue fed from
-// the event.Manager.
-type Queue struct {
-	EventChannel chan Event
-}
-
-// NewEventQueue initializes an event.Queue of the given buffer size.
-func NewEventQueue(buffer int) *Queue {
-	queue := new(Queue)
-	queue.EventChannel = make(chan Event, buffer)
-	return queue
-}
-
-// Backlog returns the length of the queue backlog
-func (eq *Queue) Backlog() int {
-	return len(eq.EventChannel)
-}
-
-// Next returns the next available event from the front of the queue
-func (eq *Queue) Next() (event Event) {
-	event = <-eq.EventChannel
-	return
-}
-
-// Shutdown closes our EventChannel
-func (eq *Queue) Shutdown() {
-	close(eq.EventChannel)
-}

event/bridge/goChanBridge.go

@@ -0,0 +1,57 @@
+package bridge
+
+import (
+	"cwtch.im/cwtch/event"
+	"sync"
+)
+
+type goChanBridge struct {
+	in         chan event.IPCMessage
+	out        chan event.IPCMessage
+	closedChan chan bool
+	closed     bool
+	lock       sync.Mutex
+}
+
+// MakeGoChanBridge returns a simple testing IPCBridge made from inprocess go channels
+func MakeGoChanBridge() (b1, b2 event.IPCBridge) {
+	chan1 := make(chan event.IPCMessage)
+	chan2 := make(chan event.IPCMessage)
+	closed := make(chan bool)
+
+	a := &goChanBridge{in: chan1, out: chan2, closedChan: closed, closed: false}
+	b := &goChanBridge{in: chan2, out: chan1, closedChan: closed, closed: false}
+
+	go monitor(a, b)
+
+	return a, b
+}
+
+func monitor(a, b *goChanBridge) {
+	<-a.closedChan
+	a.closed = true
+	b.closed = true
+	a.closedChan <- true
+}
+
+func (pb *goChanBridge) Read() (*event.IPCMessage, bool) {
+	message, ok := <-pb.in
+	return &message, ok
+}
+
+func (pb *goChanBridge) Write(message *event.IPCMessage) {
+	pb.lock.Lock()
+	defer pb.lock.Unlock()
+	if !pb.closed {
+		pb.out <- *message
+	}
+}
+
+func (pb *goChanBridge) Shutdown() {
+	if !pb.closed {
+		close(pb.in)
+		close(pb.out)
+		pb.closedChan <- true
+		<-pb.closedChan
+	}
+}

event/bridge/infinite_chan.go

@@ -0,0 +1,72 @@
+package bridge
+
+/* Todo: When go generics ships, refactor this and event.infiniteChannel into one */
+
+// InfiniteChannel implements the Channel interface with an infinite buffer between the input and the output.
+type InfiniteChannel struct {
+	input, output chan interface{}
+	length        chan int
+	buffer        *InfiniteQueue
+}
+
+func newInfiniteChannel() *InfiniteChannel {
+	ch := &InfiniteChannel{
+		input:  make(chan interface{}),
+		output: make(chan interface{}),
+		length: make(chan int),
+		buffer: newInfiniteQueue(),
+	}
+	go ch.infiniteBuffer()
+	return ch
+}
+
+// In returns the input channel
+func (ch *InfiniteChannel) In() chan<- interface{} {
+	return ch.input
+}
+
+// Out returns the output channel
+func (ch *InfiniteChannel) Out() <-chan interface{} {
+	return ch.output
+}
+
+// Len returns the length of items in queue
+func (ch *InfiniteChannel) Len() int {
+	return <-ch.length
+}
+
+// Close closes the InfiniteChanel
+func (ch *InfiniteChannel) Close() {
+	close(ch.input)
+}
+
+func (ch *InfiniteChannel) infiniteBuffer() {
+	var input, output chan interface{}
+	var next interface{}
+	input = ch.input
+
+	for input != nil || output != nil {
+		select {
+		case elem, open := <-input:
+			if open {
+				ch.buffer.Add(elem)
+			} else {
+				input = nil
+			}
+		case output <- next:
+			ch.buffer.Remove()
+		case ch.length <- ch.buffer.Length():
+		}
+
+		if ch.buffer.Length() > 0 {
+			output = ch.output
+			next = ch.buffer.Peek()
+		} else {
+			output = nil
+			next = nil
+		}
+	}
+
+	close(ch.output)
+	close(ch.length)
+}

event/bridge/infinite_queue.go

@@ -0,0 +1,105 @@
+package bridge
+
+/* Todo: When go generics ships, refactor this and event.infinitQueue channel into one */
+
+/*
+Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki.
+Using this instead of other, simpler, queue implementations (slice+append or linked list) provides
+substantial memory and time benefits, and fewer GC pauses.
+
+The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe.
+*/
+
+// minQueueLen is smallest capacity that queue may have.
+// Must be power of 2 for bitwise modulus: x % n == x & (n - 1).
+const minQueueLen = 16
+
+// InfiniteQueue represents a single instance of the queue data structure.
+type InfiniteQueue struct {
+	buf               []interface{}
+	head, tail, count int
+}
+
+// New constructs and returns a new Queue.
+func newInfiniteQueue() *InfiniteQueue {
+	return &InfiniteQueue{
+		buf: make([]interface{}, minQueueLen),
+	}
+}
+
+// Length returns the number of elements currently stored in the queue.
+func (q *InfiniteQueue) Length() int {
+	return q.count
+}
+
+// resizes the queue to fit exactly twice its current contents
+// this can result in shrinking if the queue is less than half-full
+func (q *InfiniteQueue) resize() {
+	newBuf := make([]interface{}, q.count<<1)
+
+	if q.tail > q.head {
+		copy(newBuf, q.buf[q.head:q.tail])
+	} else {
+		n := copy(newBuf, q.buf[q.head:])
+		copy(newBuf[n:], q.buf[:q.tail])
+	}
+
+	q.head = 0
+	q.tail = q.count
+	q.buf = newBuf
+}
+
+// Add puts an element on the end of the queue.
+func (q *InfiniteQueue) Add(elem interface{}) {
+	if q.count == len(q.buf) {
+		q.resize()
+	}
+
+	q.buf[q.tail] = elem
+	// bitwise modulus
+	q.tail = (q.tail + 1) & (len(q.buf) - 1)
+	q.count++
+}
+
+// Peek returns the element at the head of the queue. This call panics
+// if the queue is empty.
+func (q *InfiniteQueue) Peek() interface{} {
+	if q.count <= 0 {
+		panic("queue: Peek() called on empty queue")
+	}
+	return q.buf[q.head]
+}
+
+// Get returns the element at index i in the queue. If the index is
+// invalid, the call will panic. This method accepts both positive and
+// negative index values. Index 0 refers to the first element, and
+// index -1 refers to the last.
+func (q *InfiniteQueue) Get(i int) interface{} {
+	// If indexing backwards, convert to positive index.
+	if i < 0 {
+		i += q.count
+	}
+	if i < 0 || i >= q.count {
+		panic("queue: Get() called with index out of range")
+	}
+	// bitwise modulus
+	return q.buf[(q.head+i)&(len(q.buf)-1)]
+}
+
+// Remove removes and returns the element from the front of the queue. If the
+// queue is empty, the call will panic.
+func (q *InfiniteQueue) Remove() interface{} {
+	if q.count <= 0 {
+		panic("queue: Remove() called on empty queue")
+	}
+	ret := q.buf[q.head]
+	q.buf[q.head] = nil
+	// bitwise modulus
+	q.head = (q.head + 1) & (len(q.buf) - 1)
+	q.count--
+	// Resize down if buffer 1/4 full.
+	if len(q.buf) > minQueueLen && (q.count<<2) == len(q.buf) {
+		q.resize()
+	}
+	return ret
+}

event/bridge/pipeBridge-windows.go

@@ -0,0 +1,19 @@
+// +build windows
+
+package bridge
+
+import (
+	"cwtch.im/cwtch/event"
+	"log"
+)
+
+func NewPipeBridgeClient(inFilename, outFilename string) event.IPCBridge {
+	log.Fatal("Not supported on windows")
+	return nil
+}
+
+// NewPipeBridgeService returns a pipe backed IPCBridge for a service
+func NewPipeBridgeService(inFilename, outFilename string) event.IPCBridge {
+	log.Fatal("Not supported on windows")
+	return nil
+}

event/bridge/pipeBridge.go

@@ -0,0 +1,357 @@
+// +build !windows
+
+package bridge
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/protocol/connections"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"git.openprivacy.ca/openprivacy/log"
+	"os"
+	"sync"
+	"syscall"
+	"time"
+)
+
+/* pipeBridge creates a pair of named pipes
+   Needs a call to new client and service to fully successfully open
+*/
+
+const maxBufferSize = 1000
+
+const serviceName = "service"
+const clientName = "client"
+
+const syn = "SYN"
+const synack = "SYNACK"
+const ack = "ACK"
+
+type pipeBridge struct {
+	infile, outfile string
+	in, out         *os.File
+	read            chan event.IPCMessage
+	write           *InfiniteChannel
+	closedChan      chan bool
+	state           connections.ConnectionState
+	lock            sync.Mutex
+	threeShake      func() bool
+
+	// For logging / debugging purposes
+	name string
+}
+
+func newPipeBridge(inFilename, outFilename string) *pipeBridge {
+	syscall.Mkfifo(inFilename, 0600)
+	syscall.Mkfifo(outFilename, 0600)
+	pb := &pipeBridge{infile: inFilename, outfile: outFilename, state: connections.DISCONNECTED}
+	pb.read = make(chan event.IPCMessage, maxBufferSize)
+	pb.write = newInfiniteChannel() //make(chan event.IPCMessage, maxBufferSize)
+	return pb
+}
+
+// NewPipeBridgeClient returns a pipe backed IPCBridge for a client
+func NewPipeBridgeClient(inFilename, outFilename string) event.IPCBridge {
+	log.Debugf("Making new PipeBridge Client...\n")
+	pb := newPipeBridge(inFilename, outFilename)
+	pb.name = clientName
+	pb.threeShake = pb.threeShakeClient
+	go pb.connectionManager()
+
+	return pb
+}
+
+// NewPipeBridgeService returns a pipe backed IPCBridge for a service
+func NewPipeBridgeService(inFilename, outFilename string) event.IPCBridge {
+	log.Debugf("Making new PipeBridge Service...\n")
+	pb := newPipeBridge(inFilename, outFilename)
+	pb.name = serviceName
+	pb.threeShake = pb.threeShakeService
+
+	go pb.connectionManager()
+
+	log.Debugf("Successfully created new PipeBridge Service!\n")
+	return pb
+}
+
+func (pb *pipeBridge) setState(state connections.ConnectionState) {
+	pb.lock.Lock()
+	defer pb.lock.Unlock()
+
+	pb.state = state
+}
+
+func (pb *pipeBridge) getState() connections.ConnectionState {
+	pb.lock.Lock()
+	defer pb.lock.Unlock()
+
+	return pb.state
+}
+
+func (pb *pipeBridge) connectionManager() {
+	for pb.getState() != connections.KILLED {
+		log.Debugf("clientConnManager loop start init\n")
+		pb.setState(connections.CONNECTING)
+
+		var err error
+		log.Debugf("%v open file infile\n", pb.name)
+		pb.in, err = os.OpenFile(pb.infile, os.O_RDWR, 0600)
+		if err != nil {
+			pb.setState(connections.DISCONNECTED)
+			continue
+		}
+
+		log.Debugf("%v open file outfile\n", pb.name)
+		pb.out, err = os.OpenFile(pb.outfile, os.O_RDWR, 0600)
+		if err != nil {
+			pb.setState(connections.DISCONNECTED)
+			continue
+		}
+
+		log.Debugf("Successfully connected PipeBridge %v!\n", pb.name)
+
+		pb.handleConns()
+	}
+	log.Debugf("exiting %v ConnectionManager\n", pb.name)
+
+}
+
+// threeShake performs a 3way handshake sync up
+func (pb *pipeBridge) threeShakeService() bool {
+	synacked := false
+
+	for {
+		resp, err := pb.readString()
+		if err != nil {
+			return false
+		}
+
+		if string(resp) == syn {
+			if !synacked {
+				err = pb.writeString([]byte(synack))
+				if err != nil {
+					return false
+				}
+				synacked = true
+			}
+		} else if string(resp) == ack {
+			return true
+		}
+	}
+}
+
+func (pb *pipeBridge) synLoop(stop chan bool) {
+	delay := time.Duration(0)
+	for {
+		select {
+		case <-time.After(delay):
+			err := pb.writeString([]byte(syn))
+			if err != nil {
+				return
+			}
+			delay = time.Second
+		case <-stop:
+			return
+		}
+	}
+}
+
+func (pb *pipeBridge) threeShakeClient() bool {
+	stop := make(chan bool)
+	go pb.synLoop(stop)
+	for {
+		resp, err := pb.readString()
+		if err != nil {
+			return false
+		}
+
+		if string(resp) == synack {
+			stop <- true
+			err := pb.writeString([]byte(ack))
+			return err == nil
+		}
+	}
+}
+
+func (pb *pipeBridge) handleConns() {
+
+	if !pb.threeShake() {
+		pb.setState(connections.FAILED)
+		pb.closeReset()
+		return
+	}
+
+	pb.setState(connections.AUTHENTICATED)
+
+	pb.closedChan = make(chan bool, 5)
+
+	log.Debugf("handleConns authed, %v 2xgo\n", pb.name)
+
+	go pb.handleRead()
+	go pb.handleWrite()
+
+	<-pb.closedChan
+	log.Debugf("handleConns <-closedChan (%v)\n", pb.name)
+	if pb.getState() != connections.KILLED {
+		pb.setState(connections.FAILED)
+	}
+	pb.closeReset()
+	log.Debugf("handleConns done for %v, exit\n", pb.name)
+}
+
+func (pb *pipeBridge) closeReset() {
+	pb.in.Close()
+	pb.out.Close()
+	close(pb.read)
+	pb.write.Close()
+
+	if pb.getState() != connections.KILLED {
+		pb.read = make(chan event.IPCMessage, maxBufferSize)
+		pb.write = newInfiniteChannel()
+	}
+}
+
+func (pb *pipeBridge) handleWrite() {
+	log.Debugf("handleWrite() %v\n", pb.name)
+	defer log.Debugf("exiting handleWrite() %v\n", pb.name)
+
+	for {
+		select {
+		case messageInf := <-pb.write.output:
+			if messageInf == nil {
+				pb.closedChan <- true
+				return
+			}
+			message := messageInf.(event.IPCMessage)
+			if message.Message.EventType == event.EncryptedGroupMessage || message.Message.EventType == event.SendMessageToGroup || message.Message.EventType == event.NewMessageFromGroup {
+				log.Debugf("handleWrite <- message: %v %v ...\n", message.Dest, message.Message.EventType)
+			} else {
+				log.Debugf("handleWrite <- message: %v\n", message)
+			}
+			if pb.getState() == connections.AUTHENTICATED {
+				encMessage := &event.IPCMessage{Dest: message.Dest, Message: event.Event{EventType: message.Message.EventType, EventID: message.Message.EventID, Data: make(map[event.Field]string)}}
+				for k, v := range message.Message.Data {
+					encMessage.Message.Data[k] = base64.StdEncoding.EncodeToString([]byte(v))
+				}
+
+				messageJSON, _ := json.Marshal(encMessage)
+				err := pb.writeString(messageJSON)
+				if err != nil {
+					pb.closedChan <- true
+					return
+				}
+			} else {
+				return
+			}
+		}
+	}
+}
+
+func (pb *pipeBridge) handleRead() {
+	log.Debugf("handleRead() %v\n", pb.name)
+	defer log.Debugf("exiting handleRead() %v", pb.name)
+
+	for {
+		log.Debugf("Waiting to handleRead()...\n")
+
+		buffer, err := pb.readString()
+		if err != nil {
+			pb.closedChan <- true
+			return
+		}
+
+		var message event.IPCMessage
+		err = json.Unmarshal(buffer, &message)
+		if err != nil {
+			log.Errorf("Read error: '%v', value: '%v'", err, buffer)
+			pb.closedChan <- true
+			return // probably new connection trying to initialize
+		}
+		for k, v := range message.Message.Data {
+			val, _ := base64.StdEncoding.DecodeString(v)
+			message.Message.Data[k] = string(val)
+		}
+		if message.Message.EventType == event.EncryptedGroupMessage || message.Message.EventType == event.SendMessageToGroup || message.Message.EventType == event.NewMessageFromGroup {
+			log.Debugf("handleRead read<-: %v %v ...\n", message.Dest, message.Message.EventType)
+		} else {
+			log.Debugf("handleRead read<-: %v\n", message)
+		}
+		pb.read <- message
+		log.Debugf("handleRead wrote\n")
+	}
+}
+
+func (pb *pipeBridge) Read() (*event.IPCMessage, bool) {
+	log.Debugf("Read() %v...\n", pb.name)
+	var ok = false
+	var message event.IPCMessage
+	for !ok && pb.getState() != connections.KILLED {
+		message, ok = <-pb.read
+		if message.Message.EventType == event.EncryptedGroupMessage || message.Message.EventType == event.SendMessageToGroup || message.Message.EventType == event.NewMessageFromGroup {
+			log.Debugf("Read %v: %v %v ...\n", pb.name, message.Dest, message.Message.EventType)
+		} else {
+			log.Debugf("Read %v: %v\n", pb.name, message)
+		}
+	}
+	return &message, pb.getState() != connections.KILLED
+}
+
+func (pb *pipeBridge) Write(message *event.IPCMessage) {
+	if message.Message.EventType == event.EncryptedGroupMessage || message.Message.EventType == event.SendMessageToGroup || message.Message.EventType == event.NewMessageFromGroup {
+		log.Debugf("Write %v: %v %v ...\n", pb.name, message.Dest, message.Message.EventType)
+	} else {
+		log.Debugf("Write %v: %v\n", pb.name, message)
+	}
+	pb.write.input <- *message
+	log.Debugf("Wrote\n")
+}
+
+func (pb *pipeBridge) Shutdown() {
+	log.Debugf("pb.Shutdown() for %v currently in state: %v\n", pb.name, connections.ConnectionStateName[pb.getState()])
+	pb.state = connections.KILLED
+	pb.closedChan <- true
+	log.Debugf("Done Shutdown for %v\n", pb.name)
+}
+
+func (pb *pipeBridge) writeString(message []byte) error {
+	size := make([]byte, 2)
+	binary.LittleEndian.PutUint16(size, uint16(len(message)))
+	pb.out.Write(size)
+
+	for pos := 0; pos < len(message); {
+		n, err := pb.out.Write(message[pos:])
+		if err != nil {
+			log.Errorf("Writing out on pipeBridge: %v\n", err)
+			return err
+		}
+		pos += n
+	}
+	return nil
+}
+
+func (pb *pipeBridge) readString() ([]byte, error) {
+	var n int
+	size := make([]byte, 2)
+	var err error
+
+	n, err = pb.in.Read(size)
+	if err != nil || n != 2 {
+		log.Errorf("Could not read len int from stream: %v\n", err)
+		return nil, err
+	}
+
+	n = int(binary.LittleEndian.Uint16(size))
+	pos := 0
+	buffer := make([]byte, n)
+	for n > 0 {
+		m, err := pb.in.Read(buffer[pos:])
+		if err != nil {
+			log.Errorf("Reading into buffer from pipe: %v\n", err)
+			return nil, err
+		}
+		n -= m
+		pos += m
+	}
+	return buffer, nil
+}

event/bridge/pipeBridge_test.go

@@ -0,0 +1,131 @@
+package bridge
+
+import (
+	"cwtch.im/cwtch/event"
+	"git.openprivacy.ca/openprivacy/log"
+	"os"
+	"testing"
+	"time"
+)
+
+var (
+	clientPipe  = "./client"
+	servicePipe = "./service"
+)
+
+func clientHelper(t *testing.T, in, out string, messageOrig *event.IPCMessage, done chan bool) {
+	client := NewPipeBridgeClient(in, out)
+
+	messageAfter, ok := client.Read()
+	if !ok {
+		t.Errorf("Reading from client IPCBridge failed")
+		done <- true
+		return
+	}
+
+	if messageOrig.Dest != messageAfter.Dest {
+		t.Errorf("Dest's value differs expected: %v actaul: %v", messageOrig.Dest, messageAfter.Dest)
+	}
+
+	if messageOrig.Message.EventType != messageAfter.Message.EventType {
+		t.Errorf("EventTypes's value differs expected: %v actaul: %v", messageOrig.Message.EventType, messageAfter.Message.EventType)
+	}
+
+	if messageOrig.Message.Data[event.Identity] != messageAfter.Message.Data[event.Identity] {
+		t.Errorf("Data[Identity]'s value differs expected: %v actaul: %v", messageOrig.Message.Data[event.Identity], messageAfter.Message.Data[event.Identity])
+	}
+
+	done <- true
+}
+
+func serviceHelper(t *testing.T, in, out string, messageOrig *event.IPCMessage, done chan bool) {
+	service := NewPipeBridgeService(in, out)
+
+	service.Write(messageOrig)
+
+	done <- true
+}
+
+func TestPipeBridge(t *testing.T) {
+	os.Remove(servicePipe)
+	os.Remove(clientPipe)
+
+	messageOrig := &event.IPCMessage{Dest: "ABC", Message: event.NewEventList(event.NewPeer, event.Identity, "It is I")}
+	serviceDone := make(chan bool)
+	clientDone := make(chan bool)
+
+	go clientHelper(t, clientPipe, servicePipe, messageOrig, clientDone)
+	go serviceHelper(t, servicePipe, clientPipe, messageOrig, serviceDone)
+
+	<-serviceDone
+	<-clientDone
+}
+
+func restartingClient(t *testing.T, in, out string, done chan bool) {
+	client := NewPipeBridgeClient(in, out)
+
+	message1 := &event.IPCMessage{Dest: "ABC", Message: event.NewEventList(event.NewPeer)}
+	log.Infoln("client writing message 1")
+	client.Write(message1)
+
+	time.Sleep(100 * time.Millisecond)
+	log.Infoln("client shutdown")
+	client.Shutdown()
+
+	log.Infoln("client new client")
+	client = NewPipeBridgeClient(in, out)
+	message2 := &event.IPCMessage{Dest: "ABC", Message: event.NewEventList(event.DeleteContact)}
+	log.Infoln("client2 write message2")
+	client.Write(message2)
+
+	done <- true
+}
+
+func stableService(t *testing.T, in, out string, done chan bool) {
+	service := NewPipeBridgeService(in, out)
+
+	log.Infoln("service wait read 1")
+	message1, ok := service.Read()
+	log.Infof("service read 1 %v ok:%v\n", message1, ok)
+	if !ok {
+		t.Errorf("Reading from client IPCBridge 1st time failed")
+		done <- true
+		return
+	}
+	if message1.Message.EventType != event.NewPeer {
+		t.Errorf("Wrong message received, expected NewPeer\n")
+		done <- true
+		return
+	}
+
+	log.Infoln("service wait read 2")
+	message2, ok := service.Read()
+	log.Infof("service read 2 got %v ok:%v\n", message2, ok)
+	if !ok {
+		t.Errorf("Reading from client IPCBridge 2nd time failed")
+		done <- true
+		return
+	}
+	if message2.Message.EventType != event.DeleteContact {
+		t.Errorf("Wrong message received, expected DeleteContact, got %v\n", message2)
+		done <- true
+		return
+	}
+
+	done <- true
+}
+
+func TestReconnect(t *testing.T) {
+	log.Infoln("TestReconnect")
+	os.Remove(servicePipe)
+	os.Remove(clientPipe)
+
+	serviceDone := make(chan bool)
+	clientDone := make(chan bool)
+
+	go restartingClient(t, clientPipe, servicePipe, clientDone)
+	go stableService(t, servicePipe, clientPipe, serviceDone)
+
+	<-serviceDone
+	<-clientDone
+}

event/common.go

@@ -8,20 +8,344 @@ const (
 	StatusRequest        = Type("StatusRequest")
 	ProtocolEngineStatus = Type("ProtocolEngineStatus")
 
+	// Attempt to outbound peer with a given remote peer
+	// attributes:
+	// RemotePeer: [eg "chpr7qm6op5vfcg2pi4vllco3h6aa7exexc4rqwnlupqhoogx2zgd6qd"
 	PeerRequest = Type("PeerRequest")
-	BlockPeer   = Type("BlockPeer")
-	JoinServer  = Type("JoinServer")
+
+	// RetryPeerRequest
+	// Identical to PeerRequest, but allows Engine to make decisions regarding blocked peers
+	// attributes:
+	// RemotePeer: [eg "chpr7qm6op5vfcg2pi4vllco3h6aa7exexc4rqwnlupqhoogx2zgd6qd"
+	RetryPeerRequest = Type("RetryPeerRequest")
+
+	// RetryServerRequest
+	// Asks CwtchPeer to retry a server connection...
+	// GroupServer: [eg "chpr7qm6op5vfcg2pi4vllco3h6aa7exexc4rqwnlupqhoogx2zgd6qd"
+	RetryServerRequest = Type("RetryServerRequest")
+
+	// RemotePeer
+	// Authorization(model.peer.Auth_...)
+	SetPeerAuthorization = Type("UpdatePeerAuthorization")
+
+	// Turn on/off blocking of unknown peers (if peers aren't in the contact list then they will be autoblocked
+	BlockUnknownPeers = Type("BlockUnknownPeers")
+	AllowUnknownPeers = Type("AllowUnknownPeers")
+
+	// GroupServer
+	JoinServer = Type("JoinServer")
+
+	// attributes GroupServer - the onion of the server to leave
+	LeaveServer = Type("LeaveServer")
 
 	ProtocolEngineStartListen = Type("ProtocolEngineStartListen")
 	ProtocolEngineStopped     = Type("ProtocolEngineStopped")
 
 	InvitePeerToGroup = Type("InvitePeerToGroup")
-	NewGroupInvite    = Type("NewGroupInvite")
 
-	SendMessageToGroup    = Type("SendMessagetoGroup")
+	// a group invite has been received from a remote peer
+	// attributes:
+	// TimestampReceived [eg time.Now().Format(time.RFC3339Nano)]
+	// RemotePeer: [eg "chpr7qm6op5vfcg2pi4vllco3h6aa7exexc4rqwnlupqhoogx2zgd6qd"]
+	// GroupInvite: [eg "torv3....."]
+	// Imported
+	NewGroupInvite = Type("NewGroupInvite")
+
+	// Inform the UI about a new group
+	// GroupID: groupID (allows them to fetch from the peer)
+	NewGroup = Type("NewGroup")
+
+	// GroupID
+	AcceptGroupInvite = Type("AcceptGroupInvite")
+	RejectGroupInvite = Type("RejectGroupInvite")
+
+	SendMessageToGroup = Type("SendMessagetoGroup")
+
+	//Ciphertext, Signature:
 	EncryptedGroupMessage = Type("EncryptedGroupMessage")
-	NewMessageFromGroup   = Type("NewMessageFromGroup")
+	//TimestampReceived, TimestampSent, Data(Message), GroupID, Signature, PreviousSignature, RemotePeer
+	NewMessageFromGroup = Type("NewMessageFromGroup")
+
+	// Sent if a Group Key is detected as being used outside of expected parameters (e.g. with tampered signatures)
+	// GroupID: The ID of the Group that is presumed compromised
+	GroupCompromised = Type("GroupCompromised")
+
+	// an error was encountered trying to send a particular Message to a group
+	// attributes:
+	// GroupServer: The server the Message was sent to
+	// Signature: The signature of the Message that failed to send
+	// Error: string describing the error
+	SendMessageToGroupError = Type("SendMessageToGroupError")
 
 	SendMessageToPeer  = Type("SendMessageToPeer")
 	NewMessageFromPeer = Type("NewMessageFromPeer")
+
+	// RemotePeer, scope, path
+	NewGetValMessageFromPeer = Type("NewGetValMessageFromPeer")
+
+	// RemotePeer, val, exists
+	SendRetValMessageToPeer = Type("SendRetValMessageToPeer")
+
+	// RemotePeer, scope, val
+	SendGetValMessageToPeer = Type("SendGetValMessageToPeer")
+
+	// RemotePeer, scope, path, data, exists
+	NewRetValMessageFromPeer = Type("NewRetValMessageFromPeer")
+
+	// Peer acknowledges a previously sent message
+	// attributes
+	// EventID: The original event id that the peer is responding too.
+	// RemotePeer: The peer associated with the acknowledgement
+	PeerAcknowledgement = Type("PeerAcknowledgement")
+
+	// Like PeerAcknowledgement but with message index instead of event ID
+	// attributes
+	// Index: The original index of the message that the peer is responding too.
+	// RemotePeer: The peer associated with the acknowledgement
+	IndexedAcknowledgement = Type("IndexedAcknowledgement")
+
+	// Like PeerAcknowledgement but with message index instead of event ID
+	// attributes
+	// Index: The original index of the message that the peer is responding too.
+	// RemotePeer: The peer associated with the acknowledgement
+	IndexedFailure = Type("IndexedFailure")
+
+	// UpdateMessageFlags will change the flags associated with a given message.
+	// Handle
+	// Message Index
+	// Flags
+	UpdateMessageFlags = Type("UpdateMessageFlags")
+
+	// attributes:
+	// RemotePeer: [eg "chpr7qm6op5vfcg2pi4vllco3h6aa7exexc4rqwnlupqhoogx2zgd6qd"]
+	// Error: string describing the error
+	SendMessageToPeerError = Type("SendMessageToPeerError")
+
+	// REQUESTS TO STORAGE ENGINE
+
+	// a peer contact has been added
+	// attributes:
+	// RemotePeer [eg ""]
+	// Authorization
+	PeerCreated = Type("PeerCreated")
+
+	// Password, NewPassword
+	ChangePassword = Type("ChangePassword")
+
+	// Error(err), EventID
+	ChangePasswordError = Type("ChangePasswordError")
+
+	// EventID
+	ChangePasswordSuccess = Type("ChangePasswordSuccess")
+
+	// a group has been successfully added or newly created
+	// attributes:
+	// Data [serialized *model.Group]
+	GroupCreated = Type("GroupCreated")
+
+	// RemotePeer
+	DeleteContact = Type("DeleteContact")
+
+	// GroupID
+	DeleteGroup = Type("DeleteGroup")
+
+	// change the .Name attribute of a profile (careful - this is not a custom attribute. it is used in the underlying protocol during handshakes!)
+	// attributes:
+	// ProfileName [eg "erinn"]
+	SetProfileName = Type("SetProfileName")
+
+	// request to store a profile-wide attribute (good for e.g. per-profile settings like theme prefs)
+	// attributes:
+	// Key [eg "fontcolor"]
+	// Data [eg "red"]
+	SetAttribute = Type("SetAttribute")
+
+	// request to store a per-contact attribute (e.g. display names for a peer)
+	// attributes:
+	// RemotePeer [eg ""]
+	// Key [eg "nick"]
+	// Data [eg "erinn"]
+	SetPeerAttribute = Type("SetPeerAttribute")
+
+	// request to store a per-cwtch-group attribute (e.g. display name for a group)
+	// attributes:
+	// GroupID [eg ""]
+	// Key [eg "nick"]
+	// Data [eg "open privacy board"]
+	SetGroupAttribute = Type("SetGroupAttribute")
+
+	// PeerStateChange servers as a new incoming connection message as well, and can/is consumed by frontends to alert of new p2p connections
+	// RemotePeer
+	// ConnectionState
+	PeerStateChange = Type("PeerStateChange")
+
+	// GroupServer
+	// ConnectionState
+	ServerStateChange = Type("ServerStateChange")
+
+	/***** Application client / service messages *****/
+
+	// ProfileName, Password, Data(tag)
+	CreatePeer = Type("CreatePeer")
+
+	// app: Identity(onion), Created(bool)
+	// service -> client: Identity(localId), Password, [Status(new/default=blank || from reload='running')], Created(bool)
+	NewPeer = Type("NewPeer")
+
+	// Identity(onion)
+	DeletePeer = Type("DeletePeer")
+
+	// Identity(onion), Data(pluginID)
+	AddPeerPlugin = Type("AddPeerPlugin")
+
+	// Password
+	LoadProfiles = Type("LoadProfiles")
+
+	// Client has reloaded, triggers NewPeer s then ReloadDone
+	ReloadClient = Type("ReloadClient")
+
+	ReloadDone = Type("ReloadDone")
+
+	// Identity - Ask service to resend all connection states
+	ReloadPeer = Type("ReloadPeer")
+
+	// Identity(onion)
+	ShutdownPeer = Type("ShutdownPeer")
+
+	Shutdown = Type("Shutdown")
+
+	// Error(err)
+	// Error creating peer
+	PeerError = Type("PeerError")
+
+	// Error(err)
+	AppError = Type("AppError")
+
+	GetACNStatus  = Type("GetACNStatus")
+	GetACNVersion = Type("GetACNVersion")
+
+	// Progress, Status
+	ACNStatus = Type("ACNStatus")
+
+	// Data
+	ACNVersion = Type("ACNVersion")
+
+	// Network Status
+	// Status: Success || Error
+	// Error: Description of the Error
+	// Onion: the local onion we attempt to check
+	NetworkStatus = Type("NetworkError")
+
+	// Notify the UI that a Server has been added
+	// Onion = Server Onion
+	ServerCreated = Type("ServerAdded")
+
+	// For debugging. Allows test to emit a Syn and get a response Ack(eventID) when the subsystem is done processing a queue
+	Syn = Type("Syn")
+	Ack = Type("Ack")
+)
+
+// Field defines common event attributes
+type Field string
+
+// Defining Common Field Types
+const (
+
+	// A peers local onion address
+	Onion = Field("Onion")
+
+	RemotePeer        = Field("RemotePeer")
+	Ciphertext        = Field("Ciphertext")
+	Signature         = Field("Signature")
+	PreviousSignature = Field("PreviousSignature")
+	TimestampSent     = Field("TimestampSent")
+	TimestampReceived = Field("TimestampReceived")
+
+	Identity = Field("Identity")
+
+	GroupID          = Field("GroupID")
+	GroupServer      = Field("GroupServer")
+	ServerTokenY     = Field("ServerTokenY")
+	ServerTokenOnion = Field("ServerTokenOnion")
+	GroupInvite      = Field("GroupInvite")
+
+	ProfileName = Field("ProfileName")
+	Password    = Field("Password")
+	NewPassword = Field("NewPassword")
+
+	Created = Field("Created")
+
+	ConnectionState = Field("ConnectionState")
+
+	Key    = Field("Key")
+	Data   = Field("Data")
+	Scope  = Field("Scope")
+	Path   = Field("Path")
+	Exists = Field("Exists")
+
+	Salt = Field("Salt")
+
+	Error = Field("Error")
+
+	Progress     = Field("Progress")
+	Status       = Field("Status")
+	EventID      = Field("EventID")
+	EventContext = Field("EventContext")
+	Index        = Field("Index")
+
+	// Handle denotes a contact handle of any type.
+	Handle = Field("Handle")
+
+	// Flags denotes a set of message flags
+	Flags = Field("Flags")
+
+	Authorization = Field("Authorization")
+
+	KeyBundle = Field("KeyBundle")
+
+	// Indicate whether an event was triggered by a user import
+	Imported = Field("Imported")
+
+	Source = Field("Source")
+)
+
+// Defining Common errors
+const (
+	AppErrLoaded0 = "Loaded 0 profiles"
+)
+
+// Values to be suplied in event.NewPeer for Status
+const (
+	StorageRunning = "running"
+	StorageNew     = "new"
+)
+
+// Defining Protocol Contexts
+const (
+	ContextAck    = "im.cwtch.acknowledgement"
+	ContextInvite = "im.cwtch.invite"
+	ContextRaw    = "im.cwtch.raw"
+	ContextGetVal = "im.cwtch.getVal"
+	ContextRetVal = "im.cwtch.retVal"
+)
+
+// Define Default Attribute Keys
+const (
+	SaveHistoryKey = "SavePeerHistory"
+)
+
+// Define Default Attribute Values
+const (
+	// Save History has 3 distinct states. By default we don't save history (DefaultDeleteHistory), if the peer confirms this
+	// we change to DeleteHistoryConfirmed, if they confirm they want to save then this becomes SaveHistoryConfirmed
+	// We use this distinction between default and confirmed to drive UI
+	DeleteHistoryDefault   = "DefaultDeleteHistory"
+	SaveHistoryConfirmed   = "SaveHistory"
+	DeleteHistoryConfirmed = "DeleteHistoryConfirmed"
+)
+
+// Bool strings
+const (
+	True  = "true"
+	False = "false"
 )

event/eventQueue.go

@@ -0,0 +1,112 @@
+package event
+
+import (
+	"sync"
+)
+
+type queue struct {
+	infChan infiniteChannel
+	lock    sync.Mutex
+	closed  bool
+}
+
+type simpleQueue struct {
+	eventChannel chan Event
+	lock         sync.Mutex
+	closed       bool
+}
+
+// Queue is a wrapper around a channel for handling Events in a consistent way across subsystems.
+// The expectation is that each subsystem in Cwtch will manage a given an event.Queue fed from
+// the event.Manager.
+type Queue interface {
+	Publish(event Event)
+	Next() Event
+	Shutdown()
+	OutChan() <-chan Event
+	Len() int
+}
+
+// NewQueue initializes an event.Queue
+func NewQueue() Queue {
+	queue := &queue{infChan: *newInfiniteChannel()}
+	return queue
+}
+
+// NewSimpleQueue initializes an event.Queue of the given buffer size.
+func NewSimpleQueue(buffer int) Queue {
+	queue := new(simpleQueue)
+	queue.eventChannel = make(chan Event, buffer)
+	return queue
+}
+
+func (sq *simpleQueue) inChan() chan<- Event {
+	return sq.eventChannel
+}
+
+func (sq *simpleQueue) OutChan() <-chan Event {
+	return sq.eventChannel
+}
+
+// Backlog returns the length of the queue backlog
+func (sq *simpleQueue) Len() int {
+	return len(sq.eventChannel)
+}
+
+// Next returns the next available event from the front of the queue
+func (sq *simpleQueue) Next() Event {
+	event := <-sq.eventChannel
+	return event
+}
+
+// Shutdown closes our eventChannel
+func (sq *simpleQueue) Shutdown() {
+	sq.lock.Lock()
+	sq.closed = true
+	close(sq.eventChannel)
+	sq.lock.Unlock()
+}
+
+// Shutdown closes our eventChannel
+func (sq *simpleQueue) Publish(event Event) {
+	sq.lock.Lock()
+	if !sq.closed {
+		sq.inChan() <- event
+	}
+	sq.lock.Unlock()
+}
+
+func (iq *queue) inChan() chan<- Event {
+	return iq.infChan.In()
+}
+
+func (iq *queue) OutChan() <-chan Event {
+	return iq.infChan.Out()
+}
+
+// Out returns the next available event from the front of the queue
+func (iq *queue) Next() Event {
+	event := <-iq.infChan.Out()
+	return event
+}
+
+func (iq *queue) Len() int {
+	return iq.infChan.Len()
+}
+
+// Shutdown closes our eventChannel
+func (iq *queue) Shutdown() {
+	iq.lock.Lock()
+	iq.closed = true
+	iq.infChan.Close()
+	iq.lock.Unlock()
+
+}
+
+func (iq *queue) Publish(event Event) {
+	iq.lock.Lock()
+	if !iq.closed {
+		iq.inChan() <- event
+	}
+	iq.lock.Unlock()
+}

event/eventmanager.go

@@ -1,64 +1,151 @@
 package event
 
 import (
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/log"
+	"math"
+	"math/big"
+	"os"
+	"runtime"
+	"strings"
 	"sync"
 )
 
-// Event is a structure which binds a given set of data to an Type
+// Event is the core struct type passed around between various subsystems. Events consist of a type which can be
+// filtered on, an event ID for tracing and a map of Fields to string values.
 type Event struct {
 	EventType Type
 	EventID   string
-	Data      map[string]string
+	Data      map[Field]string
+}
+
+// GetRandNumber is a helper function which returns a random integer, this is
+// currently mostly used to generate messageids
+func GetRandNumber() *big.Int {
+	num, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32))
+	// If we can't generate random numbers then panicking is probably
+	// the best option.
+	if err != nil {
+		panic(err.Error())
+	}
+	return num
 }
 
 // NewEvent creates a new event object with a unique ID and the given type and data.
-func NewEvent(eventType Type, data map[string]string) Event {
-	return Event{EventType: eventType, EventID: utils.GetRandNumber().String(), Data: data}
+func NewEvent(eventType Type, data map[Field]string) Event {
+	return Event{EventType: eventType, EventID: GetRandNumber().String(), Data: data}
+}
+
+// NewEventList creates a new event object with a unique ID and the given type and data supplied in a list format and composed into a map of Type:string
+func NewEventList(eventType Type, args ...interface{}) Event {
+	data := map[Field]string{}
+	for i := 0; i < len(args); i += 2 {
+		key, kok := args[i].(Field)
+		val, vok := args[i+1].(string)
+		if kok && vok {
+			data[key] = val
+		}
+	}
+	return Event{EventType: eventType, EventID: GetRandNumber().String(), Data: data}
 }
 
 // Manager is an Event Bus which allows subsystems to subscribe to certain EventTypes and publish others.
-type Manager struct {
-	subscribers map[Type][]chan Event
-	events      chan Event
+type manager struct {
+	subscribers map[Type][]Queue
+	events      chan []byte
 	mapMutex    sync.Mutex
 	internal    chan bool
+	closed      bool
+	trace       bool
+}
+
+// Manager is an interface for an event bus
+// FIXME this interface lends itself to race conditions around channels
+type Manager interface {
+	Subscribe(Type, Queue)
+	Publish(Event)
+	PublishLocal(Event)
+	Shutdown()
+}
+
+// NewEventManager returns an initialized EventManager
+func NewEventManager() Manager {
+	em := &manager{}
+	em.initialize()
+	return em
 }
 
 // Initialize sets up the Manager.
-func (em *Manager) Initialize() {
-	em.subscribers = make(map[Type][]chan Event)
-	em.events = make(chan Event)
+func (em *manager) initialize() {
+	em.subscribers = make(map[Type][]Queue)
+	em.events = make(chan []byte)
 	em.internal = make(chan bool)
+	em.closed = false
+
+	_, em.trace = os.LookupEnv("CWTCH_EVENT_SOURCE")
+
 	go em.eventBus()
 }
 
 // Subscribe takes an eventType and an Channel and associates them in the eventBus. All future events of that type
 // will be sent to the eventChannel.
-func (em *Manager) Subscribe(eventType Type, eventChannel chan Event) {
+func (em *manager) Subscribe(eventType Type, queue Queue) {
 	em.mapMutex.Lock()
 	defer em.mapMutex.Unlock()
-	em.subscribers[eventType] = append(em.subscribers[eventType], eventChannel)
+	em.subscribers[eventType] = append(em.subscribers[eventType], queue)
 }
 
 // Publish takes an Event and sends it to the internal eventBus where it is distributed to all Subscribers
-func (em *Manager) Publish(event Event) {
-	if event.EventType != "" {
-		em.events <- event
+func (em *manager) Publish(event Event) {
+	if event.EventType != "" && !em.closed {
+
+		// Debug Events for Tracing, locked behind an environment variable
+		// for now.
+		if em.trace {
+			pc, _, _, _ := runtime.Caller(1)
+			funcName := runtime.FuncForPC(pc).Name()
+			lastSlash := strings.LastIndexByte(funcName, '/')
+			if lastSlash < 0 {
+				lastSlash = 0
+			}
+			lastDot := strings.LastIndexByte(funcName[lastSlash:], '.') + lastSlash
+			event.Data[Source] = fmt.Sprintf("%v.%v", funcName[:lastDot], funcName[lastDot+1:])
+		}
+
+		// Deep Copy the Event...
+		eventJSON, err := json.Marshal(event)
+		if err != nil {
+			log.Errorf("Error serializing event: %v", event)
+		}
+		em.events <- eventJSON
 	}
 }
 
+// Publish an event only locally, not going over an IPC bridge if there is one
+func (em *manager) PublishLocal(event Event) {
+	em.Publish(event)
+}
+
 // eventBus is an internal function that is used to distribute events to all subscribers
-func (em *Manager) eventBus() {
+func (em *manager) eventBus() {
 	for {
-		event := <-em.events
+		eventJSON := <-em.events
 
 		// In the case on an empty event. Teardown the Queue
-		if event.EventType == "" {
+		if len(eventJSON) == 0 {
+			log.Errorf("Received zero length event")
 			break
 		}
 
+		var event Event
+		err := json.Unmarshal(eventJSON, &event)
+
+		if err != nil {
+			log.Errorf("Error on Deep Copy: %v %v", eventJSON, err)
+		}
+
 		// maps aren't thread safe
 		em.mapMutex.Lock()
 		subscribers := em.subscribers[event.EventType]
@@ -66,13 +153,10 @@ func (em *Manager) eventBus() {
 
 		// Send the event to any subscribers to that event type
 		for _, subscriber := range subscribers {
-			select {
-			case subscriber <- event:
-				log.Debugf("Sending %v to %v", event.EventType, subscriber)
-			default:
-				log.Errorf("Failed to send %v to %v. The subsystem might be running too slow!", event.EventType, subscriber)
-			}
-
+			// Deep Copy for Each Subscriber
+			var eventCopy Event
+			json.Unmarshal(eventJSON, &eventCopy)
+			subscriber.Publish(eventCopy)
 		}
 	}
 
@@ -81,8 +165,9 @@ func (em *Manager) eventBus() {
 }
 
 // Shutdown triggers, and waits for, the internal eventBus goroutine to finish
-func (em *Manager) Shutdown() {
-	em.events <- Event{}
+func (em *manager) Shutdown() {
+	em.events <- []byte{}
+	em.closed = true
 	// wait for eventBus to finish
 	<-em.internal
 	close(em.events)

event/eventmanager_test.go

@@ -1,20 +1,21 @@
 package event
 
 import (
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/openprivacy/log"
+	"sync"
 	"testing"
 	"time"
 )
 
 // Most basic Manager Test, Initialize, Subscribe, Publish, Receive
 func TestEventManager(t *testing.T) {
-	eventManager := new(Manager)
-	eventManager.Initialize()
+	eventManager := NewEventManager()
 
 	// We need to make this buffer at least 1, otherwise we will log an error!
 	testChan := make(chan Event, 1)
-	eventManager.Subscribe("TEST", testChan)
-	eventManager.Publish(Event{EventType: "TEST", Data: map[string]string{"Value": "Hello World"}})
+	simpleQueue := &simpleQueue{testChan, sync.Mutex{}, false}
+	eventManager.Subscribe("TEST", simpleQueue)
+	eventManager.Publish(Event{EventType: "TEST", Data: map[Field]string{"Value": "Hello World"}})
 
 	event := <-testChan
 	if event.EventType == "TEST" && event.Data["Value"] == "Hello World" {
@@ -28,35 +29,34 @@ func TestEventManager(t *testing.T) {
 
 // Most basic Manager Test, Initialize, Subscribe, Publish, Receive
 func TestEventManagerOverflow(t *testing.T) {
-	eventManager := new(Manager)
-	eventManager.Initialize()
+	eventManager := NewEventManager()
 
 	// Explicitly setting this to 0 log an error!
 	testChan := make(chan Event)
-	eventManager.Subscribe("TEST", testChan)
+	simpleQueue := &simpleQueue{testChan, sync.Mutex{}, false}
+	eventManager.Subscribe("TEST", simpleQueue)
 	eventManager.Publish(Event{EventType: "TEST"})
 }
 
 func TestEventManagerMultiple(t *testing.T) {
 	log.SetLevel(log.LevelDebug)
-	eventManager := new(Manager)
-	eventManager.Initialize()
+	eventManager := NewEventManager()
 
-	groupEventQueue := NewEventQueue(10)
-	peerEventQueue := NewEventQueue(10)
-	allEventQueue := NewEventQueue(10)
+	groupEventQueue := NewQueue()
+	peerEventQueue := NewQueue()
+	allEventQueue := NewQueue()
 
-	eventManager.Subscribe("PeerEvent", peerEventQueue.EventChannel)
-	eventManager.Subscribe("GroupEvent", groupEventQueue.EventChannel)
-	eventManager.Subscribe("PeerEvent", allEventQueue.EventChannel)
-	eventManager.Subscribe("GroupEvent", allEventQueue.EventChannel)
-	eventManager.Subscribe("ErrorEvent", allEventQueue.EventChannel)
+	eventManager.Subscribe("PeerEvent", peerEventQueue)
+	eventManager.Subscribe("GroupEvent", groupEventQueue)
+	eventManager.Subscribe("PeerEvent", allEventQueue)
+	eventManager.Subscribe("GroupEvent", allEventQueue)
+	eventManager.Subscribe("ErrorEvent", allEventQueue)
 
-	eventManager.Publish(Event{EventType: "PeerEvent", Data: map[string]string{"Value": "Hello World Peer"}})
-	eventManager.Publish(Event{EventType: "GroupEvent", Data: map[string]string{"Value": "Hello World Group"}})
-	eventManager.Publish(Event{EventType: "PeerEvent", Data: map[string]string{"Value": "Hello World Peer"}})
-	eventManager.Publish(Event{EventType: "ErrorEvent", Data: map[string]string{"Value": "Hello World Error"}})
-	eventManager.Publish(Event{EventType: "NobodyIsSubscribedToThisEvent", Data: map[string]string{"Value": "Noone should see this!"}})
+	eventManager.Publish(Event{EventType: "PeerEvent", Data: map[Field]string{"Value": "Hello World Peer"}})
+	eventManager.Publish(Event{EventType: "GroupEvent", Data: map[Field]string{"Value": "Hello World Group"}})
+	eventManager.Publish(Event{EventType: "PeerEvent", Data: map[Field]string{"Value": "Hello World Peer"}})
+	eventManager.Publish(Event{EventType: "ErrorEvent", Data: map[Field]string{"Value": "Hello World Error"}})
+	eventManager.Publish(Event{EventType: "NobodyIsSubscribedToThisEvent", Data: map[Field]string{"Value": "Noone should see this!"}})
 
 	assertLength := func(len int, expected int, label string) {
 		if len != expected {
@@ -66,9 +66,9 @@ func TestEventManagerMultiple(t *testing.T) {
 
 	time.Sleep(time.Second)
 
-	assertLength(groupEventQueue.Backlog(), 1, "Group Event Queue Length")
-	assertLength(peerEventQueue.Backlog(), 2, "Peer Event Queue Length")
-	assertLength(allEventQueue.Backlog(), 4, "All Event Queue Length")
+	assertLength(groupEventQueue.Len(), 1, "Group Event Queue Length")
+	assertLength(peerEventQueue.Len(), 2, "Peer Event Queue Length")
+	assertLength(allEventQueue.Len(), 4, "All Event Queue Length")
 
 	checkEvent := func(eventType Type, expected Type, label string) {
 		if eventType != expected {

event/eventmanageripc.go

@@ -0,0 +1,38 @@
+package event
+
+type ipcManager struct {
+	manager Manager
+
+	onion     string
+	ipcBridge IPCBridge
+}
+
+// NewIPCEventManager returns an EvenetManager that also pipes events over and supplied IPCBridge
+func NewIPCEventManager(bridge IPCBridge, onion string) Manager {
+	em := &ipcManager{onion: onion, ipcBridge: bridge, manager: NewEventManager()}
+	return em
+}
+
+// IPCEventManagerFrom returns an IPCEventManger from the supplied manager and IPCBridge
+func IPCEventManagerFrom(bridge IPCBridge, onion string, manager Manager) Manager {
+	em := &ipcManager{onion: onion, ipcBridge: bridge, manager: manager}
+	return em
+}
+
+func (ipcm *ipcManager) Publish(ev Event) {
+	ipcm.manager.Publish(ev)
+	message := &IPCMessage{Dest: ipcm.onion, Message: ev}
+	ipcm.ipcBridge.Write(message)
+}
+
+func (ipcm *ipcManager) PublishLocal(ev Event) {
+	ipcm.manager.Publish(ev)
+}
+
+func (ipcm *ipcManager) Subscribe(eventType Type, queue Queue) {
+	ipcm.manager.Subscribe(eventType, queue)
+}
+
+func (ipcm *ipcManager) Shutdown() {
+	ipcm.manager.Shutdown()
+}

event/infinitechannel.go

@@ -0,0 +1,72 @@
+package event
+
+/*
+This package is taken from https://github.com/eapache/channels
+as per their suggestion we are not importing the entire package and instead cherry picking and adapting what is needed
+
+It is covered by the MIT License https://github.com/eapache/channels/blob/master/LICENSE
+*/
+
+// infiniteChannel implements the Channel interface with an infinite buffer between the input and the output.
+type infiniteChannel struct {
+	input, output chan Event
+	length        chan int
+	buffer        *infiniteQueue
+}
+
+func newInfiniteChannel() *infiniteChannel {
+	ch := &infiniteChannel{
+		input:  make(chan Event),
+		output: make(chan Event),
+		length: make(chan int),
+		buffer: newInfinitQueue(),
+	}
+	go ch.infiniteBuffer()
+	return ch
+}
+func (ch *infiniteChannel) In() chan<- Event {
+	return ch.input
+}
+
+func (ch *infiniteChannel) Out() <-chan Event {
+	return ch.output
+}
+
+func (ch *infiniteChannel) Len() int {
+	return <-ch.length
+}
+
+func (ch *infiniteChannel) Close() {
+	close(ch.input)
+}
+
+func (ch *infiniteChannel) infiniteBuffer() {
+	var input, output chan Event
+	var next Event
+	input = ch.input
+
+	for input != nil || output != nil {
+		select {
+		case elem, open := <-input:
+			if open {
+				ch.buffer.Add(elem)
+			} else {
+				input = nil
+			}
+		case output <- next:
+			ch.buffer.Remove()
+		case ch.length <- ch.buffer.Length():
+		}
+
+		if ch.buffer.Length() > 0 {
+			output = ch.output
+			next = ch.buffer.Peek()
+		} else {
+			output = nil
+			//next = nil
+		}
+	}
+
+	close(ch.output)
+	close(ch.length)
+}

event/infinitequeue.go

@@ -0,0 +1,108 @@
+package event
+
+/*
+This package is taken from https://github.com/eapache/channels
+as per their suggestion we are not importing the entire package and instead cherry picking and adapting what is needed
+
+It is covered by the MIT License https://github.com/eapache/channels/blob/master/LICENSE
+*/
+/*
+Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki.
+Using this instead of other, simpler, queue implementations (slice+append or linked list) provides
+substantial memory and time benefits, and fewer GC pauses.
+The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe.
+*/
+
+// minQueueLen is smallest capacity that queue may have.
+// Must be power of 2 for bitwise modulus: x % n == x & (n - 1).
+const minQueueLen = 16
+
+// Queue represents a single instance of the queue data structure.
+type infiniteQueue struct {
+	buf               []Event
+	head, tail, count int
+}
+
+// New constructs and returns a new Queue.
+func newInfinitQueue() *infiniteQueue {
+	return &infiniteQueue{
+		buf: make([]Event, minQueueLen),
+	}
+}
+
+// Length returns the number of elements currently stored in the queue.
+func (q *infiniteQueue) Length() int {
+	return q.count
+}
+
+// resizes the queue to fit exactly twice its current contents
+// this can result in shrinking if the queue is less than half-full
+func (q *infiniteQueue) resize() {
+	newBuf := make([]Event, q.count<<1)
+
+	if q.tail > q.head {
+		copy(newBuf, q.buf[q.head:q.tail])
+	} else {
+		n := copy(newBuf, q.buf[q.head:])
+		copy(newBuf[n:], q.buf[:q.tail])
+	}
+
+	q.head = 0
+	q.tail = q.count
+	q.buf = newBuf
+}
+
+// Add puts an element on the end of the queue.
+func (q *infiniteQueue) Add(elem Event) {
+	if q.count == len(q.buf) {
+		q.resize()
+	}
+
+	q.buf[q.tail] = elem
+	// bitwise modulus
+	q.tail = (q.tail + 1) & (len(q.buf) - 1)
+	q.count++
+}
+
+// Peek returns the element at the head of the queue. This call panics
+// if the queue is empty.
+func (q *infiniteQueue) Peek() Event {
+	if q.count <= 0 {
+		panic("queue: Peek() called on empty queue")
+	}
+	return q.buf[q.head]
+}
+
+// Get returns the element at index i in the queue. If the index is
+// invalid, the call will panic. This method accepts both positive and
+// negative index values. Index 0 refers to the first element, and
+// index -1 refers to the last.
+func (q *infiniteQueue) Get(i int) Event {
+	// If indexing backwards, convert to positive index.
+	if i < 0 {
+		i += q.count
+	}
+	if i < 0 || i >= q.count {
+		panic("queue: Get() called with index out of range")
+	}
+	// bitwise modulus
+	return q.buf[(q.head+i)&(len(q.buf)-1)]
+}
+
+// Remove removes and returns the element from the front of the queue. If the
+// queue is empty, the call will panic.
+func (q *infiniteQueue) Remove() Event {
+	if q.count <= 0 {
+		panic("queue: Remove() called on empty queue")
+	}
+	ret := q.buf[q.head]
+	//q.buf[q.head] = nil
+	// bitwise modulus
+	q.head = (q.head + 1) & (len(q.buf) - 1)
+	q.count--
+	// Resize down if buffer 1/4 full.
+	if len(q.buf) > minQueueLen && (q.count<<2) == len(q.buf) {
+		q.resize()
+	}
+	return ret
+}

event/ipc.go

@@ -0,0 +1,14 @@
+package event
+
+// IPCMessage is a wrapper for a regular eventMessage with a destination (onion|AppDest) so the other side of the bridge can route appropriately
+type IPCMessage struct {
+	Dest    string
+	Message Event
+}
+
+// IPCBridge is an interface to a IPC construct used to communicate IPCMessages
+type IPCBridge interface {
+	Read() (*IPCMessage, bool)
+	Write(message *IPCMessage)
+	Shutdown()
+}

go.mod

@@ -0,0 +1,17 @@
+module cwtch.im/cwtch
+
+go 1.14
+
+require (
+	git.openprivacy.ca/cwtch.im/tapir v0.4.3
+	git.openprivacy.ca/openprivacy/connectivity v1.4.4
+	git.openprivacy.ca/openprivacy/log v1.0.2
+	github.com/gtank/ristretto255 v0.1.2
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/struCoder/pidusage v0.1.3
+	golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
+	golang.org/x/tools v0.1.2 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+)

go.sum

@@ -0,0 +1,110 @@
+git.openprivacy.ca/cwtch.im/tapir v0.3.1 h1:+d1dHyPvZ8JmdfFe/oXWJPardzflRIhcdILtkeArkW8=
+git.openprivacy.ca/cwtch.im/tapir v0.3.1/go.mod h1:q6RMI/TQvRN8SCtRY3GryOawMcB0uG6NjP6M77oSMx8=
+git.openprivacy.ca/cwtch.im/tapir v0.3.2 h1:thLWqqY1LkirWFcy9Tg6NgWeYbvo9xBm+s2XVnCIvpY=
+git.openprivacy.ca/cwtch.im/tapir v0.3.2/go.mod h1:q6RMI/TQvRN8SCtRY3GryOawMcB0uG6NjP6M77oSMx8=
+git.openprivacy.ca/cwtch.im/tapir v0.3.3 h1:Q7F8JijgOMMYSy3IdZl7+r6qkWckEWV1+EY7q6MAkVs=
+git.openprivacy.ca/cwtch.im/tapir v0.3.3/go.mod h1:ZMg9Jzh0n3Os2aSF4z+bx/n8WBCJBN7KCQESXperYts=
+git.openprivacy.ca/cwtch.im/tapir v0.3.4 h1:g7yZkfz/vWr/t2tFXa/t0Ebr/w665uIKpxpCZ3lIPCo=
+git.openprivacy.ca/cwtch.im/tapir v0.3.4/go.mod h1:+Niy2AHhQC351ZTtfhC0uLjViCICyOxCJZsIlGKKNAU=
+git.openprivacy.ca/cwtch.im/tapir v0.3.5 h1:AlqAhluY4ivznGoHh37Khyxy0u9IbtYskP93wgtmYx8=
+git.openprivacy.ca/cwtch.im/tapir v0.3.5/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
+git.openprivacy.ca/cwtch.im/tapir v0.4.0 h1:clG8uORt0NKEhT4P+Dpw1pzyUuYzYBMevGqn2pciKk8=
+git.openprivacy.ca/cwtch.im/tapir v0.4.0/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
+git.openprivacy.ca/cwtch.im/tapir v0.4.1 h1:9LMpQX41IzecNNlRc1FZKXHg6wlFss679tFsa3vzb3Y=
+git.openprivacy.ca/cwtch.im/tapir v0.4.1/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
+git.openprivacy.ca/cwtch.im/tapir v0.4.2 h1:bxMWZnVJXX4dqqOFS7ELW4iFkVL4GS8wiRkjRv5rJe8=
+git.openprivacy.ca/cwtch.im/tapir v0.4.2/go.mod h1:eH6dZxXrhW0C4KZX18ksUa6XJCrEvtg8cJJ/Fy6gv+E=
+git.openprivacy.ca/cwtch.im/tapir v0.4.3 h1:sctSfUXHDIqaHfJPDl+5lHtmoEJolQiHTcHZGAe5Qc4=
+git.openprivacy.ca/cwtch.im/tapir v0.4.3/go.mod h1:10qEaib5x021zgyZ/97JKWsEpedH5+Vfy2CvB2V+08E=
+git.openprivacy.ca/openprivacy/bine v0.0.4 h1:CO7EkGyz+jegZ4ap8g5NWRuDHA/56KKvGySR6OBPW+c=
+git.openprivacy.ca/openprivacy/bine v0.0.4/go.mod h1:13ZqhKyqakDsN/ZkQkIGNULsmLyqtXc46XBcnuXm/mU=
+git.openprivacy.ca/openprivacy/connectivity v1.4.0 h1:c7AANUCrlA4hIqXxIGDOWMtSe8CpDleD1877PShScbM=
+git.openprivacy.ca/openprivacy/connectivity v1.4.0/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
+git.openprivacy.ca/openprivacy/connectivity v1.4.1 h1:zoM+j7PFj8mQeUCNiDNMe7Uq9dhcJDOhaZcSANfeDL4=
+git.openprivacy.ca/openprivacy/connectivity v1.4.1/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
+git.openprivacy.ca/openprivacy/connectivity v1.4.2 h1:rQFIjWunLlRmXL5Efsv+7+1cA70T6Uza6RCy2PRm9zc=
+git.openprivacy.ca/openprivacy/connectivity v1.4.2/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
+git.openprivacy.ca/openprivacy/connectivity v1.4.3 h1:i2Ad/U9FlL9dKr2bhRck7lJ8NoWyGtoEfUwoCyMT0fU=
+git.openprivacy.ca/openprivacy/connectivity v1.4.3/go.mod h1:bR0Myx9nm2YzWtsThRelkNMV4Pp7sPDa123O1qsAbVo=
+git.openprivacy.ca/openprivacy/connectivity v1.4.4 h1:11M3akVCyy/luuhMpZTM1r9Jayl7IHD944Bxsn2FDpU=
+git.openprivacy.ca/openprivacy/connectivity v1.4.4/go.mod h1:JVRCIdL+lAG6ohBFWiKeC/MN42nnC0sfFszR9XG6vPQ=
+git.openprivacy.ca/openprivacy/log v1.0.1/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+git.openprivacy.ca/openprivacy/log v1.0.2 h1:HLP4wsw4ljczFAelYnbObIs821z+jgMPCe8uODPnGQM=
+git.openprivacy.ca/openprivacy/log v1.0.2/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is=
+github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
+github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc=
+github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0=
+github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/struCoder/pidusage v0.1.3 h1:pZcSa6asBE38TJtW0Nui6GeCjLTpaT/jAnNP7dUTLSQ=
+github.com/struCoder/pidusage v0.1.3/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI=
+github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee h1:4yd7jl+vXjalO5ztz6Vc1VADv+S/80LGJmyl1ROJ2AI=
+golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

model/attr/scope.go

@@ -0,0 +1,60 @@
+package attr
+
+import (
+	"strings"
+)
+
+/*
+Scope model for peer attributes and requests
+
+A local peer "Alice" has a PublicScope that is queryable by getVal requests.
+By default, for now, other scopes are private, of which we here define SettingsScope
+
+Alice's peer structs of remote peers such as "Bob" keep the queried
+PublicScope values in the PeerScope, which can be overridden by the same named
+values stored in the LocalScope.
+
+*/
+
+// scopes for attributes
+const (
+	// on a peer, local and peer supplied data
+	LocalScope = "local"
+	PeerScope  = "peer"
+
+	// on a local profile, public data and private settings
+	PublicScope   = "public"
+	SettingsScope = "settings"
+)
+
+// Separator for scope and the rest of path
+const Separator = "."
+
+// GetPublicScope takes a path and attaches the pubic scope to it
+func GetPublicScope(path string) string {
+	return PublicScope + Separator + path
+}
+
+// GetSettingsScope takes a path and attaches the settings scope to it
+func GetSettingsScope(path string) string {
+	return SettingsScope + Separator + path
+}
+
+// GetLocalScope takes a path and attaches the local scope to it
+func GetLocalScope(path string) string {
+	return LocalScope + Separator + path
+}
+
+// GetPeerScope takes a path and attaches the peer scope to it
+func GetPeerScope(path string) string {
+	return PeerScope + Separator + path
+}
+
+// GetScopePath take a full path and returns the scope and the scope-less path
+func GetScopePath(fullPath string) (string, string) {
+	parts := strings.SplitN(fullPath, Separator, 1)
+	if len(parts) != 2 {
+		return "", ""
+	}
+	return parts[0], parts[1]
+}

model/errors.go

@@ -0,0 +1,15 @@
+package model
+
+// Error models some common errors that need to be handled by applications that use Cwtch
+type Error string
+
+// Error is the error interface
+func (e Error) Error() string {
+	return string(e)
+}
+
+// Error definitions
+const (
+	InvalidEd25519PublicKey    = Error("InvalidEd25519PublicKey")
+	InconsistentKeyBundleError = Error("InconsistentKeyBundleError")
+)

model/group.go

@@ -1,42 +1,59 @@
 package model
 
 import (
+	"crypto/ed25519"
 	"crypto/rand"
-	"cwtch.im/cwtch/protocol"
+	"crypto/sha512"
+	"cwtch.im/cwtch/model/attr"
+	"cwtch.im/cwtch/protocol/groups"
+	"encoding/base32"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"github.com/golang/protobuf/proto"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
 	"golang.org/x/crypto/nacl/secretbox"
+	"golang.org/x/crypto/pbkdf2"
 	"io"
+	"strings"
 	"sync"
 	"time"
 )
 
+// CurrentGroupVersion is used to set the version of newly created groups and make sure group structs stored are correct and up to date
+const CurrentGroupVersion = 3
+
+// GroupInvitePrefix identifies a particular string as being a serialized group invite.
+const GroupInvitePrefix = "torv3"
+
 // Group defines and encapsulates Cwtch's conception of group chat. Which are sessions
-// tied to a server under a given group key. Each group has a set of messages.
+// tied to a server under a given group key. Each group has a set of Messages.
 type Group struct {
-	GroupID        string
-	SignedGroupID  []byte
-	GroupKey       [32]byte
-	GroupServer    string
-	Timeline       Timeline
-	Accepted       bool
-	Owner          string
-	IsCompromised  bool
-	InitialMessage []byte
-	Attributes     map[string]string
-	lock           sync.Mutex
-	NewMessage     chan Message `json:"-"`
+	// GroupID is now derived from the GroupKey and the GroupServer
+	GroupID                string
+	GroupKey               [32]byte
+	GroupServer            string
+	Timeline               Timeline `json:"-"`
+	Accepted               bool
+	IsCompromised          bool
+	Attributes             map[string]string
+	lock                   sync.Mutex
+	LocalID                string
+	State                  string `json:"-"`
+	UnacknowledgedMessages []Message
+	Version                int
 }
 
 // NewGroup initializes a new group associated with a given CwtchServer
 func NewGroup(server string) (*Group, error) {
 	group := new(Group)
-
-	if utils.IsValidHostname(server) == false {
-		return nil, errors.New("Server is not a valid v3 onion")
+	group.Version = CurrentGroupVersion
+	group.LocalID = GenerateRandomID()
+	group.Accepted = true // we are starting a group, so we assume we want to connect to it...
+	if !tor.IsValidHostname(server) {
+		return nil, errors.New("server is not a valid v3 onion")
 	}
 
 	group.GroupServer = server
@@ -54,105 +71,153 @@ func NewGroup(server string) (*Group, error) {
 		return nil, err
 	}
 	copy(group.GroupKey[:], groupKey[:])
-	group.Owner = "self"
+
+	// Derive Group ID from the group key and the server public key. This binds the group to a particular server
+	// and key.
+	group.GroupID = deriveGroupID(groupKey[:], server)
+
 	group.Attributes = make(map[string]string)
+	// By default we set the "name" of the group to a random string, we can override this later, but to simplify the
+	// codes around invite, we assume that this is always set.
+	group.Attributes[attr.GetLocalScope("name")] = group.GroupID
 	return group, nil
 }
 
-// SignGroup adds a signature to the group.
-func (g *Group) SignGroup(signature []byte) {
-	g.SignedGroupID = signature
-	copy(g.Timeline.SignedGroupID[:], g.SignedGroupID)
+// CheckGroup returns true only if the ID of the group is cryptographically valid.
+func (g *Group) CheckGroup() bool {
+	return g.GroupID == deriveGroupID(g.GroupKey[:], g.GroupServer)
 }
 
-// Compromised should be called if we detect a a groupkey leak.
+// deriveGroupID hashes together the key and the hostname to create a bound identifier that can later
+// be referenced and checked by profiles when they receive invites and messages.
+func deriveGroupID(groupKey []byte, serverHostname string) string {
+	data, _ := base32.StdEncoding.DecodeString(strings.ToUpper(serverHostname))
+	pubkey := data[0:ed25519.PublicKeySize]
+	return hex.EncodeToString(pbkdf2.Key(groupKey, pubkey, 4096, 16, sha512.New))
+}
+
+// Compromised should be called if we detect a groupkey leak
 func (g *Group) Compromised() {
 	g.IsCompromised = true
 }
 
-// GetInitialMessage returns the first message of the group, if one was sent with the invite.
-func (g *Group) GetInitialMessage() []byte {
-	g.lock.Lock()
-	defer g.lock.Unlock()
-	return g.InitialMessage
+// Invite generates a invitation that can be sent to a cwtch peer
+func (g *Group) Invite() (string, error) {
+
+	gci := &groups.GroupInvite{
+		GroupID:    g.GroupID,
+		GroupName:  g.Attributes[attr.GetLocalScope("name")],
+		SharedKey:  g.GroupKey[:],
+		ServerHost: g.GroupServer,
+	}
+
+	invite, err := json.Marshal(gci)
+	serializedInvite := fmt.Sprintf("%v%v", GroupInvitePrefix, base64.StdEncoding.EncodeToString(invite))
+	return serializedInvite, err
 }
 
-// Invite generates a invitation that can be sent to a cwtch peer
-func (g *Group) Invite(initialMessage []byte) ([]byte, error) {
-
-	if g.SignedGroupID == nil {
-		return nil, errors.New("group isn't signed")
+// AddSentMessage takes a DecryptedGroupMessage and adds it to the Groups Timeline
+func (g *Group) AddSentMessage(message *groups.DecryptedGroupMessage, sig []byte) Message {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+	timelineMessage := Message{
+		Message:            message.Text,
+		Timestamp:          time.Unix(int64(message.Timestamp), 0),
+		Received:           time.Unix(0, 0),
+		Signature:          sig,
+		PeerID:             message.Onion,
+		PreviousMessageSig: message.PreviousMessageSig,
+		ReceivedByServer:   false,
 	}
+	g.UnacknowledgedMessages = append(g.UnacknowledgedMessages, timelineMessage)
+	return timelineMessage
+}
 
-	g.InitialMessage = initialMessage[:]
+// ErrorSentMessage removes a sent message from the unacknowledged list and sets its error flag if found, otherwise returns false
+func (g *Group) ErrorSentMessage(sig []byte, error string) bool {
+	g.lock.Lock()
+	defer g.lock.Unlock()
+	var message *Message
 
-	gci := &protocol.GroupChatInvite{
-		GroupName:      g.GroupID,
-		GroupSharedKey: g.GroupKey[:],
-		ServerHost:     g.GroupServer,
-		SignedGroupId:  g.SignedGroupID[:],
-		InitialMessage: initialMessage[:],
+	// Delete the message from the unack'd buffer if it exists
+	for i, unAckedMessage := range g.UnacknowledgedMessages {
+		if compareSignatures(unAckedMessage.Signature, sig) {
+			message = &unAckedMessage
+			g.UnacknowledgedMessages = append(g.UnacknowledgedMessages[:i], g.UnacknowledgedMessages[i+1:]...)
+
+			message.Error = error
+			g.Timeline.Insert(message)
+			return true
+		}
 	}
-
-	cp := &protocol.CwtchPeerPacket{
-		GroupChatInvite: gci,
-	}
-	invite, err := proto.Marshal(cp)
-	return invite, err
+	return false
 }
 
 // AddMessage takes a DecryptedGroupMessage and adds it to the Groups Timeline
-func (g *Group) AddMessage(message *protocol.DecryptedGroupMessage, sig []byte) *Message {
+func (g *Group) AddMessage(message *groups.DecryptedGroupMessage, sig []byte) (*Message, bool) {
+
 	g.lock.Lock()
+	defer g.lock.Unlock()
+
+	// Delete the message from the unack'd buffer if it exists
+	for i, unAckedMessage := range g.UnacknowledgedMessages {
+		if compareSignatures(unAckedMessage.Signature, sig) {
+			g.UnacknowledgedMessages = append(g.UnacknowledgedMessages[:i], g.UnacknowledgedMessages[i+1:]...)
+			break
+		}
+	}
+
 	timelineMessage := &Message{
-		Message:            message.GetText(),
-		Timestamp:          time.Unix(int64(message.GetTimestamp()), 0),
+		Message:            message.Text,
+		Timestamp:          time.Unix(int64(message.Timestamp), 0),
 		Received:           time.Now(),
 		Signature:          sig,
-		PeerID:             message.GetOnion(),
-		PreviousMessageSig: message.GetPreviousMessageSig(),
+		PeerID:             message.Onion,
+		PreviousMessageSig: message.PreviousMessageSig,
+		ReceivedByServer:   true,
+		Error:              "",
+		Acknowledged:       true,
 	}
 	seen := g.Timeline.Insert(timelineMessage)
-	g.lock.Unlock()
 
-	// Send a new Message notification if we have an app that is listening.
-	if g.NewMessage != nil && !seen {
-		g.NewMessage <- *timelineMessage
-	}
-	return timelineMessage
+	return timelineMessage, seen
 }
 
 // GetTimeline provides a safe copy of the timeline
 func (g *Group) GetTimeline() (timeline []Message) {
 	g.lock.Lock()
 	defer g.lock.Unlock()
-	return g.Timeline.GetMessages()
+	return append(g.Timeline.GetMessages(), g.UnacknowledgedMessages...)
 }
 
 //EncryptMessage takes a message and encrypts the message under the group key.
-func (g *Group) EncryptMessage(message *protocol.DecryptedGroupMessage) ([]byte, error) {
+func (g *Group) EncryptMessage(message *groups.DecryptedGroupMessage) ([]byte, error) {
 	var nonce [24]byte
 	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
 		log.Errorf("Cannot read from random: %v\n", err)
 		return nil, err
 	}
-	wire, err := proto.Marshal(message)
-	utils.CheckError(err)
+	wire, err := json.Marshal(message)
+	if err != nil {
+		return nil, err
+	}
 	encrypted := secretbox.Seal(nonce[:], []byte(wire), &nonce, &g.GroupKey)
 	return encrypted, nil
 }
 
 // DecryptMessage takes a ciphertext and returns true and the decrypted message if the
 // cipher text can be successfully decrypted,else false.
-func (g *Group) DecryptMessage(ciphertext []byte) (bool, *protocol.DecryptedGroupMessage) {
-	var decryptNonce [24]byte
-	copy(decryptNonce[:], ciphertext[:24])
-	decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &g.GroupKey)
-	if ok {
-		dm := &protocol.DecryptedGroupMessage{}
-		err := proto.Unmarshal(decrypted, dm)
-		if err == nil {
-			return true, dm
+func (g *Group) DecryptMessage(ciphertext []byte) (bool, *groups.DecryptedGroupMessage) {
+	if len(ciphertext) > 24 {
+		var decryptNonce [24]byte
+		copy(decryptNonce[:], ciphertext[:24])
+		decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &g.GroupKey)
+		if ok {
+			dm := &groups.DecryptedGroupMessage{}
+			err := json.Unmarshal(decrypted, dm)
+			if err == nil {
+				return true, dm
+			}
 		}
 	}
 	return false, nil
@@ -172,3 +237,41 @@ func (g *Group) GetAttribute(name string) (value string, exists bool) {
 	value, exists = g.Attributes[name]
 	return
 }
+
+// ValidateInvite takes in a serialized invite and returns the invite structure if it is cryptographically valid
+// and an error if it is not
+func ValidateInvite(invite string) (*groups.GroupInvite, error) {
+	// We prefix invites for groups with torv3
+	if strings.HasPrefix(invite, GroupInvitePrefix) {
+		data, err := base64.StdEncoding.DecodeString(invite[len(GroupInvitePrefix):])
+		if err == nil {
+			// First attempt to unmarshal the json...
+			var gci groups.GroupInvite
+			err := json.Unmarshal(data, &gci)
+			if err == nil {
+
+				// Validate the Invite by first checking that the server is a valid v3 onion
+				if !tor.IsValidHostname(gci.ServerHost) {
+					return nil, errors.New("server is not a valid v3 onion")
+				}
+
+				// Validate the length of the shared key...
+				if len(gci.SharedKey) != 32 {
+					return nil, errors.New("key length is not 32 bytes")
+				}
+
+				// Derive the servers public key (we can ignore the error checking here because it's already been
+				// done by IsValidHostname, and check that we derive the same groupID...
+				derivedGroupID := deriveGroupID(gci.SharedKey, gci.ServerHost)
+				if derivedGroupID != gci.GroupID {
+					return nil, errors.New("group id is invalid")
+				}
+
+				// Replace the original with the derived, this should be a no-op at this point but defense in depth...
+				gci.GroupID = derivedGroupID
+				return &gci, nil
+			}
+		}
+	}
+	return nil, errors.New("invite has invalid structure")
+}

model/group_test.go

@@ -1,25 +1,44 @@
 package model
 
 import (
-	"cwtch.im/cwtch/protocol"
-	"github.com/golang/protobuf/proto"
+	"crypto/sha256"
+	"cwtch.im/cwtch/protocol/groups"
+	"strings"
+	"sync"
 	"testing"
 	"time"
 )
 
 func TestGroup(t *testing.T) {
 	g, _ := NewGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	dgm := &protocol.DecryptedGroupMessage{
-		Onion:              proto.String("onion"),
-		Text:               proto.String("Hello World!"),
-		Timestamp:          proto.Int32(int32(time.Now().Unix())),
-		SignedGroupId:      []byte{},
+	dgm := &groups.DecryptedGroupMessage{
+		Onion:              "onion",
+		Text:               "Hello World!",
+		Timestamp:          uint64(time.Now().Unix()),
+		SignedGroupID:      []byte{},
 		PreviousMessageSig: []byte{},
 		Padding:            []byte{},
 	}
+
+	invite, err := g.Invite()
+
+	if err != nil {
+		t.Fatalf("error creating group invite: %v", err)
+	}
+
+	validatedInvite, err := ValidateInvite(invite)
+
+	if err != nil {
+		t.Fatalf("error validating group invite: %v", err)
+	}
+
+	if validatedInvite.GroupID != g.GroupID {
+		t.Fatalf("after validate group invite id should be identical to original: %v", err)
+	}
+
 	encMessage, _ := g.EncryptMessage(dgm)
 	ok, message := g.DecryptMessage(encMessage)
-	if !ok || message.GetText() != "Hello World!" {
+	if !ok || message.Text != "Hello World!" {
 		t.Errorf("group encryption was invalid, or returned wrong message decrypted:%v message:%v", ok, message)
 		return
 	}
@@ -37,3 +56,60 @@ func TestGroupErr(t *testing.T) {
 		t.Errorf("Group Setup Should Have Failed")
 	}
 }
+
+// Test various group invite validation failures...
+func TestGroupValidation(t *testing.T) {
+
+	group := &Group{
+		GroupID:                "",
+		GroupKey:               [32]byte{},
+		GroupServer:            "",
+		Timeline:               Timeline{},
+		Accepted:               false,
+		IsCompromised:          false,
+		Attributes:             nil,
+		lock:                   sync.Mutex{},
+		LocalID:                "",
+		State:                  "",
+		UnacknowledgedMessages: nil,
+		Version:                0,
+	}
+
+	invite, _ := group.Invite()
+	_, err := ValidateInvite(invite)
+
+	if err == nil {
+		t.Fatalf("Group with empty group id should have been an error")
+	}
+	t.Logf("Error: %v", err)
+
+	// Generate a valid group but replace the group server...
+	group, _ = NewGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+	group.GroupServer = "tcnkoch4nyr3cldkemejtkpqok342rbql6iclnjjs3ndgnjgufzyxvqd"
+	invite, _ = group.Invite()
+	_, err = ValidateInvite(invite)
+
+	if err == nil {
+		t.Fatalf("Group with empty group id should have been an error")
+	}
+	t.Logf("Error: %v", err)
+
+	// Generate a valid group but replace the group key...
+	group, _ = NewGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+	group.GroupKey = sha256.Sum256([]byte{})
+	invite, _ = group.Invite()
+	_, err = ValidateInvite(invite)
+
+	if err == nil {
+		t.Fatalf("Group with different group key should have errored")
+	}
+	t.Logf("Error: %v", err)
+
+	// mangle the invite
+	_, err = ValidateInvite(strings.ReplaceAll(invite, GroupInvitePrefix, ""))
+	if err == nil {
+		t.Fatalf("Group with different group key should have errored")
+	}
+	t.Logf("Error: %v", err)
+
+}

model/keyBundle.go

@@ -0,0 +1,103 @@
+package model
+
+import (
+	"crypto/ed25519"
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"strings"
+)
+
+// KeyType provides a wrapper for a generic public key type identifier (could be an onion address, a zcash address etc.)
+type KeyType string
+
+const (
+
+	// BundleType - the attribute under which the signed server bundle is stored...
+	BundleType = KeyType("server_key_bundle")
+
+	// KeyTypeServerOnion - a cwtch address
+	KeyTypeServerOnion = KeyType("bulletin_board_onion") // bulletin board
+
+	// KeyTypeTokenOnion - a cwtch peer with a PoW based token protocol
+	KeyTypeTokenOnion = KeyType("token_service_onion")
+
+	//KeyTypePrivacyPass - a privacy pass based token server
+	KeyTypePrivacyPass = KeyType("privacy_pass_public_key")
+)
+
+// Key provides a wrapper for a generic public key identifier (could be an onion address, a zcash address etc.)
+type Key string
+
+// KeyBundle manages a collection of related keys for various different services.
+type KeyBundle struct {
+	Keys      map[KeyType]Key
+	Signature []byte
+}
+
+// NewKeyBundle creates a new KeyBundle initialized with no keys.
+func NewKeyBundle() *KeyBundle {
+	keyBundle := new(KeyBundle)
+	keyBundle.Keys = make(map[KeyType]Key)
+	return keyBundle
+}
+
+// HasKeyType returns true if the bundle has a public key of a given type.
+func (kb *KeyBundle) HasKeyType(keytype KeyType) bool {
+	_, exists := kb.Keys[keytype]
+	return exists
+}
+
+// GetKey retrieves a key with a given type from the bundle
+func (kb *KeyBundle) GetKey(keytype KeyType) (Key, error) {
+	key, exists := kb.Keys[keytype]
+	if exists {
+		return key, nil
+	}
+	return "", errors.New("no such key")
+}
+
+// Serialize produces a json encoded byte array.
+func (kb KeyBundle) Serialize() []byte {
+	// json.Marshal sorts map keys
+	bundle, _ := json.Marshal(kb)
+	return bundle
+}
+
+// Sign allows a server to authenticate a key bundle by signing it (this uses the tapir identity interface)
+func (kb *KeyBundle) Sign(identity primitives.Identity) {
+	kb.Signature = identity.Sign(kb.Serialize())
+}
+
+// DeserializeAndVerify takes in a json formatted bundle and only returns a valid key bundle
+// if it has been signed by the server.
+func DeserializeAndVerify(bundle []byte) (*KeyBundle, error) {
+	keyBundle := new(KeyBundle)
+	err := json.Unmarshal(bundle, &keyBundle)
+	if err == nil {
+		signature := keyBundle.Signature
+		keyBundle.Signature = nil
+		serverKey, _ := keyBundle.GetKey(KeyTypeServerOnion)
+
+		// We have to do convert the encoded key to a format that can be used to verify the signature
+		var decodedPub []byte
+		decodedPub, err = base32.StdEncoding.DecodeString(strings.ToUpper(string(serverKey)))
+		if err == nil && len(decodedPub) == 35 {
+			if ed25519.Verify(decodedPub[:32], keyBundle.Serialize(), signature) { // == true
+				return keyBundle, nil
+			}
+		}
+		err = InvalidEd25519PublicKey
+	}
+	return nil, err
+}
+
+// AttributeBundle returns a map that can be used as part of a peer attribute bundle
+func (kb *KeyBundle) AttributeBundle() map[string]string {
+	ab := make(map[string]string)
+	for k, v := range kb.Keys {
+		ab[string(k)] = string(v)
+	}
+	return ab
+}

model/keyBundle_test.go

@@ -0,0 +1,64 @@
+package model
+
+import (
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"testing"
+)
+
+func TestDeserializeAndVerify(t *testing.T) {
+	server, _ := primitives.InitializeEphemeralIdentity()
+
+	serverKeyBundle := NewKeyBundle()
+
+	serverKeyBundle.Keys[KeyTypeServerOnion] = Key(server.Hostname())
+	serverKeyBundle.Keys[KeyTypePrivacyPass] = Key("random 1")
+	serverKeyBundle.Keys[KeyTypeTokenOnion] = Key("random 2")
+	serverKeyBundle.Sign(server)
+
+	//eyeball keys are sorted
+	t.Logf("%s", serverKeyBundle.Serialize())
+	serialize := serverKeyBundle.Serialize()
+
+	newKeyBundle, err := DeserializeAndVerify(serialize)
+	if err != nil {
+		t.Fatalf("Key Bundle did not Deserialize %v", err)
+	}
+
+	if newKeyBundle.Keys[KeyTypeServerOnion] != Key(server.Hostname()) {
+		t.Fatalf("Key Bundle did not Serialize Correctly Actual: %v Expected: %v", newKeyBundle, serverKeyBundle)
+	}
+}
+
+func TestDeserializeAndVerifyMaliciousSignShouldFail(t *testing.T) {
+	server, _ := primitives.InitializeEphemeralIdentity()
+	maliciousServer, _ := primitives.InitializeEphemeralIdentity()
+	serverKeyBundle := NewKeyBundle()
+
+	serverKeyBundle.Keys[KeyTypeServerOnion] = Key(server.Hostname())
+
+	// This time we sign with a malicious server
+	serverKeyBundle.Sign(maliciousServer)
+	serialize := serverKeyBundle.Serialize()
+
+	newKeyBundle, err := DeserializeAndVerify(serialize)
+	if err == nil {
+		t.Fatalf("Key Bundle did Deserialize (it should have failed): %v", newKeyBundle)
+	}
+}
+
+func TestDeserializeAndVerifyUnsignedShouldFail(t *testing.T) {
+	server, _ := primitives.InitializeEphemeralIdentity()
+
+	serverKeyBundle := NewKeyBundle()
+
+	serverKeyBundle.Keys[KeyTypeServerOnion] = Key(server.Hostname())
+
+	// This time we don't sign
+	// serverKeyBundle.Sign(server)
+	serialize := serverKeyBundle.Serialize()
+
+	newKeyBundle, err := DeserializeAndVerify(serialize)
+	if err == nil {
+		t.Fatalf("Key Bundle did Deserialize (it should have failed): %v", newKeyBundle)
+	}
+}

model/message.go

@@ -1,12 +1,13 @@
 package model
 
 import (
+	"encoding/json"
 	"sort"
 	"sync"
 	"time"
 )
 
-// Timeline encapsulates a collection of ordered messages, and a mechanism to access them
+// Timeline encapsulates a collection of ordered Messages, and a mechanism to access them
 // in a threadsafe manner.
 type Timeline struct {
 	Messages      []Message
@@ -22,18 +23,25 @@ type Message struct {
 	Message            string
 	Signature          []byte
 	PreviousMessageSig []byte
+	ReceivedByServer   bool   // messages sent to a server
+	Acknowledged       bool   // peer to peer
+	Error              string `json:",omitempty"`
+	// Application specific flags, useful for storing small amounts of metadata
+	Flags uint64
 }
 
+// MessageBaseSize is a rough estimate of the base number of bytes the struct uses before strings are populated
+const MessageBaseSize = 104
+
 func compareSignatures(a []byte, b []byte) bool {
 	if len(a) != len(b) {
 		return false
 	}
+	int d = 0
 	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
+		d := d | (a[i] ^ b[i])
 	}
-	return true
+	return d == 0
 }
 
 // GetMessages returns a copy of the entire timeline
@@ -45,17 +53,34 @@ func (t *Timeline) GetMessages() []Message {
 	return messages
 }
 
+// GetCopy returns a duplicate of the Timeline
+func (t *Timeline) GetCopy() *Timeline {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	bytes, _ := json.Marshal(t)
+	newt := &Timeline{}
+	json.Unmarshal(bytes, newt)
+	return newt
+}
+
+// SetMessages sets the Messages of this timeline. Only to be used in loading/initialization
+func (t *Timeline) SetMessages(messages []Message) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	t.Messages = messages
+}
+
 // Len gets the length of the timeline
 func (t *Timeline) Len() int {
 	return len(t.Messages)
 }
 
-// Swap swaps 2 messages on the timeline.
+// Swap swaps 2 Messages on the timeline.
 func (t *Timeline) Swap(i, j int) {
 	t.Messages[i], t.Messages[j] = t.Messages[j], t.Messages[i]
 }
 
-// Less checks 2 messages (i and j) in the timeline and returns true if i occurred before j, else false
+// Less checks 2 Messages (i and j) in the timeline and returns true if i occurred before j, else false
 func (t *Timeline) Less(i, j int) bool {
 
 	if t.Messages[i].Timestamp.Before(t.Messages[j].Timestamp) {
@@ -79,7 +104,6 @@ func (t *Timeline) Less(i, j int) bool {
 }
 
 // Sort sorts the timeline in a canonical order.
-// TODO: There is almost definitely a more efficient way of doing things that involve not calling this method on every timeline load.
 func (t *Timeline) Sort() {
 	t.lock.Lock()
 	defer t.lock.Unlock()

model/message_test.go

@@ -1,8 +1,6 @@
 package model
 
 import (
-	"cwtch.im/cwtch/protocol"
-	"github.com/golang/protobuf/proto"
 	"strconv"
 	"testing"
 	"time"
@@ -17,11 +15,10 @@ func TestMessagePadding(t *testing.T) {
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
 
 	gid, invite, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	gci := &protocol.CwtchPeerPacket{}
-	proto.Unmarshal(invite, gci)
-	sarah.ProcessInvite(gci.GetGroupChatInvite(), alice.Onion)
 
-	group := alice.GetGroupByGroupID(gid)
+	sarah.ProcessInvite(invite)
+
+	group := alice.GetGroup(gid)
 
 	c1, s1, err := sarah.EncryptMessageToGroup("Hello World 1", group.GroupID)
 	t.Logf("Length of Encrypted Message: %v %v", len(c1), err)
@@ -50,12 +47,14 @@ func TestTranscriptConsistency(t *testing.T) {
 	sarah.AddContact(alice.Onion, &alice.PublicProfile)
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
 
-	gid, invite, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	gci := &protocol.CwtchPeerPacket{}
-	proto.Unmarshal(invite, gci)
-	sarah.ProcessInvite(gci.GetGroupChatInvite(), alice.Onion)
+	// The lightest weight server entry possible (usually we would import a key bundle...)
+	sarah.AddContact("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", &PublicProfile{Attributes: map[string]string{string(KeyTypeServerOnion): "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"}})
 
-	group := alice.GetGroupByGroupID(gid)
+	gid, invite, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+
+	sarah.ProcessInvite(invite)
+
+	group := alice.GetGroup(gid)
 
 	t.Logf("group: %v, sarah %v", group, sarah)
 
@@ -80,14 +79,14 @@ func TestTranscriptConsistency(t *testing.T) {
 	c5, s5, _ := alice.EncryptMessageToGroup("Hello World 5", group.GroupID)
 	t.Logf("Length of Encrypted Message: %v", len(c5))
 
-	_, _, m1 := sarah.AttemptDecryption(c1, s1)
+	_, _, m1, _ := sarah.AttemptDecryption(c1, s1)
 	sarah.AttemptDecryption(c1, s1) // Try a duplicate
-	_, _, m2 := sarah.AttemptDecryption(c2, s2)
-	_, _, m3 := sarah.AttemptDecryption(c3, s3)
-	_, _, m4 := sarah.AttemptDecryption(c4, s4)
-	_, _, m5 := sarah.AttemptDecryption(c5, s5)
+	_, _, m2, _ := sarah.AttemptDecryption(c2, s2)
+	_, _, m3, _ := sarah.AttemptDecryption(c3, s3)
+	_, _, m4, _ := sarah.AttemptDecryption(c4, s4)
+	_, _, m5, _ := sarah.AttemptDecryption(c5, s5)
 
-	// Now we simulate a client receiving these messages completely out of order
+	// Now we simulate a client receiving these Messages completely out of order
 	timeline.Insert(m1)
 	timeline.Insert(m5)
 	timeline.Insert(m4)

model/profile.go

@@ -2,28 +2,47 @@ package model
 
 import (
 	"crypto/rand"
-	"cwtch.im/cwtch/protocol"
+	"cwtch.im/cwtch/model/attr"
+	"cwtch.im/cwtch/protocol/groups"
 	"encoding/base32"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"github.com/golang/protobuf/proto"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
 	"golang.org/x/crypto/ed25519"
 	"io"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 )
 
+// Authorization is a type determining client assigned authorization to a peer
+type Authorization string
+
+const (
+	// AuthUnknown is an initial state for a new unseen peer
+	AuthUnknown Authorization = "unknown"
+	// AuthApproved means the client has approved the peer, it can send messages to us, perform GetVals, etc
+	AuthApproved Authorization = "approved"
+	// AuthBlocked means the client has blocked the peer, it's messages and connections should be rejected
+	AuthBlocked Authorization = "blocked"
+)
+
 // PublicProfile is a local copy of a CwtchIdentity
 type PublicProfile struct {
-	Name             string
-	Ed25519PublicKey ed25519.PublicKey
-	Trusted          bool
-	Blocked          bool
-	Onion            string
-	Attributes       map[string]string
-	Timeline         Timeline
-	lock             sync.Mutex
+	Name                   string
+	Ed25519PublicKey       ed25519.PublicKey
+	Authorization          Authorization
+	DeprecatedBlocked      bool `json:"Blocked"`
+	Onion                  string
+	Attributes             map[string]string
+	Timeline               Timeline `json:"-"`
+	LocalID                string   // used by storage engine
+	State                  string   `json:"-"`
+	lock                   sync.Mutex
+	UnacknowledgedMessages map[string]int
 }
 
 // Profile encapsulates all the attributes necessary to be a Cwtch Peer.
@@ -32,16 +51,25 @@ type Profile struct {
 	Contacts          map[string]*PublicProfile
 	Ed25519PrivateKey ed25519.PrivateKey
 	Groups            map[string]*Group
-	Custom            map[string]string
-	lock              sync.Mutex
 }
 
 // MaxGroupMessageLength is the maximum length of a message posted to a server group.
 // TODO: Should this be per server?
-const MaxGroupMessageLength = 1024
+const MaxGroupMessageLength = 1800
+
+// GenerateRandomID generates a random 16 byte hex id code
+func GenerateRandomID() string {
+	randBytes := make([]byte, 16)
+	rand.Read(randBytes)
+	return filepath.Join(hex.EncodeToString(randBytes))
+}
 
 func (p *PublicProfile) init() {
-	p.Attributes = make(map[string]string)
+	if p.Attributes == nil {
+		p.Attributes = make(map[string]string)
+	}
+	p.UnacknowledgedMessages = make(map[string]int)
+	p.LocalID = GenerateRandomID()
 }
 
 // SetAttribute allows applications to store arbitrary configuration info at the profile level.
@@ -51,6 +79,12 @@ func (p *PublicProfile) SetAttribute(name string, value string) {
 	p.Attributes[name] = value
 }
 
+// IsServer returns true if the profile is associated with a server.
+func (p *PublicProfile) IsServer() (isServer bool) {
+	_, isServer = p.GetAttribute(string(KeyTypeServerOnion))
+	return
+}
+
 // GetAttribute returns the value of a value set with SetCustomAttribute. If no such value has been set exists is set to false.
 func (p *PublicProfile) GetAttribute(name string) (value string, exists bool) {
 	p.lock.Lock()
@@ -62,44 +96,64 @@ func (p *PublicProfile) GetAttribute(name string) (value string, exists bool) {
 // GenerateNewProfile creates a new profile, with new encryption and signing keys, and a profile name.
 func GenerateNewProfile(name string) *Profile {
 	p := new(Profile)
+	p.init()
 	p.Name = name
 	pub, priv, _ := ed25519.GenerateKey(rand.Reader)
 	p.Ed25519PublicKey = pub
 	p.Ed25519PrivateKey = priv
-	p.Onion = utils.GetTorV3Hostname(pub)
+	p.Onion = tor.GetTorV3Hostname(pub)
 
 	p.Contacts = make(map[string]*PublicProfile)
 	p.Contacts[p.Onion] = &p.PublicProfile
 	p.Groups = make(map[string]*Group)
-	p.Custom = make(map[string]string)
 	return p
 }
 
-// GetCwtchIdentityPacket returns the wire message for conveying this profiles identity.
-func (p *Profile) GetCwtchIdentityPacket() (message []byte) {
-	ci := &protocol.CwtchIdentity{
-		Name:             p.Name,
-		Ed25519PublicKey: p.Ed25519PublicKey,
-	}
-	cpp := &protocol.CwtchPeerPacket{
-		CwtchIdentify: ci,
-	}
-	message, err := proto.Marshal(cpp)
-	utils.CheckError(err)
-	return
-}
-
 // AddContact allows direct manipulation of cwtch contacts
 func (p *Profile) AddContact(onion string, profile *PublicProfile) {
 	p.lock.Lock()
 	profile.init()
-	// TODO: More Robust V3 Onion Handling
-	decodedPub, _ := base32.StdEncoding.DecodeString(strings.ToUpper(onion[:56]))
-	profile.Ed25519PublicKey = ed25519.PublicKey(decodedPub[:32])
-	p.Contacts[onion] = profile
+	// We expect callers to verify addresses before we get to this point, so if this isn't a
+	// valid address this is a noop.
+	if tor.IsValidHostname(onion) {
+		decodedPub, err := base32.StdEncoding.DecodeString(strings.ToUpper(onion[:56]))
+		if err == nil {
+			profile.Ed25519PublicKey = ed25519.PublicKey(decodedPub[:32])
+			p.Contacts[onion] = profile
+		}
+	}
 	p.lock.Unlock()
 }
 
+// UpdateMessageFlags updates the flags stored with a message
+func (p *Profile) UpdateMessageFlags(handle string, mIdx int, flags uint64) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	if contact, exists := p.Contacts[handle]; exists {
+		if len(contact.Timeline.Messages) > mIdx {
+			contact.Timeline.Messages[mIdx].Flags = flags
+		}
+	} else if group, exists := p.Groups[handle]; exists {
+		if len(group.Timeline.Messages) > mIdx {
+			group.Timeline.Messages[mIdx].Flags = flags
+		}
+	}
+}
+
+// DeleteContact deletes a peer contact
+func (p *Profile) DeleteContact(onion string) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	delete(p.Contacts, onion)
+}
+
+// DeleteGroup deletes a group
+func (p *Profile) DeleteGroup(groupID string) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	delete(p.Groups, groupID)
+}
+
 // RejectInvite rejects and removes a group invite
 func (p *Profile) RejectInvite(groupID string) {
 	p.lock.Lock()
@@ -107,22 +161,86 @@ func (p *Profile) RejectInvite(groupID string) {
 	p.lock.Unlock()
 }
 
+// AddSentMessageToContactTimeline allows the saving of a message sent via a direct connection chat to the profile.
+func (p *Profile) AddSentMessageToContactTimeline(onion string, messageTxt string, sent time.Time, eventID string) *Message {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	contact, ok := p.Contacts[onion]
+	if ok {
+		now := time.Now()
+		sig := p.SignMessage(onion + messageTxt + sent.String() + now.String())
+
+		message := &Message{PeerID: p.Onion, Message: messageTxt, Timestamp: sent, Received: now, Signature: sig, Acknowledged: false}
+		if contact.UnacknowledgedMessages == nil {
+			contact.UnacknowledgedMessages = make(map[string]int)
+		}
+		contact.Timeline.Insert(message)
+		contact.UnacknowledgedMessages[eventID] = contact.Timeline.Len() - 1
+		return message
+	}
+	return nil
+}
+
 // AddMessageToContactTimeline allows the saving of a message sent via a direct connection chat to the profile.
-func (p *Profile) AddMessageToContactTimeline(onion string, fromMe bool, message string, sent time.Time) {
+func (p *Profile) AddMessageToContactTimeline(onion string, messageTxt string, sent time.Time) (message *Message) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	contact, ok := p.Contacts[onion]
 
 	// We don't really need a Signature here, but we use it to maintain order
 	now := time.Now()
-	sig := p.SignMessage(onion + message + sent.String() + now.String())
+	sig := p.SignMessage(onion + messageTxt + sent.String() + now.String())
 	if ok {
-		if fromMe {
-			contact.Timeline.Insert(&Message{PeerID: p.Onion, Message: message, Timestamp: sent, Received: now, Signature: sig})
-		} else {
-			contact.Timeline.Insert(&Message{PeerID: onion, Message: message, Timestamp: sent, Received: now, Signature: sig})
+		message = &Message{PeerID: onion, Message: messageTxt, Timestamp: sent, Received: now, Signature: sig, Acknowledged: true}
+		contact.Timeline.Insert(message)
+	}
+	return
+}
+
+// ErrorSentMessageToPeer sets a sent message's error message and removes it from the unacknowledged list
+func (p *Profile) ErrorSentMessageToPeer(onion string, eventID string, error string) int {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	contact, ok := p.Contacts[onion]
+	if ok {
+		mIdx, ok := contact.UnacknowledgedMessages[eventID]
+		if ok {
+			contact.Timeline.Messages[mIdx].Error = error
+			delete(contact.UnacknowledgedMessages, eventID)
+			return mIdx
 		}
 	}
+	return -1
+}
+
+// AckSentMessageToPeer sets mesage to a peer as acknowledged
+func (p *Profile) AckSentMessageToPeer(onion string, eventID string) int {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	contact, ok := p.Contacts[onion]
+	if ok {
+		mIdx, ok := contact.UnacknowledgedMessages[eventID]
+		if ok {
+			contact.Timeline.Messages[mIdx].Acknowledged = true
+			delete(contact.UnacknowledgedMessages, eventID)
+			return mIdx
+		}
+	}
+
+	return -1
+}
+
+// AddGroupSentMessageError searches matching groups for the message by sig and marks it as an error
+func (p *Profile) AddGroupSentMessageError(groupID string, signature []byte, error string) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	group, exists := p.Groups[groupID]
+	if exists {
+		group.ErrorSentMessage(signature, error)
+	}
 }
 
 // AcceptInvite accepts a group invite
@@ -149,21 +267,6 @@ func (p *Profile) GetGroups() []string {
 	return keys
 }
 
-// SetCustomAttribute allows applications to store arbitrary configuration info at the profile level.
-func (p *Profile) SetCustomAttribute(name string, value string) {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-	p.Custom[name] = value
-}
-
-// GetCustomAttribute returns the value of a value set with SetCustomAttribute. If no such value has been set exists is set to false.
-func (p *Profile) GetCustomAttribute(name string) (value string, exists bool) {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-	value, exists = p.Custom[name]
-	return
-}
-
 // GetContacts returns an unordered list of contact onions associated with this profile.
 func (p *Profile) GetContacts() []string {
 	p.lock.Lock()
@@ -177,39 +280,38 @@ func (p *Profile) GetContacts() []string {
 	return keys
 }
 
-// BlockPeer blocks a contact
-func (p *Profile) BlockPeer(onion string) (err error) {
+// SetContactAuthorization sets the authoirization level of a peer
+func (p *Profile) SetContactAuthorization(onion string, auth Authorization) (err error) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	contact, ok := p.Contacts[onion]
 	if ok {
-		contact.Blocked = true
+		contact.Authorization = auth
 	} else {
 		err = errors.New("peer does not exist")
 	}
 	return
 }
 
-// TrustPeer sets a contact to trusted
-func (p *Profile) TrustPeer(onion string) (err error) {
+// GetContactAuthorization returns the contact's authorization level
+func (p *Profile) GetContactAuthorization(onion string) Authorization {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	contact, ok := p.Contacts[onion]
 	if ok {
-		contact.Trusted = true
-	} else {
-		err = errors.New("peer does not exist")
+		return contact.Authorization
 	}
-	return
+	return AuthUnknown
 }
 
-// IsBlocked returns true if the contact has been blocked, false otherwise
-func (p *Profile) IsBlocked(onion string) bool {
-	contact, ok := p.GetContact(onion)
-	if ok {
-		return contact.Blocked
+// ContactsAuthorizations calculates a list of Peers who are at the supplied auth levels
+func (p *Profile) ContactsAuthorizations(authorizationFilter ...Authorization) map[string]Authorization {
+	authorizations := map[string]Authorization{}
+	for _, contact := range p.GetContacts() {
+		c, _ := p.GetContact(contact)
+		authorizations[c.Onion] = c.Authorization
 	}
-	return false
+	return authorizations
 }
 
 // GetContact returns a contact if the profile has it
@@ -220,22 +322,36 @@ func (p *Profile) GetContact(onion string) (*PublicProfile, bool) {
 	return contact, ok
 }
 
-// VerifyGroupMessage confirms the authenticity of a message given an onion, message and signature.
-func (p *Profile) VerifyGroupMessage(onion string, groupID string, message string, timestamp int32, ciphertext []byte, signature []byte) bool {
+// VerifyGroupMessage confirms the authenticity of a message given an sender onion, ciphertext and signature.
+// The goal of this function is 2-fold:
+// 1. We confirm that the sender referenced in the group text is the actual sender of the message (or at least
+//     knows the senders private key)
+// 2. Secondly, we confirm that the sender sent the message to a particular group id on a specific server (it doesn't
+//      matter if we actually received this message from the server or from a hybrid protocol, all that matters is
+//      that the sender and receivers agree that this message was intended for the group
+// The 2nd point is important as it prevents an attack documented in the original Cwtch paper (and later at
+// https://docs.openprivacy.ca/cwtch-security-handbook/groups.html) in which a malicious profile sets up 2 groups
+// on two different servers with the same key and then forwards messages between them to convince the parties in
+// each group that they are actually in one big group (with the intent to later censor and/or selectively send messages
+// to each group).
+func (p *Profile) VerifyGroupMessage(onion string, groupID string, ciphertext []byte, signature []byte) bool {
 
-	group := p.GetGroupByGroupID(groupID)
+	group := p.GetGroup(groupID)
 	if group == nil {
 		return false
 	}
 
+	// We use our group id, a known reference server and the ciphertext of the message.
+	m := groupID + group.GroupServer + string(ciphertext)
+
+	// If the message is ostensibly from us then we check it against our public key...
 	if onion == p.Onion {
-		m := groupID + group.GroupServer + string(ciphertext)
 		return ed25519.Verify(p.Ed25519PublicKey, []byte(m), signature)
 	}
 
-	m := groupID + group.GroupServer + string(ciphertext)
+	// Otherwise we derive the public key from the sender and check it against that.
 	decodedPub, err := base32.StdEncoding.DecodeString(strings.ToUpper(onion))
-	if err == nil {
+	if err == nil && len(decodedPub) >= 32 {
 		return ed25519.Verify(decodedPub[:32], []byte(m), signature)
 	}
 	return false
@@ -249,109 +365,91 @@ func (p *Profile) SignMessage(message string) []byte {
 
 // StartGroup when given a server, creates a new Group under this profile and returns the group id an a precomputed
 // invite which can be sent on the wire.
-func (p *Profile) StartGroup(server string) (groupID string, invite []byte, err error) {
-	return p.StartGroupWithMessage(server, []byte{})
-}
-
-// StartGroupWithMessage when given a server, and an initial message creates a new Group under this profile and returns the group id an a precomputed
-// invite which can be sent on the wire.
-func (p *Profile) StartGroupWithMessage(server string, initialMessage []byte) (groupID string, invite []byte, err error) {
+func (p *Profile) StartGroup(server string) (groupID string, invite string, err error) {
 	group, err := NewGroup(server)
 	if err != nil {
-		return "", nil, err
+		return "", "", err
 	}
 	groupID = group.GroupID
-	signedGroupID := p.SignMessage(groupID + server)
-	group.SignGroup(signedGroupID)
-	invite, err = group.Invite(initialMessage)
+	invite, err = group.Invite()
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	p.Groups[group.GroupID] = group
 	return
 }
 
-// GetGroupByGroupID a pointer to a Group by the group Id, returns nil if no group found.
-func (p *Profile) GetGroupByGroupID(groupID string) (g *Group) {
+// GetGroup a pointer to a Group by the group Id, returns nil if no group found.
+func (p *Profile) GetGroup(groupID string) (g *Group) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	g = p.Groups[groupID]
 	return
 }
 
-// ProcessInvite adds a new group invite to the profile.
-func (p *Profile) ProcessInvite(gci *protocol.GroupChatInvite, peerHostname string) {
-	group := new(Group)
-	group.GroupID = gci.GetGroupName()
-	group.SignedGroupID = gci.GetSignedGroupId()
-	copy(group.GroupKey[:], gci.GetGroupSharedKey()[:])
-	group.GroupServer = gci.GetServerHost()
-	group.InitialMessage = gci.GetInitialMessage()[:]
-	group.Accepted = false
-	group.Owner = peerHostname
-	p.AddGroup(group)
+// ProcessInvite validates a group invite and adds a new group invite to the profile if it is valid.
+// returns the new group ID on success, error on fail.
+func (p *Profile) ProcessInvite(invite string) (string, error) {
+	gci, err := ValidateInvite(invite)
+	if err == nil {
+		if server, exists := p.GetContact(gci.ServerHost); !exists || !server.IsServer() {
+			return "", fmt.Errorf("unknown server. a server key bundle needs to be imported before this group can be verified")
+		}
+		group := new(Group)
+		group.Version = CurrentGroupVersion
+		group.GroupID = gci.GroupID
+		group.LocalID = GenerateRandomID()
+		copy(group.GroupKey[:], gci.SharedKey[:])
+		group.GroupServer = gci.ServerHost
+		group.Accepted = false
+		group.Attributes = make(map[string]string)
+		group.Attributes[attr.GetLocalScope("name")] = gci.GroupName
+		p.AddGroup(group)
+		return gci.GroupID, nil
+	}
+	return "", err
 }
 
 // AddGroup is a convenience method for adding a group to a profile.
 func (p *Profile) AddGroup(group *Group) {
-	existingGroup, exists := p.Groups[group.GroupID]
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	_, exists := p.Groups[group.GroupID]
 	if !exists {
-		// TODO More robust error handling (confirm this onion checksum is correct)
-		decodedPub, _ := base32.StdEncoding.DecodeString(strings.ToUpper(group.Owner[:56]))
-		valid := ed25519.Verify(ed25519.PublicKey(decodedPub[:32]), []byte(group.GroupID+group.GroupServer), group.SignedGroupID)
-		if valid {
-			p.lock.Lock()
-			defer p.lock.Unlock()
-			p.Groups[group.GroupID] = group
-		}
-	} else if exists && existingGroup.Owner == group.Owner {
-		p.lock.Lock()
-		defer p.lock.Unlock()
 		p.Groups[group.GroupID] = group
 	}
-
-	// If we are sent an invite or group update by someone who is not an owner
-	// then we reject the group.
 }
 
 // AttemptDecryption takes a ciphertext and signature and attempts to decrypt it under known groups.
-func (p *Profile) AttemptDecryption(ciphertext []byte, signature []byte) (bool, string, *Message) {
+// If successful, adds the message to the group's timeline
+func (p *Profile) AttemptDecryption(ciphertext []byte, signature []byte) (bool, string, *Message, bool) {
 	for _, group := range p.Groups {
 		success, dgm := group.DecryptMessage(ciphertext)
 		if success {
-
-			// Assert that we know the owner of the group
-			owner, ok := p.Contacts[group.Owner]
-			if ok {
-				valid := ed25519.Verify(owner.Ed25519PublicKey, []byte(group.GroupID+group.GroupServer), dgm.SignedGroupId)
-				// If we can decrypt the message, but the group id is wrong that means that
-				// this message is from someone who was not invited to the group.
-				// As such this group has been compromised, probably by one of the other members.
-				// We set the flag to be handled by the UX and reject the message.
-				if !valid {
-					group.Compromised()
-					return false, "", nil
-				}
-			}
-
-			verified := p.VerifyGroupMessage(dgm.GetOnion(), group.GroupID, dgm.GetText(), dgm.GetTimestamp(), ciphertext, signature)
+			verified := p.VerifyGroupMessage(dgm.Onion, group.GroupID, ciphertext, signature)
 
 			// So we have a message that has a valid group key, but the signature can't be verified.
 			// The most obvious explanation for this is that the group key has been compromised (or we are in an open group and the server is being malicious)
 			// Either way, someone who has the private key is being detectably bad so we are just going to throw this message away and mark the group as Compromised.
 			if !verified {
 				group.Compromised()
-				return false, "", nil
+				return false, group.GroupID, nil, false
 			}
-
-			return true, group.GroupID, group.AddMessage(dgm, signature)
+			message, seen := group.AddMessage(dgm, signature)
+			return true, group.GroupID, message, seen
 		}
 	}
-	return false, "", nil
+
+	// If we couldn't find a group to decrypt the message with we just return false. This is an expected case
+	return false, "", nil, false
 }
 
 func getRandomness(arr *[]byte) {
 	if _, err := io.ReadFull(rand.Reader, (*arr)[:]); err != nil {
-		utils.CheckError(err)
+		if err != nil {
+			// If we can't do randomness, just crash something is very very wrong and we are not going
+			// to resolve it here....
+			panic(err.Error())
+		}
 	}
 }
 
@@ -363,35 +461,64 @@ func (p *Profile) EncryptMessageToGroup(message string, groupID string) ([]byte,
 		return nil, nil, errors.New("group message is too long")
 	}
 
-	group := p.GetGroupByGroupID(groupID)
+	group := p.GetGroup(groupID)
 	if group != nil {
 		timestamp := time.Now().Unix()
 
+		// Select the latest message from the timeline as a reference point.
 		var prevSig []byte
 		if len(group.Timeline.Messages) > 0 {
 			prevSig = group.Timeline.Messages[len(group.Timeline.Messages)-1].Signature
 		} else {
-			prevSig = group.SignedGroupID
+			prevSig = []byte(group.GroupID)
 		}
 
 		lenPadding := MaxGroupMessageLength - len(message)
 		padding := make([]byte, lenPadding)
 		getRandomness(&padding)
+		hexGroupID, err := hex.DecodeString(group.GroupID)
+		if err != nil {
+			return nil, nil, err
+		}
 
-		dm := &protocol.DecryptedGroupMessage{
-			Onion:              proto.String(p.Onion),
-			Text:               proto.String(message),
-			SignedGroupId:      group.SignedGroupID[:],
-			Timestamp:          proto.Int32(int32(timestamp)),
+		dm := &groups.DecryptedGroupMessage{
+			Onion:              p.Onion,
+			Text:               message,
+			SignedGroupID:      hexGroupID,
+			Timestamp:          uint64(timestamp),
 			PreviousMessageSig: prevSig,
 			Padding:            padding[:],
 		}
+
 		ciphertext, err := group.EncryptMessage(dm)
 		if err != nil {
 			return nil, nil, err
 		}
 		signature := p.SignMessage(groupID + group.GroupServer + string(ciphertext))
+		group.AddSentMessage(dm, signature)
 		return ciphertext, signature, nil
 	}
 	return nil, nil, errors.New("group does not exist")
 }
+
+// GetCopy returns a full deep copy of the Profile struct and its members (timeline inclusion control by arg)
+func (p *Profile) GetCopy(timeline bool) *Profile {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	newp := new(Profile)
+	bytes, _ := json.Marshal(p)
+	json.Unmarshal(bytes, &newp)
+
+	if timeline {
+		for groupID := range newp.Groups {
+			newp.Groups[groupID].Timeline = *p.Groups[groupID].Timeline.GetCopy()
+		}
+
+		for peerID := range newp.Contacts {
+			newp.Contacts[peerID].Timeline = *p.Contacts[peerID].Timeline.GetCopy()
+		}
+	}
+
+	return newp
+}

model/profile_test.go

@@ -1,46 +1,13 @@
 package model
 
 import (
-	"cwtch.im/cwtch/protocol"
-	"github.com/golang/protobuf/proto"
 	"testing"
-	"time"
 )
 
-func TestP2P(t *testing.T) {
-	sarah := GenerateNewProfile("Sarah")
-	alice := GenerateNewProfile("Alice")
-
-	sarah.AddContact(alice.Onion, &alice.PublicProfile)
-
-	sarah.AddMessageToContactTimeline(alice.Onion, false, "hello", time.Now())
-	sarah.AddMessageToContactTimeline(alice.Onion, true, "world", time.Now())
-
-	contact, _ := sarah.GetContact(alice.Onion)
-	for i, m := range contact.Timeline.GetMessages() {
-		if i == 0 && (m.Message != "hello" || m.PeerID != alice.Onion) {
-			t.Fatalf("Timeline is invalid: %v", m)
-		}
-
-		if i == 1 && (m.Message != "world" || m.PeerID != sarah.Onion) {
-			t.Fatalf("Timeline is invalid: %v", m)
-		}
-		t.Logf("Message: %v", m)
-	}
-}
-
 func TestProfileIdentity(t *testing.T) {
 	sarah := GenerateNewProfile("Sarah")
 	alice := GenerateNewProfile("Alice")
 
-	message := sarah.GetCwtchIdentityPacket()
-
-	ci := &protocol.CwtchPeerPacket{}
-	err := proto.Unmarshal(message, ci)
-	if err != nil {
-		t.Errorf("alice should have added sarah as a contact %v", err)
-	}
-
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
 	if alice.Contacts[sarah.Onion].Name != "Sarah" {
 		t.Errorf("alice should have added sarah as a contact %v", alice.Contacts)
@@ -50,8 +17,8 @@ func TestProfileIdentity(t *testing.T) {
 		t.Errorf("alice should be only contact: %v", alice.GetContacts())
 	}
 
-	alice.SetCustomAttribute("test", "hello world")
-	value, _ := alice.GetCustomAttribute("test")
+	alice.SetAttribute("test", "hello world")
+	value, _ := alice.GetAttribute("test")
 	if value != "hello world" {
 		t.Errorf("value from custom attribute should have been 'hello world', instead was: %v", value)
 	}
@@ -64,13 +31,9 @@ func TestTrustPeer(t *testing.T) {
 	alice := GenerateNewProfile("Alice")
 	sarah.AddContact(alice.Onion, &alice.PublicProfile)
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
-	alice.TrustPeer(sarah.Onion)
-	if alice.IsBlocked(sarah.Onion) {
-		t.Errorf("peer should not be blocked")
-	}
-
-	if alice.TrustPeer("") == nil {
-		t.Errorf("trusting a non existent peer should error")
+	alice.SetContactAuthorization(sarah.Onion, AuthApproved)
+	if alice.GetContactAuthorization(sarah.Onion) != AuthApproved {
+		t.Errorf("peer should be approved")
 	}
 }
 
@@ -79,13 +42,13 @@ func TestBlockPeer(t *testing.T) {
 	alice := GenerateNewProfile("Alice")
 	sarah.AddContact(alice.Onion, &alice.PublicProfile)
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
-	alice.BlockPeer(sarah.Onion)
-	if !alice.IsBlocked(sarah.Onion) {
-		t.Errorf("peer should not be blocked")
+	alice.SetContactAuthorization(sarah.Onion, AuthBlocked)
+	if alice.GetContactAuthorization(sarah.Onion) != AuthBlocked {
+		t.Errorf("peer should be blocked")
 	}
 
-	if alice.BlockPeer("") == nil {
-		t.Errorf("blocking a non existent peer should error")
+	if alice.SetContactAuthorization("", AuthUnknown) == nil {
+		t.Errorf("Seting Auth level of a non existent peer should error")
 	}
 }
 
@@ -99,14 +62,14 @@ func TestRejectGroupInvite(t *testing.T) {
 	alice := GenerateNewProfile("Alice")
 	sarah.AddContact(alice.Onion, &alice.PublicProfile)
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
+	// The lightest weight server entry possible (usually we would import a key bundle...)
+	sarah.AddContact("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", &PublicProfile{Attributes: map[string]string{string(KeyTypeServerOnion): "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"}})
 
 	gid, invite, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	gci := &protocol.CwtchPeerPacket{}
-	proto.Unmarshal(invite, gci)
-	sarah.ProcessInvite(gci.GetGroupChatInvite(), alice.Onion)
-	group := alice.GetGroupByGroupID(gid)
+	sarah.ProcessInvite(invite)
+	group := alice.GetGroup(gid)
 	if len(sarah.Groups) == 1 {
-		if sarah.GetGroupByGroupID(group.GroupID).Accepted {
+		if sarah.GetGroup(group.GroupID).Accepted {
 			t.Errorf("Group should not be accepted")
 		}
 		sarah.RejectInvite(group.GroupID)
@@ -124,33 +87,26 @@ func TestProfileGroup(t *testing.T) {
 	sarah.AddContact(alice.Onion, &alice.PublicProfile)
 	alice.AddContact(sarah.Onion, &sarah.PublicProfile)
 
-	gid, invite, _ := alice.StartGroupWithMessage("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", []byte("Hello World"))
-	gci := &protocol.CwtchPeerPacket{}
-	proto.Unmarshal(invite, gci)
-	sarah.ProcessInvite(gci.GetGroupChatInvite(), alice.Onion)
+	gid, invite, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+
+	// The lightest weight server entry possible (usually we would import a key bundle...)
+	sarah.AddContact("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", &PublicProfile{Attributes: map[string]string{string(KeyTypeServerOnion): "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"}})
+	sarah.ProcessInvite(invite)
 	if len(sarah.GetGroups()) != 1 {
 		t.Errorf("sarah should only be in 1 group instead: %v", sarah.GetGroups())
 	}
 
-	group := alice.GetGroupByGroupID(gid)
+	group := alice.GetGroup(gid)
 	sarah.AcceptInvite(group.GroupID)
 	c, s1, _ := sarah.EncryptMessageToGroup("Hello World", group.GroupID)
 	alice.AttemptDecryption(c, s1)
 
 	gid2, invite2, _ := alice.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	gci2 := &protocol.CwtchPeerPacket{}
-	proto.Unmarshal(invite2, gci2)
-	sarah.ProcessInvite(gci2.GetGroupChatInvite(), alice.Onion)
-	group2 := alice.GetGroupByGroupID(gid2)
+	sarah.ProcessInvite(invite2)
+	group2 := alice.GetGroup(gid2)
 	c2, s2, _ := sarah.EncryptMessageToGroup("Hello World", group2.GroupID)
 	alice.AttemptDecryption(c2, s2)
 
-	sarahGroup := sarah.GetGroupByGroupID(group.GroupID)
-	im := sarahGroup.GetInitialMessage()
-	if string(im) != "Hello World" {
-		t.Errorf("Initial Message was not stored properly: %v", im)
-	}
-
 	_, _, err := sarah.EncryptMessageToGroup(string(make([]byte, MaxGroupMessageLength*2)), group2.GroupID)
 	if err == nil {
 		t.Errorf("Overly long message should have returned an error")
@@ -158,18 +114,21 @@ func TestProfileGroup(t *testing.T) {
 
 	bob := GenerateNewProfile("bob")
 	bob.AddContact(alice.Onion, &alice.PublicProfile)
-	bob.ProcessInvite(gci2.GetGroupChatInvite(), alice.Onion)
+	// The lightest weight server entry possible (usually we would import a key bundle...)
+	bob.AddContact("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", &PublicProfile{Attributes: map[string]string{string(KeyTypeServerOnion): "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"}})
+
+	bob.ProcessInvite(invite2)
 	c3, s3, err := bob.EncryptMessageToGroup("Bobs Message", group2.GroupID)
 	if err == nil {
-		ok, _, message := alice.AttemptDecryption(c3, s3)
+		ok, _, message, _ := alice.AttemptDecryption(c3, s3)
 		if !ok {
 			t.Errorf("Bobs message to the group should be decrypted %v %v", message, ok)
 		}
 
 		eve := GenerateNewProfile("eve")
-		ok, _, _ = eve.AttemptDecryption(c3, s3)
+		ok, _, _, _ = eve.AttemptDecryption(c3, s3)
 		if ok {
-			t.Errorf("Eves hould not be able to decrypt messages!")
+			t.Errorf("Eves hould not be able to decrypt Messages!")
 		}
 	} else {
 		t.Errorf("Bob failed to encrypt a message to the group")

peer/cwtch_peer.go

@@ -3,66 +3,192 @@ package peer
 import (
 	"cwtch.im/cwtch/event"
 	"cwtch.im/cwtch/model"
-	"cwtch.im/cwtch/protocol"
+	"cwtch.im/cwtch/model/attr"
 	"cwtch.im/cwtch/protocol/connections"
+	"encoding/base32"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/golang/protobuf/proto"
-	"golang.org/x/crypto/ed25519"
+	"git.openprivacy.ca/openprivacy/log"
+	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
+const lastKnownSignature = "LastKnowSignature"
+
+var autoHandleableEvents = map[event.Type]bool{event.EncryptedGroupMessage: true, event.PeerStateChange: true,
+	event.ServerStateChange: true, event.NewGroupInvite: true, event.NewMessageFromPeer: true,
+	event.PeerAcknowledgement: true, event.PeerError: true, event.SendMessageToPeerError: true, event.SendMessageToGroupError: true,
+	event.NewGetValMessageFromPeer: true, event.NewRetValMessageFromPeer: true, event.ProtocolEngineStopped: true, event.RetryServerRequest: true}
+
+// DefaultEventsToHandle specifies which events will be subscribed to
+//  when a peer has its Init() function called
+var DefaultEventsToHandle = []event.Type{
+	event.EncryptedGroupMessage,
+	event.NewMessageFromPeer,
+	event.PeerAcknowledgement,
+	event.NewGroupInvite,
+	event.PeerError,
+	event.SendMessageToGroupError,
+	event.NewGetValMessageFromPeer,
+	event.ProtocolEngineStopped,
+	event.RetryServerRequest,
+}
+
 // cwtchPeer manages incoming and outgoing connections and all processing for a Cwtch cwtchPeer
 type cwtchPeer struct {
-	Profile  *model.Profile
-	mutex    sync.Mutex
-	shutdown bool
-	started  bool
+	Profile      *model.Profile
+	mutex        sync.Mutex
+	shutdown     bool
+	listenStatus bool
 
-	engine   *connections.Engine
-	queue    *event.Queue
-	eventBus *event.Manager
+	queue    event.Queue
+	eventBus event.Manager
+}
+
+func (cp *cwtchPeer) UpdateMessageFlags(handle string, mIdx int, flags uint64) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	log.Debugf("Updating Flags for %v %v %v", handle, mIdx, flags)
+	cp.Profile.UpdateMessageFlags(handle, mIdx, flags)
+	cp.eventBus.Publish(event.NewEvent(event.UpdateMessageFlags, map[event.Field]string{event.Handle: handle, event.Index: strconv.Itoa(mIdx), event.Flags: strconv.FormatUint(flags, 2)}))
+}
+
+// BlockUnknownConnections will auto disconnect from connections if authentication doesn't resolve a hostname
+// known to peer.
+func (cp *cwtchPeer) BlockUnknownConnections() {
+	cp.eventBus.Publish(event.NewEvent(event.BlockUnknownPeers, map[event.Field]string{}))
+}
+
+// AllowUnknownConnections will permit connections from unknown contacts.
+func (cp *cwtchPeer) AllowUnknownConnections() {
+	cp.eventBus.Publish(event.NewEvent(event.AllowUnknownPeers, map[event.Field]string{}))
+}
+
+// ReadContacts is a meta-interface intended to restrict callers to read-only access to contacts
+type ReadContacts interface {
+	GetContacts() []string
+	GetContact(string) *model.PublicProfile
+	GetContactAttribute(string, string) (string, bool)
+}
+
+// ModifyContacts is a meta-interface intended to restrict callers to modify-only access to contacts
+type ModifyContacts interface {
+	AddContact(nick, onion string, authorization model.Authorization)
+	SetContactAuthorization(string, model.Authorization) error
+	SetContactAttribute(string, string, string)
+	DeleteContact(string)
+}
+
+// AccessPeeringState provides access to functions relating to the underlying connections of a peer.
+type AccessPeeringState interface {
+	GetPeerState(string) (connections.ConnectionState, bool)
+}
+
+// ModifyPeeringState is a meta-interface intended to restrict callers to modify-only access to connection peers
+type ModifyPeeringState interface {
+	BlockUnknownConnections()
+	AllowUnknownConnections()
+	PeerWithOnion(string)
+	JoinServer(string) error
+}
+
+// ModifyContactsAndPeers is a meta-interface intended to restrict a call to reading and modifying contacts
+// and peers.
+type ModifyContactsAndPeers interface {
+	ReadContacts
+	ModifyContacts
+	ModifyPeeringState
+}
+
+// ReadServers provides access to the servers
+type ReadServers interface {
+	GetServers() []string
+}
+
+// ReadGroups provides read-only access to group state
+type ReadGroups interface {
+	GetGroup(string) *model.Group
+	GetGroupState(string) (connections.ConnectionState, bool)
+	GetGroups() []string
+	GetGroupAttribute(string, string) (string, bool)
+	ExportGroup(string) (string, error)
+}
+
+// ModifyGroups provides write-only access add/edit/remove new groups
+type ModifyGroups interface {
+	ImportGroup(string) (string, error)
+	StartGroup(string) (string, string, error)
+	AcceptInvite(string) error
+	RejectInvite(string)
+	DeleteGroup(string)
+	SetGroupAttribute(string, string, string)
+}
+
+// ModifyServers provides write-only access to servers
+type ModifyServers interface {
+	AddServer(string) error
+	ResyncServer(onion string) error
+}
+
+// SendMessages enables a caller to sender messages to a contact
+// Note:
+type SendMessages interface {
+	SendGetValToPeer(string, string, string)
+	SendMessageToPeer(string, string) string
+
+	// TODO This should probably not be exposed
+	StoreMessage(onion string, messageTxt string, sent time.Time)
+
+	// TODO Extract once groups are stable
+	InviteOnionToGroup(string, string) error
+}
+
+// SendMessagesToGroup enables a caller to sender messages to a group
+type SendMessagesToGroup interface {
+	SendMessageToGroupTracked(string, string) (string, error)
+}
+
+// ModifyMessages enables a caller to modify the messages in a timline
+type ModifyMessages interface {
+	UpdateMessageFlags(string, int, uint64)
 }
 
 // CwtchPeer provides us with a way of testing systems built on top of cwtch without having to
 // directly implement a cwtchPeer.
 type CwtchPeer interface {
-	Init(connectivity.ACN, *event.Manager)
-	PeerWithOnion(string) *connections.PeerPeerConnection
-	InviteOnionToGroup(string, string) error
-	SendMessageToPeer(string, string)
 
-	TrustPeer(string) error
-	BlockPeer(string) error
-	AcceptInvite(string) error
-	RejectInvite(string)
-
-	JoinServer(string)
-	SendMessageToGroup(string, string) error
-
-	GetProfile() *model.Profile
-
-	GetPeers() map[string]connections.ConnectionState
-	GetServers() map[string]connections.ConnectionState
-
-	StartGroup(string) (string, []byte, error)
-
-	ImportGroup(string) (string, error)
-	ExportGroup(string) (string, error)
-
-	GetGroup(string) *model.Group
-	GetGroups() []string
-	GetContacts() []string
-	GetContact(string) *model.PublicProfile
-
-	IsStarted() bool
+	// Core Cwtch Peer Functions that should not be exposed to
+	// most functions
+	Init(event.Manager)
+	AutoHandleEvents(events []event.Type)
 	Listen()
+	StartPeersConnections()
+	StartServerConnections()
 	Shutdown()
+
+	// Relating to local attributes
+	GetOnion() string
+	SetAttribute(string, string)
+	GetAttribute(string) (string, bool)
+
+	ReadContacts
+	ModifyContacts
+
+	AccessPeeringState
+	ModifyPeeringState
+
+	ReadGroups
+	ModifyGroups
+
+	ReadServers
+	ModifyServers
+
+	SendMessages
+	ModifyMessages
+	SendMessagesToGroup
 }
 
 // NewCwtchPeer creates and returns a new cwtchPeer with the given name.
@@ -77,199 +203,500 @@ func NewCwtchPeer(name string) CwtchPeer {
 func FromProfile(profile *model.Profile) CwtchPeer {
 	cp := new(cwtchPeer)
 	cp.Profile = profile
+	cp.shutdown = false
 	return cp
 }
 
 // Init instantiates a cwtchPeer
-func (cp *cwtchPeer) Init(acn connectivity.ACN, eventBus *event.Manager) {
-	cp.queue = event.NewEventQueue(100)
+func (cp *cwtchPeer) Init(eventBus event.Manager) {
+	cp.InitForEvents(eventBus, DefaultEventsToHandle)
+}
+
+func (cp *cwtchPeer) InitForEvents(eventBus event.Manager, toBeHandled []event.Type) {
+	cp.queue = event.NewQueue()
 	go cp.eventHandler()
 
 	cp.eventBus = eventBus
-	cp.eventBus.Subscribe(event.EncryptedGroupMessage, cp.queue.EventChannel)
-	cp.eventBus.Subscribe(event.NewGroupInvite, cp.queue.EventChannel)
-
-	// Calculate a list of Peers who have been Blocked.
-	blockedPeers := []string{}
-	for _, contact := range cp.Profile.GetContacts() {
-		c, _ := cp.Profile.GetContact(contact)
-		if c.Blocked {
-			blockedPeers = append(blockedPeers, c.Onion)
-		}
-	}
-
-	// TODO: Would be nice if ProtocolEngine did not need to explictly be given the Private Key.
-	cp.engine = connections.NewProtocolEngine(cp.Profile.Ed25519PrivateKey, acn, eventBus, blockedPeers)
-	cp.engine.Identity = identity.InitializeV3(cp.Profile.Name, &cp.Profile.Ed25519PrivateKey, &cp.Profile.Ed25519PublicKey)
+	cp.AutoHandleEvents(toBeHandled)
 }
 
-// ImportGroup intializes a group from an imported source rather than a peer invite
-func (cp *cwtchPeer) ImportGroup(exportedInvite string) (groupID string, err error) {
-	if strings.HasPrefix(exportedInvite, "torv3") {
-		data, err := base64.StdEncoding.DecodeString(exportedInvite[5+44:])
-		if err == nil {
-			cpp := &protocol.CwtchPeerPacket{}
-			err := proto.Unmarshal(data, cpp)
-			if err == nil {
-				pk, err := base64.StdEncoding.DecodeString(exportedInvite[5 : 5+44])
-				if err == nil {
-					edpk := ed25519.PublicKey(pk)
-					onion := utils.GetTorV3Hostname(edpk)
-					cp.Profile.AddContact(onion, &model.PublicProfile{Name: "", Ed25519PublicKey: edpk, Trusted: true, Blocked: false, Onion: onion})
-					cp.Profile.ProcessInvite(cpp.GetGroupChatInvite(), onion)
-					return cpp.GroupChatInvite.GetGroupName(), nil
-				}
-			}
+// AutoHandleEvents sets an event (if able) to be handled by this peer
+func (cp *cwtchPeer) AutoHandleEvents(events []event.Type) {
+	for _, ev := range events {
+		if _, exists := autoHandleableEvents[ev]; exists {
+			cp.eventBus.Subscribe(ev, cp.queue)
+		} else {
+			log.Errorf("Peer asked to autohandle event it cannot: %v\n", ev)
 		}
-	} else {
-		err = errors.New("unsupported exported group type")
 	}
-	return
+}
+
+// ImportGroup initializes a group from an imported source rather than a peer invite
+func (cp *cwtchPeer) ImportGroup(exportedInvite string) (string, error) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	gid, err := cp.Profile.ProcessInvite(exportedInvite)
+
+	if err == nil {
+		cp.eventBus.Publish(event.NewEvent(event.NewGroup, map[event.Field]string{event.GroupID: gid, event.GroupInvite: exportedInvite}))
+	}
+
+	return gid, err
 }
 
 // ExportGroup serializes a group invite so it can be given offline
 func (cp *cwtchPeer) ExportGroup(groupID string) (string, error) {
-	group := cp.Profile.GetGroupByGroupID(groupID)
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	group := cp.Profile.GetGroup(groupID)
 	if group != nil {
-		invite, err := group.Invite(group.GetInitialMessage())
-		if err == nil {
-			exportedInvite := "torv3" + base64.StdEncoding.EncodeToString(cp.Profile.Ed25519PublicKey) + base64.StdEncoding.EncodeToString(invite)
-			return exportedInvite, err
-		}
+		return group.Invite()
 	}
 	return "", errors.New("group id could not be found")
 }
 
 // StartGroup create a new group linked to the given server and returns the group ID, an invite or an error.
-func (cp *cwtchPeer) StartGroup(server string) (string, []byte, error) {
-	return cp.Profile.StartGroup(server)
-}
-
-// StartGroupWithMessage create a new group linked to the given server and returns the group ID, an invite or an error.
-func (cp *cwtchPeer) StartGroupWithMessage(server string, initialMessage []byte) (string, []byte, error) {
-	return cp.Profile.StartGroupWithMessage(server, initialMessage)
+func (cp *cwtchPeer) StartGroup(server string) (string, string, error) {
+	cp.mutex.Lock()
+	groupID, invite, err := cp.Profile.StartGroup(server)
+	cp.mutex.Unlock()
+	if err == nil {
+		group := cp.GetGroup(groupID)
+		jsobj, err := json.Marshal(group)
+		if err == nil {
+			cp.eventBus.Publish(event.NewEvent(event.GroupCreated, map[event.Field]string{
+				event.GroupID:     groupID,
+				event.GroupServer: group.GroupServer,
+				event.GroupInvite: invite,
+				// Needed for Storage Engine...
+				event.Data: string(jsobj),
+			}))
+		}
+	} else {
+		log.Errorf("error creating group: %v", err)
+	}
+	return groupID, invite, err
 }
 
 // GetGroups returns an unordered list of all group IDs.
 func (cp *cwtchPeer) GetGroups() []string {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
 	return cp.Profile.GetGroups()
 }
 
 // GetGroup returns a pointer to a specific group, nil if no group exists.
 func (cp *cwtchPeer) GetGroup(groupID string) *model.Group {
-	return cp.Profile.GetGroupByGroupID(groupID)
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	return cp.Profile.GetGroup(groupID)
+}
+
+func (cp *cwtchPeer) AddContact(nick, onion string, authorization model.Authorization) {
+	decodedPub, _ := base32.StdEncoding.DecodeString(strings.ToUpper(onion))
+	pp := &model.PublicProfile{Name: nick, Ed25519PublicKey: decodedPub, Authorization: authorization, Onion: onion, Attributes: map[string]string{"nick": nick}}
+	cp.Profile.AddContact(onion, pp)
+	pd, _ := json.Marshal(pp)
+	cp.eventBus.Publish(event.NewEvent(event.PeerCreated, map[event.Field]string{
+		event.Data:       string(pd),
+		event.RemotePeer: onion,
+	}))
+	cp.eventBus.Publish(event.NewEventList(event.SetPeerAuthorization, event.RemotePeer, onion, event.Authorization, string(authorization)))
+
+	// Default to Deleting Peer History
+	cp.eventBus.Publish(event.NewEventList(event.SetPeerAttribute, event.RemotePeer, onion, event.SaveHistoryKey, event.DeleteHistoryDefault))
+}
+
+// AddServer takes in a serialized server specification (a bundle of related keys) and adds a contact for the
+// server assuming there are no errors and the contact doesn't already exist.
+// TODO in the future this function should also integrate with a trust provider to validate the key bundle.
+func (cp *cwtchPeer) AddServer(serverSpecification string) error {
+	// This confirms that the server did at least sign the bundle
+	keyBundle, err := model.DeserializeAndVerify([]byte(serverSpecification))
+	if err != nil {
+		return err
+	}
+	log.Debugf("Got new key bundle %v", keyBundle.Serialize())
+
+	// TODO if the key bundle is incomplete then error out. In the future we may allow servers to attest to new
+	// keys or subsets of keys, but for now they must commit only to a complete set of keys required for Cwtch Groups
+	// (that way we can be assured that the keybundle we store is a valid one)
+	if !keyBundle.HasKeyType(model.KeyTypeTokenOnion) || !keyBundle.HasKeyType(model.KeyTypeServerOnion) || !keyBundle.HasKeyType(model.KeyTypePrivacyPass) {
+		return errors.New("keybundle is incomplete")
+	}
+
+	if keyBundle.HasKeyType(model.KeyTypeServerOnion) {
+		onionKey, _ := keyBundle.GetKey(model.KeyTypeServerOnion)
+		onion := string(onionKey)
+
+		// Add the contact if we don't already have it
+		if cp.GetContact(onion) == nil {
+			decodedPub, _ := base32.StdEncoding.DecodeString(strings.ToUpper(onion))
+			ab := keyBundle.AttributeBundle()
+			pp := &model.PublicProfile{Name: onion, Ed25519PublicKey: decodedPub, Authorization: model.AuthUnknown, Onion: onion, Attributes: ab}
+
+			// The only part of this function that actually modifies the profile...
+			cp.mutex.Lock()
+			cp.Profile.AddContact(onion, pp)
+			cp.mutex.Unlock()
+
+			pd, _ := json.Marshal(pp)
+
+			// Sync the Storage Engine
+			cp.eventBus.Publish(event.NewEvent(event.PeerCreated, map[event.Field]string{
+				event.Data:       string(pd),
+				event.RemotePeer: onion,
+			}))
+		}
+
+		// At this point we know the server exists
+		server := cp.GetContact(onion)
+		ab := keyBundle.AttributeBundle()
+
+		// Check server bundle for consistency if we have different keys stored than in the tofu bundle then we
+		// abort...
+		for k, v := range ab {
+			val, exists := server.GetAttribute(k)
+			if exists {
+				if val != v {
+					// this is inconsistent!
+					return model.InconsistentKeyBundleError
+				}
+			}
+			// we haven't seen this key associated with the server before
+		}
+
+		// Store the key bundle for the server so we can reconstruct a tofubundle invite
+		cp.SetContactAttribute(onion, string(model.BundleType), serverSpecification)
+
+		// If we have gotten to this point we can assume this is a safe key bundle signed by the
+		// server with no conflicting keys. So we are going to publish all the keys
+		for k, v := range ab {
+			log.Debugf("Server (%v) has %v key %v", onion, k, v)
+			cp.SetContactAttribute(onion, k, v)
+		}
+
+		return nil
+	}
+	return err
 }
 
 // GetContacts returns an unordered list of onions
 func (cp *cwtchPeer) GetContacts() []string {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
 	return cp.Profile.GetContacts()
 }
 
+// GetServers returns an unordered list of servers
+func (cp *cwtchPeer) GetServers() []string {
+	contacts := cp.Profile.GetContacts()
+	var servers []string
+	for _, contact := range contacts {
+		if cp.GetContact(contact).IsServer() {
+			servers = append(servers, contact)
+		}
+	}
+	return servers
+}
+
 // GetContact returns a given contact, nil is no such contact exists
 func (cp *cwtchPeer) GetContact(onion string) *model.PublicProfile {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
 	contact, _ := cp.Profile.GetContact(onion)
 	return contact
 }
 
-// GetProfile returns the profile associated with this cwtchPeer.
-func (cp *cwtchPeer) GetProfile() *model.Profile {
-	return cp.Profile
+func (cp *cwtchPeer) GetOnion() string {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	return cp.Profile.Onion
+}
+func (cp *cwtchPeer) GetPeerState(onion string) (connections.ConnectionState, bool) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if peer, ok := cp.Profile.Contacts[onion]; ok {
+		return connections.ConnectionStateToType()[peer.State], true
+	}
+	return connections.DISCONNECTED, false
+}
+
+func (cp *cwtchPeer) GetGroupState(groupid string) (connections.ConnectionState, bool) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if group, ok := cp.Profile.Groups[groupid]; ok {
+		return connections.ConnectionStateToType()[group.State], true
+	}
+	return connections.DISCONNECTED, false
 }
 
 // PeerWithOnion is the entry point for cwtchPeer relationships
-func (cp *cwtchPeer) PeerWithOnion(onion string) *connections.PeerPeerConnection {
-	cp.eventBus.Publish(event.NewEvent(event.PeerRequest, map[string]string{"Onion": onion}))
-	return nil
+func (cp *cwtchPeer) PeerWithOnion(onion string) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if _, exists := cp.Profile.GetContact(onion); !exists {
+		cp.AddContact(onion, onion, model.AuthApproved)
+	}
+	cp.eventBus.Publish(event.NewEvent(event.PeerRequest, map[event.Field]string{event.RemotePeer: onion}))
+}
+
+// DeleteContact deletes a peer from the profile, storage, and handling
+func (cp *cwtchPeer) DeleteContact(onion string) {
+	cp.mutex.Lock()
+	cp.Profile.DeleteContact(onion)
+	defer cp.mutex.Unlock()
+	cp.eventBus.Publish(event.NewEventList(event.DeleteContact, event.RemotePeer, onion))
+}
+
+// DeleteGroup deletes a Group from the profile, storage, and handling
+func (cp *cwtchPeer) DeleteGroup(groupID string) {
+	cp.mutex.Lock()
+	cp.Profile.DeleteGroup(groupID)
+	defer cp.mutex.Unlock()
+	cp.eventBus.Publish(event.NewEventList(event.DeleteGroup, event.GroupID, groupID))
 }
 
 // InviteOnionToGroup kicks off the invite process
 func (cp *cwtchPeer) InviteOnionToGroup(onion string, groupid string) error {
-	group := cp.Profile.GetGroupByGroupID(groupid)
+	cp.mutex.Lock()
+	group := cp.Profile.GetGroup(groupid)
 	if group == nil {
+		cp.mutex.Unlock()
 		return errors.New("invalid group id")
 	}
-
-	invite, err := group.Invite(group.InitialMessage)
+	invite, err := group.Invite()
+	cp.mutex.Unlock()
 	if err == nil {
-		cp.eventBus.Publish(event.NewEvent(event.InvitePeerToGroup, map[string]string{"Onion": onion, "Invite": string(invite)}))
+		cp.SendMessageToPeer(onion, invite)
 	}
 	return err
 }
 
 // JoinServer manages a new server connection with the given onion address
-func (cp *cwtchPeer) JoinServer(onion string) {
-	cp.eventBus.Publish(event.NewEvent(event.JoinServer, map[string]string{"Onion": onion}))
+func (cp *cwtchPeer) JoinServer(onion string) error {
+	if cp.GetContact(onion) != nil {
+		tokenY, yExists := cp.GetContact(onion).GetAttribute(string(model.KeyTypePrivacyPass))
+		tokenOnion, onionExists := cp.GetContact(onion).GetAttribute(string(model.KeyTypeTokenOnion))
+		if yExists && onionExists {
+			signature, exists := cp.GetContactAttribute(onion, lastKnownSignature)
+			if !exists {
+				signature = base64.StdEncoding.EncodeToString([]byte{})
+			}
+			cp.eventBus.Publish(event.NewEvent(event.JoinServer, map[event.Field]string{event.GroupServer: onion, event.ServerTokenY: tokenY, event.ServerTokenOnion: tokenOnion, event.Signature: signature}))
+			return nil
+		}
+	}
+	return errors.New("no keys found for server connection")
 }
 
-// SendMessageToGroup attemps to sent the given message to the given group id.
-func (cp *cwtchPeer) SendMessageToGroup(groupid string, message string) error {
-	group := cp.Profile.GetGroupByGroupID(groupid)
+// ResyncServer completely tears down and resyncs a new server connection with the given onion address
+func (cp *cwtchPeer) ResyncServer(onion string) error {
+	if cp.GetContact(onion) != nil {
+		tokenY, yExists := cp.GetContact(onion).GetAttribute(string(model.KeyTypePrivacyPass))
+		tokenOnion, onionExists := cp.GetContact(onion).GetAttribute(string(model.KeyTypeTokenOnion))
+		if yExists && onionExists {
+			signature := base64.StdEncoding.EncodeToString([]byte{})
+			cp.eventBus.Publish(event.NewEvent(event.JoinServer, map[event.Field]string{event.GroupServer: onion, event.ServerTokenY: tokenY, event.ServerTokenOnion: tokenOnion, event.Signature: signature}))
+			return nil
+		}
+	}
+	return errors.New("no keys found for server connection")
+}
+
+// SendMessageToGroupTracked attempts to sent the given message to the given group id.
+// It returns the signature of the message which can be used to identify it in any UX layer.
+func (cp *cwtchPeer) SendMessageToGroupTracked(groupid string, message string) (string, error) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	group := cp.Profile.GetGroup(groupid)
+
 	if group == nil {
-		return errors.New("invalid group id")
+		return "", errors.New("invalid group id")
 	}
 	ct, sig, err := cp.Profile.EncryptMessageToGroup(message, groupid)
 
 	if err == nil {
-		cp.eventBus.Publish(event.NewEvent(event.SendMessageToGroup, map[string]string{"Server": group.GroupServer, "Ciphertext": string(ct), "Signature": string(sig)}))
+		cp.eventBus.Publish(event.NewEvent(event.SendMessageToGroup, map[event.Field]string{event.GroupID: groupid, event.GroupServer: group.GroupServer, event.Ciphertext: base64.StdEncoding.EncodeToString(ct), event.Signature: base64.StdEncoding.EncodeToString(sig)}))
 	}
 
-	return err
+	return base64.StdEncoding.EncodeToString(sig), err
 }
 
-func (cp *cwtchPeer) SendMessageToPeer(onion string, message string) {
-	cp.eventBus.Publish(event.NewEvent(event.SendMessageToPeer, map[string]string{"Peer": onion, "Message": message}))
+func (cp *cwtchPeer) SendMessageToPeer(onion string, message string) string {
+	event := event.NewEvent(event.SendMessageToPeer, map[event.Field]string{event.RemotePeer: onion, event.Data: message})
+
+	cp.mutex.Lock()
+	contact, _ := cp.Profile.GetContact(onion)
+	event.EventID = strconv.Itoa(contact.Timeline.Len())
+	cp.Profile.AddSentMessageToContactTimeline(onion, message, time.Now(), event.EventID)
+	cp.mutex.Unlock()
+
+	cp.eventBus.Publish(event)
+
+	return event.EventID
 }
 
-// GetPeers returns a list of peer connections.
-func (cp *cwtchPeer) GetPeers() map[string]connections.ConnectionState {
-	return cp.engine.GetPeers()
-}
-
-// GetServers returns a list of server connections
-func (cp *cwtchPeer) GetServers() map[string]connections.ConnectionState {
-	return cp.engine.GetServers()
-}
-
-// TrustPeer sets an existing peer relationship to trusted
-func (cp *cwtchPeer) TrustPeer(peer string) error {
-	err := cp.Profile.TrustPeer(peer)
-	if err == nil {
-		cp.PeerWithOnion(peer)
-	}
-	return err
+func (cp *cwtchPeer) SendGetValToPeer(onion string, scope string, path string) {
+	event := event.NewEventList(event.SendGetValMessageToPeer, event.RemotePeer, onion, event.Scope, scope, event.Path, path)
+	cp.eventBus.Publish(event)
 }
 
 // BlockPeer blocks an existing peer relationship.
-func (cp *cwtchPeer) BlockPeer(peer string) error {
-	err := cp.Profile.BlockPeer(peer)
-	cp.eventBus.Publish(event.NewEvent(event.BlockPeer, map[string]string{"Onion": peer}))
+func (cp *cwtchPeer) SetContactAuthorization(peer string, authorization model.Authorization) error {
+	cp.mutex.Lock()
+	err := cp.Profile.SetContactAuthorization(peer, authorization)
+	cp.mutex.Unlock()
+	cp.eventBus.Publish(event.NewEvent(event.SetPeerAuthorization, map[event.Field]string{event.RemotePeer: peer, event.Authorization: string(authorization)}))
 	return err
 }
 
 // AcceptInvite accepts a given existing group invite
 func (cp *cwtchPeer) AcceptInvite(groupID string) error {
-	return cp.Profile.AcceptInvite(groupID)
+	cp.mutex.Lock()
+	err := cp.Profile.AcceptInvite(groupID)
+	cp.mutex.Unlock()
+	if err != nil {
+		return err
+	}
+	cp.eventBus.Publish(event.NewEvent(event.AcceptGroupInvite, map[event.Field]string{event.GroupID: groupID}))
+	cp.JoinServer(cp.Profile.Groups[groupID].GroupServer)
+
+	return nil
 }
 
 // RejectInvite rejects a given group invite.
 func (cp *cwtchPeer) RejectInvite(groupID string) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
 	cp.Profile.RejectInvite(groupID)
+	cp.eventBus.Publish(event.NewEvent(event.RejectGroupInvite, map[event.Field]string{event.GroupID: groupID}))
 }
 
+// Listen makes the peer open a listening port to accept incoming connections (and be detactably online)
 func (cp *cwtchPeer) Listen() {
-	cp.eventBus.Publish(event.NewEvent(event.ProtocolEngineStartListen, map[string]string{}))
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if !cp.listenStatus {
+		log.Infof("cwtchPeer Listen sending ProtocolEngineStartListen\n")
+		cp.listenStatus = true
+		cp.eventBus.Publish(event.NewEvent(event.ProtocolEngineStartListen, map[event.Field]string{event.Onion: cp.Profile.Onion}))
+	}
+	// else protocol engine is already listening
+
+}
+
+// StartPeersConnections attempts to connect to peer connections
+func (cp *cwtchPeer) StartPeersConnections() {
+	for _, contact := range cp.GetContacts() {
+		if !cp.GetContact(contact).IsServer() {
+			cp.PeerWithOnion(contact)
+		}
+	}
+}
+
+// StartServerConnections attempts to connect to all server connections
+func (cp *cwtchPeer) StartServerConnections() {
+	for _, contact := range cp.GetContacts() {
+		if cp.GetContact(contact).IsServer() {
+			cp.JoinServer(contact)
+		}
+	}
+}
+
+// SetAttribute sets an attribute for this profile and emits an event
+func (cp *cwtchPeer) SetAttribute(key string, val string) {
+	cp.mutex.Lock()
+	cp.Profile.SetAttribute(key, val)
+	defer cp.mutex.Unlock()
+	cp.eventBus.Publish(event.NewEvent(event.SetAttribute, map[event.Field]string{
+		event.Key:  key,
+		event.Data: val,
+	}))
+}
+
+// GetAttribute gets an attribute for the profile
+func (cp *cwtchPeer) GetAttribute(key string) (string, bool) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if val, exists := cp.Profile.GetAttribute(key); exists {
+		return val, true
+	}
+
+	if key == attr.GetLocalScope("name") {
+		return cp.Profile.Name, true
+	}
+
+	return "", false
+}
+
+// SetContactAttribute sets an attribute for the indicated contact and emits an event
+func (cp *cwtchPeer) SetContactAttribute(onion string, key string, val string) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if contact, ok := cp.Profile.GetContact(onion); ok {
+		contact.SetAttribute(key, val)
+		cp.eventBus.Publish(event.NewEvent(event.SetPeerAttribute, map[event.Field]string{
+			event.RemotePeer: onion,
+			event.Key:        key,
+			event.Data:       val,
+		}))
+	}
+}
+
+// GetContactAttribute gets an attribute for the indicated contact
+func (cp *cwtchPeer) GetContactAttribute(onion string, key string) (string, bool) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if contact, ok := cp.Profile.GetContact(onion); ok {
+		if val, exists := contact.GetAttribute(key); exists {
+			return val, true
+		}
+	}
+	return "", false
+}
+
+// SetGroupAttribute sets an attribute for the indicated group and emits an event
+func (cp *cwtchPeer) SetGroupAttribute(gid string, key string, val string) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if group := cp.Profile.GetGroup(gid); group != nil {
+		group.SetAttribute(key, val)
+		cp.eventBus.Publish(event.NewEvent(event.SetGroupAttribute, map[event.Field]string{
+			event.GroupID: gid,
+			event.Key:     key,
+			event.Data:    val,
+		}))
+	}
+}
+
+// GetGroupAttribute gets an attribute for the indicated group
+func (cp *cwtchPeer) GetGroupAttribute(gid string, key string) (string, bool) {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
+	if group := cp.Profile.GetGroup(gid); group != nil {
+		if val, exists := group.GetAttribute(key); exists {
+			return val, true
+		}
+	}
+	return "", false
 }
 
 // Shutdown kills all connections and cleans up all goroutines for the peer
 func (cp *cwtchPeer) Shutdown() {
+	cp.mutex.Lock()
+	defer cp.mutex.Unlock()
 	cp.shutdown = true
-	cp.engine.Shutdown()
 	cp.queue.Shutdown()
 }
 
-// IsStarted returns true if Listen() has successfully been run before on this connection (ever). TODO: we will need to properly unset this flag on error if we want to support resumption in the future
-func (cp *cwtchPeer) IsStarted() bool {
-	return cp.started
+func (cp *cwtchPeer) StoreMessage(onion string, messageTxt string, sent time.Time) {
+	if cp.GetContact(onion) == nil {
+		cp.AddContact(onion, onion, model.AuthUnknown)
+	}
+	cp.mutex.Lock()
+	cp.Profile.AddMessageToContactTimeline(onion, messageTxt, sent)
+	cp.mutex.Unlock()
 }
 
 // eventHandler process events from other subsystems
@@ -277,18 +704,127 @@ func (cp *cwtchPeer) eventHandler() {
 	for {
 		ev := cp.queue.Next()
 		switch ev.EventType {
+		/***** Default auto handled events *****/
+		case event.ProtocolEngineStopped:
+			cp.mutex.Lock()
+			cp.listenStatus = false
+			log.Infof("Protocol engine for %v has stopped listening", cp.Profile.Onion)
+			cp.mutex.Unlock()
 		case event.EncryptedGroupMessage:
-			ok, groupID, _ := cp.Profile.AttemptDecryption([]byte(ev.Data["Ciphertext"]), []byte(ev.Data["Signature"]))
-			if ok {
-				cp.eventBus.Publish(event.NewEvent(event.NewMessageFromGroup, map[string]string{"GroupID": groupID}))
+			// If successful, a side effect is the message is added to the group's timeline
+
+			ciphertext, _ := base64.StdEncoding.DecodeString(ev.Data[event.Ciphertext])
+			signature, _ := base64.StdEncoding.DecodeString(ev.Data[event.Signature])
+
+			// SECURITY NOTE: A malicious server could insert posts such that everyone always has a different lastKnownSignature
+			// However the server can always replace **all** messages in an attempt to track users
+			// This is mitigated somewhat by resync events which do wipe things entire.
+			// The security of cwtch groups are also not dependent on the servers inability to uniquely tag connections (as long as
+			// it learns nothing else about each connection).
+			cp.SetContactAttribute(ev.Data[event.GroupServer], lastKnownSignature, ev.Data[event.Signature])
+
+			cp.mutex.Lock()
+			ok, groupID, message, seen := cp.Profile.AttemptDecryption(ciphertext, signature)
+			cp.mutex.Unlock()
+			if ok && !seen {
+				cp.eventBus.Publish(event.NewEvent(event.NewMessageFromGroup, map[event.Field]string{event.TimestampReceived: message.Received.Format(time.RFC3339Nano), event.TimestampSent: message.Timestamp.Format(time.RFC3339Nano), event.Data: message.Message, event.GroupID: groupID, event.Signature: base64.StdEncoding.EncodeToString(message.Signature), event.PreviousSignature: base64.StdEncoding.EncodeToString(message.PreviousMessageSig), event.RemotePeer: message.PeerID}))
 			}
-		case event.NewGroupInvite:
-			var groupInvite protocol.GroupChatInvite
-			proto.Unmarshal([]byte(ev.Data["GroupInvite"]), &groupInvite)
-			cp.Profile.ProcessInvite(&groupInvite, ev.Data["Onion"])
+
+			// The group has been compromised
+			if !ok && groupID != "" {
+				if cp.Profile.GetGroup(groupID).IsCompromised {
+					cp.eventBus.Publish(event.NewEvent(event.GroupCompromised, map[event.Field]string{event.GroupID: groupID}))
+				}
+			}
+		case event.NewMessageFromPeer: //event.TimestampReceived, event.RemotePeer, event.Data
+			ts, _ := time.Parse(time.RFC3339Nano, ev.Data[event.TimestampReceived])
+			cp.StoreMessage(ev.Data[event.RemotePeer], ev.Data[event.Data], ts)
+
+		case event.PeerAcknowledgement:
+			cp.mutex.Lock()
+			idx := cp.Profile.AckSentMessageToPeer(ev.Data[event.RemotePeer], ev.Data[event.EventID])
+			edata := ev.Data
+			edata[event.Index] = strconv.Itoa(idx)
+			cp.eventBus.Publish(event.NewEvent(event.IndexedAcknowledgement, edata))
+			cp.mutex.Unlock()
+
+		case event.SendMessageToGroupError:
+			cp.mutex.Lock()
+			signature, _ := base64.StdEncoding.DecodeString(ev.Data[event.Signature])
+			cp.Profile.AddGroupSentMessageError(ev.Data[event.GroupID], signature, ev.Data[event.Error])
+			cp.mutex.Unlock()
+
+		case event.SendMessageToPeerError:
+			cp.mutex.Lock()
+			idx := cp.Profile.ErrorSentMessageToPeer(ev.Data[event.RemotePeer], ev.Data[event.EventID], ev.Data[event.Error])
+			edata := ev.Data
+			edata[event.Index] = strconv.Itoa(idx)
+			cp.eventBus.Publish(event.NewEvent(event.IndexedFailure, edata))
+			cp.mutex.Unlock()
+		case event.RetryServerRequest:
+			// Automated Join Server Request triggered by a plugin.
+			log.Debugf("profile received an automated retry event for %v", ev.Data[event.GroupServer])
+			cp.JoinServer(ev.Data[event.GroupServer])
+		case event.NewGetValMessageFromPeer:
+			onion := ev.Data[event.RemotePeer]
+			scope := ev.Data[event.Scope]
+			path := ev.Data[event.Path]
+
+			log.Debugf("NewGetValMessageFromPeer for %v%v from %v\n", scope, path, onion)
+
+			remotePeer := cp.GetContact(onion)
+			if remotePeer != nil && remotePeer.Authorization == model.AuthApproved {
+				if scope == attr.PublicScope {
+					val, exists := cp.GetAttribute(attr.GetPublicScope(path))
+					resp := event.NewEvent(event.SendRetValMessageToPeer, map[event.Field]string{event.RemotePeer: onion, event.Exists: strconv.FormatBool(exists)})
+					resp.EventID = ev.EventID
+					if exists {
+						resp.Data[event.Data] = val
+					} else {
+						resp.Data[event.Data] = ""
+					}
+					log.Debugf("Responding with SendRetValMessageToPeer exists:%v data: %v\n", exists, val)
+
+					cp.eventBus.Publish(resp)
+				}
+			}
+
+		/***** Non default but requestable handlable events *****/
+
+		case event.NewRetValMessageFromPeer:
+			onion := ev.Data[event.RemotePeer]
+			scope := ev.Data[event.Scope]
+			path := ev.Data[event.Path]
+			val := ev.Data[event.Data]
+			exists, _ := strconv.ParseBool(ev.Data[event.Exists])
+			log.Debugf("NewRetValMessageFromPeer %v %v%v %v %v\n", onion, scope, path, exists, val)
+			if exists {
+				if scope == attr.PublicScope {
+					cp.SetContactAttribute(onion, attr.GetPeerScope(path), val)
+				}
+			}
+		case event.PeerStateChange:
+			cp.mutex.Lock()
+			if _, exists := cp.Profile.Contacts[ev.Data[event.RemotePeer]]; exists {
+				cp.Profile.Contacts[ev.Data[event.RemotePeer]].State = ev.Data[event.ConnectionState]
+			}
+			cp.mutex.Unlock()
+		case event.ServerStateChange:
+			cp.mutex.Lock()
+			// We update both the server contact status, as well as the groups the server belongs to
+			cp.Profile.Contacts[ev.Data[event.GroupServer]].State = ev.Data[event.ConnectionState]
+
+			// TODO deprecate this, the UI should consult the server contact entry instead (it's far more efficient)
+			for _, group := range cp.Profile.Groups {
+				if group.GroupServer == ev.Data[event.GroupServer] {
+					group.State = ev.Data[event.ConnectionState]
+				}
+			}
+			cp.mutex.Unlock()
+
 		default:
 			if ev.EventType != "" {
-				log.Error("peer event handler received an event it was not subscribed for: %v")
+				log.Errorf("peer event handler received an event it was not subscribed for: %v", ev.EventType)
 			}
 			return
 		}

peer/cwtch_peer_test.go

@@ -1,79 +0,0 @@
-package peer
-
-import (
-	"testing"
-)
-
-// TODO: Rewrite these tests (and others) using the news event bus interface.
-func TestCwtchPeerGenerate(t *testing.T) {
-	/**
-	alice := NewCwtchPeer("alice")
-
-	groupID, _, _ := alice.StartGroup("test.server")
-	exportedGroup, _ := alice.ExportGroup(groupID)
-	t.Logf("Exported Group: %v from %v", exportedGroup, alice.GetProfile().Onion)
-
-	importedGroupID, err := alice.ImportGroup(exportedGroup)
-	group := alice.GetGroup(importedGroupID)
-	t.Logf("Imported Group: %v, err := %v %v", group, err, importedGroupID)
-	*/
-}
-
-func TestTrustPeer(t *testing.T) {
-	/**
-	groupName := "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"
-	alice := NewCwtchPeer("alice")
-	aem := new(event.Manager)
-	aem.Initialize()
-	alice.Init(connectivity.LocalProvider(),aem)
-	defer alice.Shutdown()
-	bob := NewCwtchPeer("bob")
-	bem := new(event.Manager)
-	bem.Initialize()
-	bob.Init(connectivity.LocalProvider(), bem)
-	defer bob.Shutdown()
-
-	bobOnion := bob.GetProfile().Onion
-	aliceOnion := alice.GetProfile().Onion
-
-	groupID, _, err := alice.StartGroup(groupName)
-	if err != nil {
-		t.Error(err)
-	}
-
-	groupAlice := alice.GetGroup(groupID)
-	if groupAlice.GroupID != groupID {
-		t.Errorf("Alice should be part of group %v, got %v instead", groupID, groupAlice)
-	}
-
-	exportedGroup, err := alice.ExportGroup(groupID)
-	if err != nil {
-		t.Error(err)
-	}
-
-	err = alice.InviteOnionToGroup(bobOnion, groupID)
-	if err == nil {
-		t.Errorf("onion invitation should fail since alice does no trust bob")
-	}
-
-	err = alice.TrustPeer(bobOnion)
-	if err == nil {
-		t.Errorf("trust peer should fail since alice does not know about bob")
-	}
-
-	// bob adds alice contact by importing serialized group created by alice
-	_, err = bob.ImportGroup(exportedGroup)
-	if err != nil {
-		t.Error(err)
-	}
-	err = bob.TrustPeer(aliceOnion)
-	if err != nil {
-		t.Errorf("bob must be able to trust alice, got %v", err)
-	}
-
-	err = bob.InviteOnionToGroup(aliceOnion, groupID)
-	if err == nil {
-		t.Errorf("bob trusts alice but peer connection is not ready yet. should not be able to invite her to group, instead got: %v", err)
-	}
-	*/
-}

protocol/ControlChannel.pb.go

@@ -1,325 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: ControlChannel.proto
-
-/*
-Package protocol is a generated protocol buffer package.
-
-It is generated from these files:
-	ControlChannel.proto
-	cwtch-profile.proto
-	group_message.proto
-
-It has these top-level messages:
-	Packet
-	OpenChannel
-	ChannelResult
-	KeepAlive
-	EnableFeatures
-	FeaturesEnabled
-	CwtchPeerPacket
-	CwtchIdentity
-	GroupChatInvite
-	CwtchServerPacket
-	FetchMessage
-	GroupMessage
-	DecryptedGroupMessage
-*/
-package protocol
-
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
-
-type ChannelResult_CommonError int32
-
-const (
-	ChannelResult_GenericError      ChannelResult_CommonError = 0
-	ChannelResult_UnknownTypeError  ChannelResult_CommonError = 1
-	ChannelResult_UnauthorizedError ChannelResult_CommonError = 2
-	ChannelResult_BadUsageError     ChannelResult_CommonError = 3
-	ChannelResult_FailedError       ChannelResult_CommonError = 4
-)
-
-var ChannelResult_CommonError_name = map[int32]string{
-	0: "GenericError",
-	1: "UnknownTypeError",
-	2: "UnauthorizedError",
-	3: "BadUsageError",
-	4: "FailedError",
-}
-var ChannelResult_CommonError_value = map[string]int32{
-	"GenericError":      0,
-	"UnknownTypeError":  1,
-	"UnauthorizedError": 2,
-	"BadUsageError":     3,
-	"FailedError":       4,
-}
-
-func (x ChannelResult_CommonError) Enum() *ChannelResult_CommonError {
-	p := new(ChannelResult_CommonError)
-	*p = x
-	return p
-}
-func (x ChannelResult_CommonError) String() string {
-	return proto.EnumName(ChannelResult_CommonError_name, int32(x))
-}
-func (x *ChannelResult_CommonError) UnmarshalJSON(data []byte) error {
-	value, err := proto.UnmarshalJSONEnum(ChannelResult_CommonError_value, data, "ChannelResult_CommonError")
-	if err != nil {
-		return err
-	}
-	*x = ChannelResult_CommonError(value)
-	return nil
-}
-func (ChannelResult_CommonError) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 0} }
-
-type Packet struct {
-	// Must contain exactly one field
-	OpenChannel      *OpenChannel     `protobuf:"bytes,1,opt,name=open_channel,json=openChannel" json:"open_channel,omitempty"`
-	ChannelResult    *ChannelResult   `protobuf:"bytes,2,opt,name=channel_result,json=channelResult" json:"channel_result,omitempty"`
-	KeepAlive        *KeepAlive       `protobuf:"bytes,3,opt,name=keep_alive,json=keepAlive" json:"keep_alive,omitempty"`
-	EnableFeatures   *EnableFeatures  `protobuf:"bytes,4,opt,name=enable_features,json=enableFeatures" json:"enable_features,omitempty"`
-	FeaturesEnabled  *FeaturesEnabled `protobuf:"bytes,5,opt,name=features_enabled,json=featuresEnabled" json:"features_enabled,omitempty"`
-	XXX_unrecognized []byte           `json:"-"`
-}
-
-func (m *Packet) Reset()                    { *m = Packet{} }
-func (m *Packet) String() string            { return proto.CompactTextString(m) }
-func (*Packet) ProtoMessage()               {}
-func (*Packet) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
-
-func (m *Packet) GetOpenChannel() *OpenChannel {
-	if m != nil {
-		return m.OpenChannel
-	}
-	return nil
-}
-
-func (m *Packet) GetChannelResult() *ChannelResult {
-	if m != nil {
-		return m.ChannelResult
-	}
-	return nil
-}
-
-func (m *Packet) GetKeepAlive() *KeepAlive {
-	if m != nil {
-		return m.KeepAlive
-	}
-	return nil
-}
-
-func (m *Packet) GetEnableFeatures() *EnableFeatures {
-	if m != nil {
-		return m.EnableFeatures
-	}
-	return nil
-}
-
-func (m *Packet) GetFeaturesEnabled() *FeaturesEnabled {
-	if m != nil {
-		return m.FeaturesEnabled
-	}
-	return nil
-}
-
-type OpenChannel struct {
-	ChannelIdentifier            *int32  `protobuf:"varint,1,req,name=channel_identifier,json=channelIdentifier" json:"channel_identifier,omitempty"`
-	ChannelType                  *string `protobuf:"bytes,2,req,name=channel_type,json=channelType" json:"channel_type,omitempty"`
-	proto.XXX_InternalExtensions `json:"-"`
-	XXX_unrecognized             []byte `json:"-"`
-}
-
-func (m *OpenChannel) Reset()                    { *m = OpenChannel{} }
-func (m *OpenChannel) String() string            { return proto.CompactTextString(m) }
-func (*OpenChannel) ProtoMessage()               {}
-func (*OpenChannel) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
-
-var extRange_OpenChannel = []proto.ExtensionRange{
-	{Start: 100, End: 536870911},
-}
-
-func (*OpenChannel) ExtensionRangeArray() []proto.ExtensionRange {
-	return extRange_OpenChannel
-}
-
-func (m *OpenChannel) GetChannelIdentifier() int32 {
-	if m != nil && m.ChannelIdentifier != nil {
-		return *m.ChannelIdentifier
-	}
-	return 0
-}
-
-func (m *OpenChannel) GetChannelType() string {
-	if m != nil && m.ChannelType != nil {
-		return *m.ChannelType
-	}
-	return ""
-}
-
-type ChannelResult struct {
-	ChannelIdentifier            *int32                     `protobuf:"varint,1,req,name=channel_identifier,json=channelIdentifier" json:"channel_identifier,omitempty"`
-	Opened                       *bool                      `protobuf:"varint,2,req,name=opened" json:"opened,omitempty"`
-	CommonError                  *ChannelResult_CommonError `protobuf:"varint,3,opt,name=common_error,json=commonError,enum=protocol.ChannelResult_CommonError" json:"common_error,omitempty"`
-	proto.XXX_InternalExtensions `json:"-"`
-	XXX_unrecognized             []byte `json:"-"`
-}
-
-func (m *ChannelResult) Reset()                    { *m = ChannelResult{} }
-func (m *ChannelResult) String() string            { return proto.CompactTextString(m) }
-func (*ChannelResult) ProtoMessage()               {}
-func (*ChannelResult) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
-
-var extRange_ChannelResult = []proto.ExtensionRange{
-	{Start: 100, End: 536870911},
-}
-
-func (*ChannelResult) ExtensionRangeArray() []proto.ExtensionRange {
-	return extRange_ChannelResult
-}
-
-func (m *ChannelResult) GetChannelIdentifier() int32 {
-	if m != nil && m.ChannelIdentifier != nil {
-		return *m.ChannelIdentifier
-	}
-	return 0
-}
-
-func (m *ChannelResult) GetOpened() bool {
-	if m != nil && m.Opened != nil {
-		return *m.Opened
-	}
-	return false
-}
-
-func (m *ChannelResult) GetCommonError() ChannelResult_CommonError {
-	if m != nil && m.CommonError != nil {
-		return *m.CommonError
-	}
-	return ChannelResult_GenericError
-}
-
-type KeepAlive struct {
-	ResponseRequested *bool  `protobuf:"varint,1,req,name=response_requested,json=responseRequested" json:"response_requested,omitempty"`
-	XXX_unrecognized  []byte `json:"-"`
-}
-
-func (m *KeepAlive) Reset()                    { *m = KeepAlive{} }
-func (m *KeepAlive) String() string            { return proto.CompactTextString(m) }
-func (*KeepAlive) ProtoMessage()               {}
-func (*KeepAlive) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
-
-func (m *KeepAlive) GetResponseRequested() bool {
-	if m != nil && m.ResponseRequested != nil {
-		return *m.ResponseRequested
-	}
-	return false
-}
-
-type EnableFeatures struct {
-	Feature                      []string `protobuf:"bytes,1,rep,name=feature" json:"feature,omitempty"`
-	proto.XXX_InternalExtensions `json:"-"`
-	XXX_unrecognized             []byte `json:"-"`
-}
-
-func (m *EnableFeatures) Reset()                    { *m = EnableFeatures{} }
-func (m *EnableFeatures) String() string            { return proto.CompactTextString(m) }
-func (*EnableFeatures) ProtoMessage()               {}
-func (*EnableFeatures) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
-
-var extRange_EnableFeatures = []proto.ExtensionRange{
-	{Start: 100, End: 536870911},
-}
-
-func (*EnableFeatures) ExtensionRangeArray() []proto.ExtensionRange {
-	return extRange_EnableFeatures
-}
-
-func (m *EnableFeatures) GetFeature() []string {
-	if m != nil {
-		return m.Feature
-	}
-	return nil
-}
-
-type FeaturesEnabled struct {
-	Feature                      []string `protobuf:"bytes,1,rep,name=feature" json:"feature,omitempty"`
-	proto.XXX_InternalExtensions `json:"-"`
-	XXX_unrecognized             []byte `json:"-"`
-}
-
-func (m *FeaturesEnabled) Reset()                    { *m = FeaturesEnabled{} }
-func (m *FeaturesEnabled) String() string            { return proto.CompactTextString(m) }
-func (*FeaturesEnabled) ProtoMessage()               {}
-func (*FeaturesEnabled) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
-
-var extRange_FeaturesEnabled = []proto.ExtensionRange{
-	{Start: 100, End: 536870911},
-}
-
-func (*FeaturesEnabled) ExtensionRangeArray() []proto.ExtensionRange {
-	return extRange_FeaturesEnabled
-}
-
-func (m *FeaturesEnabled) GetFeature() []string {
-	if m != nil {
-		return m.Feature
-	}
-	return nil
-}
-
-func init() {
-	proto.RegisterType((*Packet)(nil), "protocol.Packet")
-	proto.RegisterType((*OpenChannel)(nil), "protocol.OpenChannel")
-	proto.RegisterType((*ChannelResult)(nil), "protocol.ChannelResult")
-	proto.RegisterType((*KeepAlive)(nil), "protocol.KeepAlive")
-	proto.RegisterType((*EnableFeatures)(nil), "protocol.EnableFeatures")
-	proto.RegisterType((*FeaturesEnabled)(nil), "protocol.FeaturesEnabled")
-	proto.RegisterEnum("protocol.ChannelResult_CommonError", ChannelResult_CommonError_name, ChannelResult_CommonError_value)
-}
-
-func init() { proto.RegisterFile("ControlChannel.proto", fileDescriptor0) }
-
-var fileDescriptor0 = []byte{
-	// 461 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x4d, 0x8f, 0xd3, 0x30,
-	0x10, 0x25, 0xe9, 0xee, 0x92, 0x4e, 0xfa, 0x91, 0x9a, 0x5d, 0x30, 0xb7, 0x12, 0x2e, 0x15, 0x12,
-	0x3d, 0x54, 0x20, 0x21, 0x0e, 0x48, 0x4b, 0xd9, 0x22, 0xc4, 0x01, 0x64, 0xd1, 0x73, 0x64, 0x92,
-	0x29, 0x1b, 0x35, 0x6b, 0x1b, 0xc7, 0x05, 0x2d, 0xa7, 0xfe, 0x0e, 0xfe, 0x0c, 0x7f, 0x0d, 0xc5,
-	0x89, 0x9b, 0x14, 0x09, 0x09, 0x4e, 0xc9, 0x9b, 0xf7, 0xde, 0x8c, 0xfc, 0x66, 0xe0, 0x7c, 0x29,
-	0x85, 0xd1, 0xb2, 0x58, 0x5e, 0x73, 0x21, 0xb0, 0x98, 0x2b, 0x2d, 0x8d, 0x24, 0x81, 0xfd, 0xa4,
-	0xb2, 0x88, 0x7f, 0xf9, 0x70, 0xf6, 0x91, 0xa7, 0x5b, 0x34, 0xe4, 0x05, 0x0c, 0xa4, 0x42, 0x91,
-	0xa4, 0xb5, 0x94, 0x7a, 0x53, 0x6f, 0x16, 0x2e, 0x2e, 0xe6, 0x4e, 0x3b, 0xff, 0xa0, 0x50, 0x34,
-	0x7d, 0x58, 0x28, 0x5b, 0x40, 0x5e, 0xc1, 0xa8, 0x31, 0x25, 0x1a, 0xcb, 0x5d, 0x61, 0xa8, 0x6f,
-	0xbd, 0x0f, 0x5a, 0xaf, 0xf3, 0x59, 0x9a, 0x0d, 0xd3, 0x2e, 0x24, 0x0b, 0x80, 0x2d, 0xa2, 0x4a,
-	0x78, 0x91, 0x7f, 0x43, 0xda, 0xb3, 0xde, 0x7b, 0xad, 0xf7, 0x3d, 0xa2, 0xba, 0xac, 0x28, 0xd6,
-	0xdf, 0xba, 0x5f, 0x72, 0x09, 0x63, 0x14, 0xfc, 0x73, 0x81, 0xc9, 0x06, 0xb9, 0xd9, 0x69, 0x2c,
-	0xe9, 0x89, 0x35, 0xd2, 0xd6, 0x78, 0x65, 0x05, 0xab, 0x86, 0x67, 0x23, 0x3c, 0xc2, 0xe4, 0x0d,
-	0x44, 0xce, 0x9b, 0xd4, 0x54, 0x46, 0x4f, 0x6d, 0x8f, 0x87, 0x6d, 0x0f, 0xa7, 0xae, 0x7b, 0x65,
-	0x6c, 0xbc, 0x39, 0x2e, 0xc4, 0x39, 0x84, 0x9d, 0x60, 0xc8, 0x53, 0x20, 0x2e, 0x8b, 0x3c, 0x43,
-	0x61, 0xf2, 0x4d, 0x8e, 0x9a, 0x7a, 0x53, 0x7f, 0x76, 0xca, 0x26, 0x0d, 0xf3, 0xee, 0x40, 0x90,
-	0x47, 0x30, 0x70, 0x72, 0x73, 0xab, 0x90, 0xfa, 0x53, 0x7f, 0xd6, 0x67, 0x61, 0x53, 0xfb, 0x74,
-	0xab, 0xf0, 0x49, 0x10, 0x64, 0xd1, 0x7e, 0xbf, 0xdf, 0xfb, 0xf1, 0x4f, 0x1f, 0x86, 0x47, 0x41,
-	0xfe, 0xef, 0xb4, 0xfb, 0x70, 0x56, 0xed, 0x0d, 0x33, 0x3b, 0x27, 0x60, 0x0d, 0x22, 0x2b, 0x18,
-	0xa4, 0xf2, 0xe6, 0x46, 0x8a, 0x04, 0xb5, 0x96, 0xda, 0xae, 0x60, 0xb4, 0x78, 0xfc, 0x97, 0xf5,
-	0xcd, 0x97, 0x56, 0x7b, 0x55, 0x49, 0x59, 0x98, 0xb6, 0x20, 0x56, 0x10, 0x76, 0x38, 0x12, 0xc1,
-	0xe0, 0x2d, 0x0a, 0xd4, 0x79, 0x6a, 0x71, 0x74, 0x87, 0x9c, 0x43, 0xb4, 0x16, 0x5b, 0x21, 0xbf,
-	0x8b, 0xea, 0x69, 0x75, 0xd5, 0x23, 0x17, 0x30, 0x59, 0x0b, 0xbe, 0x33, 0xd7, 0x52, 0xe7, 0x3f,
-	0x30, 0xab, 0xcb, 0x3e, 0x99, 0xc0, 0xf0, 0x35, 0xcf, 0xd6, 0x25, 0xff, 0xd2, 0x28, 0x7b, 0x64,
-	0x0c, 0xe1, 0x8a, 0xe7, 0x85, 0xd3, 0x9c, 0x74, 0xc2, 0x79, 0x09, 0xfd, 0xc3, 0xa1, 0x54, 0xb9,
-	0x68, 0x2c, 0x95, 0x14, 0x25, 0x26, 0x1a, 0xbf, 0xee, 0xb0, 0x34, 0x98, 0xd9, 0x5c, 0x02, 0x36,
-	0x71, 0x0c, 0x73, 0x44, 0xfc, 0x0c, 0x46, 0xc7, 0xb7, 0x42, 0x28, 0xdc, 0x6d, 0x16, 0x4d, 0xbd,
-	0x69, 0x6f, 0xd6, 0x67, 0x0e, 0x76, 0x26, 0x3e, 0x87, 0xf1, 0x1f, 0xd7, 0xf1, 0x2f, 0xb6, 0xdf,
-	0x01, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x32, 0x16, 0x1e, 0x93, 0x03, 0x00, 0x00,
-}

protocol/ControlChannel.proto

@@ -1,53 +0,0 @@
-syntax = "proto2";
-package protocol;
-
-message Packet {
-    // Must contain exactly one field
-    optional OpenChannel open_channel = 1;
-    optional ChannelResult channel_result = 2;
-    optional KeepAlive keep_alive = 3;
-    optional EnableFeatures enable_features = 4;
-    optional FeaturesEnabled features_enabled = 5;
-}
-
-message OpenChannel {
-    required int32 channel_identifier = 1;      // Arbitrary unique identifier for this channel instance
-    required string channel_type = 2;           // String identifying channel type; e.g. im.ricochet.chat
-
-    // It is valid to extend the OpenChannel message to add fields specific
-    // to the requested channel_type.
-    extensions 100 to max;
-}
-
-message ChannelResult {
-    required int32 channel_identifier = 1;      // Matching the value from OpenChannel
-    required bool opened = 2;                   // If the channel is now open
-
-    enum CommonError {
-        GenericError = 0;
-        UnknownTypeError = 1;
-        UnauthorizedError = 2;
-        BadUsageError = 3;
-        FailedError = 4;
-    }
-
-    optional CommonError common_error = 3;
-
-    // As with OpenChannel, it is valid to extend this message with fields specific
-    // to the channel type.
-    extensions 100 to max;
-}
-
-message KeepAlive {
-    required bool response_requested = 1;
-}
-
-message EnableFeatures {
-    repeated string feature = 1;
-    extensions 100 to max;
-}
-
-message FeaturesEnabled {
-    repeated string feature = 1;
-    extensions 100 to max;
-}

protocol/connections/connectionsmanager.go

@@ -1,151 +0,0 @@
-package connections
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"sync"
-	"time"
-)
-
-// Manager encapsulates all the logic necessary to manage outgoing peer and server connections.
-type Manager struct {
-	peerConnections   map[string]*PeerPeerConnection
-	serverConnections map[string]*PeerServerConnection
-	lock              sync.Mutex
-	breakChannel      chan bool
-	acn               connectivity.ACN
-}
-
-// NewConnectionsManager creates a new instance of Manager.
-func NewConnectionsManager(acn connectivity.ACN) *Manager {
-	m := new(Manager)
-	m.acn = acn
-	m.peerConnections = make(map[string]*PeerPeerConnection)
-	m.serverConnections = make(map[string]*PeerServerConnection)
-	m.breakChannel = make(chan bool)
-	return m
-}
-
-// ManagePeerConnection creates a new PeerConnection for the given Host and Profile.
-func (m *Manager) ManagePeerConnection(host string, engine *Engine) *PeerPeerConnection {
-	m.lock.Lock()
-	defer m.lock.Unlock()
-
-	_, exists := m.peerConnections[host]
-	if !exists {
-		ppc := NewPeerPeerConnection(host, engine)
-		go ppc.Run()
-		m.peerConnections[host] = ppc
-		return ppc
-	}
-	return m.peerConnections[host]
-}
-
-// ManageServerConnection creates a new ServerConnection for Host with the given callback handler.
-func (m *Manager) ManageServerConnection(host string, handler func(string, *protocol.GroupMessage)) {
-	m.lock.Lock()
-
-	_, exists := m.serverConnections[host]
-	if !exists {
-		psc := NewPeerServerConnection(m.acn, host)
-		go psc.Run()
-		psc.GroupMessageHandler = handler
-		m.serverConnections[host] = psc
-	}
-	m.lock.Unlock()
-}
-
-// GetPeers returns a map of all peer connections with their state
-func (m *Manager) GetPeers() map[string]ConnectionState {
-	rm := make(map[string]ConnectionState)
-	m.lock.Lock()
-	for onion, ppc := range m.peerConnections {
-		rm[onion] = ppc.GetState()
-	}
-	m.lock.Unlock()
-	return rm
-}
-
-// GetServers returns a map of all server connections with their state.
-func (m *Manager) GetServers() map[string]ConnectionState {
-	rm := make(map[string]ConnectionState)
-	m.lock.Lock()
-	for onion, psc := range m.serverConnections {
-		rm[onion] = psc.GetState()
-	}
-	m.lock.Unlock()
-	return rm
-}
-
-// GetPeerPeerConnectionForOnion safely returns a given peer connection
-func (m *Manager) GetPeerPeerConnectionForOnion(host string) (ppc *PeerPeerConnection) {
-	m.lock.Lock()
-	ppc = m.peerConnections[host]
-	m.lock.Unlock()
-	return
-}
-
-// GetPeerServerConnectionForOnion safely returns a given host connection
-func (m *Manager) GetPeerServerConnectionForOnion(host string) (psc *PeerServerConnection) {
-	m.lock.Lock()
-	psc = m.serverConnections[host]
-	m.lock.Unlock()
-	return
-}
-
-// AttemptReconnections repeatedly attempts to reconnect with failed peers and servers.
-func (m *Manager) AttemptReconnections() {
-	timeout := time.Duration(0) // first pass right away
-
-	for {
-		select {
-		case <-time.After(timeout):
-			m.lock.Lock()
-			for _, ppc := range m.peerConnections {
-				if ppc.GetState() == FAILED {
-					go ppc.Run()
-				}
-			}
-			m.lock.Unlock()
-
-			m.lock.Lock()
-			for _, psc := range m.serverConnections {
-				if psc.GetState() == FAILED {
-					go psc.Run()
-				}
-			}
-			m.lock.Unlock()
-
-			// Launch Another Run In 30 Seconds
-			timeout = time.Duration(30 * time.Second)
-		case <-m.breakChannel:
-			return
-		}
-	}
-}
-
-// ClosePeerConnection closes an existing peer connection
-func (m *Manager) ClosePeerConnection(onion string) {
-	m.lock.Lock()
-	pc, ok := m.peerConnections[onion]
-	if ok {
-		pc.Close()
-		delete(m.peerConnections, onion)
-	}
-	m.lock.Unlock()
-}
-
-// Shutdown closes all connections under managment (freeing their goroutines)
-func (m *Manager) Shutdown() {
-	m.breakChannel <- true
-	m.lock.Lock()
-	for onion, ppc := range m.peerConnections {
-		ppc.Close()
-		delete(m.peerConnections, onion)
-	}
-	for onion, psc := range m.serverConnections {
-		psc.Close()
-		delete(m.serverConnections, onion)
-	}
-	m.lock.Unlock()
-}

protocol/connections/connectsmanager_test.go

@@ -1,11 +0,0 @@
-package connections
-
-import (
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"testing"
-)
-
-func TestConnectionsManager(t *testing.T) {
-	// TODO We need to encapsulate connections behind a well defined interface for tesintg
-	NewConnectionsManager(connectivity.LocalProvider())
-}

protocol/connections/engine.go

@@ -1,95 +1,183 @@
 package connections
 
 import (
-	"crypto/rsa"
 	"cwtch.im/cwtch/event"
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/peer"
+	"cwtch.im/cwtch/model"
+	"cwtch.im/cwtch/protocol/groups"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/application"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"github.com/golang/protobuf/proto"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/networks/tor"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	torProvider "git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
+	"github.com/gtank/ristretto255"
 	"golang.org/x/crypto/ed25519"
+	"strconv"
 	"sync"
+	"time"
 )
 
+type engine struct {
+	queue event.Queue
+
+	// Engine Attributes
+	identity primitives.Identity
+	acn      connectivity.ACN
+
+	// Authorization list of contacts to authorization status
+	authorizations sync.Map // string(onion) => model.Authorization
+
+	// Block Unknown Contacts
+	blockUnknownContacts bool
+
+	// Pointer to the Global Event Manager
+	eventManager event.Manager
+
+	// Nextgen Tapir Service
+	service tapir.Service
+
+	// Nextgen Tapir Service
+	ephemeralServices sync.Map // string(onion) => tapir.Service
+
+	// Required for listen(), inaccessible from identity
+	privateKey ed25519.PrivateKey
+
+	shuttingDown bool
+}
+
 // Engine (ProtocolEngine) encapsulates the logic necessary to make and receive Cwtch connections.
 // Note: ProtocolEngine doesn't have access to any information necessary to encrypt or decrypt GroupMessages
-type Engine struct {
-	queue              *event.Queue
-	connectionsManager *Manager
-
-	// Engine Attributes
-	Identity identity.Identity
-	ACN      connectivity.ACN
-
-	app *application.RicochetApplication
-
-	// Engine State
-	started bool
-
-	// Blocklist
-	blocked sync.Map
-
-	// Pointer to the Global Event Manager
-	eventManager *event.Manager
-	privateKey   ed25519.PrivateKey
+// Protocol Engine *can* associate Group Identifiers with Group Servers, although we don't currently make use of this fact
+// other than to route errors back to the UI.
+type Engine interface {
+	ACN() connectivity.ACN
+	EventManager() event.Manager
+	Shutdown()
 }
 
 // NewProtocolEngine initializes a new engine that runs Cwtch using the given parameters
-func NewProtocolEngine(privateKey ed25519.PrivateKey, acn connectivity.ACN, eventManager *event.Manager, blockedPeers []string) *Engine {
-	engine := new(Engine)
+func NewProtocolEngine(identity primitives.Identity, privateKey ed25519.PrivateKey, acn connectivity.ACN, eventManager event.Manager, peerAuthorizations map[string]model.Authorization) Engine {
+	engine := new(engine)
+	engine.identity = identity
 	engine.privateKey = privateKey
-	engine.queue = event.NewEventQueue(100)
+	engine.queue = event.NewQueue()
 	go engine.eventHandler()
 
-	engine.ACN = acn
-	engine.connectionsManager = NewConnectionsManager(engine.ACN)
-	go engine.connectionsManager.AttemptReconnections()
+	engine.acn = acn
+
+	// Init the Server running the Simple App.
+	engine.service = new(tor.BaseOnionService)
+	engine.service.Init(acn, privateKey, &identity)
 
 	engine.eventManager = eventManager
 
-	engine.eventManager.Subscribe(event.ProtocolEngineStartListen, engine.queue.EventChannel)
-	engine.eventManager.Subscribe(event.PeerRequest, engine.queue.EventChannel)
-	engine.eventManager.Subscribe(event.InvitePeerToGroup, engine.queue.EventChannel)
-	engine.eventManager.Subscribe(event.JoinServer, engine.queue.EventChannel)
-	engine.eventManager.Subscribe(event.SendMessageToGroup, engine.queue.EventChannel)
-	engine.eventManager.Subscribe(event.SendMessageToPeer, engine.queue.EventChannel)
+	engine.eventManager.Subscribe(event.ProtocolEngineStartListen, engine.queue)
+	engine.eventManager.Subscribe(event.PeerRequest, engine.queue)
+	engine.eventManager.Subscribe(event.RetryPeerRequest, engine.queue)
+	engine.eventManager.Subscribe(event.InvitePeerToGroup, engine.queue)
+	engine.eventManager.Subscribe(event.JoinServer, engine.queue)
+	engine.eventManager.Subscribe(event.LeaveServer, engine.queue)
+	engine.eventManager.Subscribe(event.SendMessageToGroup, engine.queue)
+	engine.eventManager.Subscribe(event.SendMessageToPeer, engine.queue)
+	engine.eventManager.Subscribe(event.SendGetValMessageToPeer, engine.queue)
+	engine.eventManager.Subscribe(event.SendRetValMessageToPeer, engine.queue)
+	engine.eventManager.Subscribe(event.DeleteContact, engine.queue)
+	engine.eventManager.Subscribe(event.DeleteGroup, engine.queue)
 
-	engine.eventManager.Subscribe(event.BlockPeer, engine.queue.EventChannel)
-	for _, peer := range blockedPeers {
-		engine.blocked.Store(peer, true)
+	engine.eventManager.Subscribe(event.SetPeerAuthorization, engine.queue)
+	engine.eventManager.Subscribe(event.BlockUnknownPeers, engine.queue)
+	engine.eventManager.Subscribe(event.AllowUnknownPeers, engine.queue)
+
+	for peer, authorization := range peerAuthorizations {
+		engine.authorizations.Store(peer, authorization)
 	}
 	return engine
 }
 
+func (e *engine) ACN() connectivity.ACN {
+	return e.acn
+}
+
+func (e *engine) EventManager() event.Manager {
+	return e.eventManager
+}
+
 // eventHandler process events from other subsystems
-func (e *Engine) eventHandler() {
+func (e *engine) eventHandler() {
 	for {
 		ev := e.queue.Next()
 		switch ev.EventType {
 		case event.StatusRequest:
 			e.eventManager.Publish(event.Event{EventType: event.ProtocolEngineStatus, EventID: ev.EventID})
 		case event.PeerRequest:
-			e.PeerWithOnion(ev.Data["Onion"])
-		case event.InvitePeerToGroup:
-			e.InviteOnionToGroup(ev.Data["Onion"], []byte(ev.Data["Invite"]))
-		case event.JoinServer:
-			e.JoinServer(ev.Data["Onion"])
-		case event.SendMessageToGroup:
-			e.SendMessageToGroup(ev.Data["Server"], []byte(ev.Data["Ciphertext"]), []byte(ev.Data["Signature"]))
-		case event.SendMessageToPeer:
-			log.Debugf("Sending Message to Peer.....")
-			ppc := e.connectionsManager.GetPeerPeerConnectionForOnion(ev.Data["Peer"])
-			if ppc != nil {
-				// TODO this will block.
-				ppc.SendPacket([]byte(ev.Data["Message"]))
+			if torProvider.IsValidHostname(ev.Data[event.RemotePeer]) {
+				go e.peerWithOnion(ev.Data[event.RemotePeer])
 			}
-		case event.BlockPeer:
-			e.blocked.Store(ev.Data["Peer"], true)
+		case event.RetryPeerRequest:
+			// This event allows engine to treat (automated) retry peering requests differently to user-specified
+			// peer events
+			if torProvider.IsValidHostname(ev.Data[event.RemotePeer]) {
+				log.Debugf("Retrying Peer Request: %v", ev.Data[event.RemotePeer])
+				go e.peerWithOnion(ev.Data[event.RemotePeer])
+			}
+		case event.InvitePeerToGroup:
+			e.sendMessageToPeer(ev.EventID, ev.Data[event.RemotePeer], event.ContextInvite, []byte(ev.Data[event.GroupInvite]))
+		case event.JoinServer:
+			signature, err := base64.StdEncoding.DecodeString(ev.Data[event.Signature])
+			if err != nil {
+				// will result in a full sync
+				signature = []byte{}
+			}
+			go e.peerWithTokenServer(ev.Data[event.GroupServer], ev.Data[event.ServerTokenOnion], ev.Data[event.ServerTokenY], signature)
+		case event.LeaveServer:
+			e.leaveServer(ev.Data[event.GroupServer])
+		case event.DeleteContact:
+			onion := ev.Data[event.RemotePeer]
+			// We remove this peer from out blocklist which will prevent them from contacting us if we have "block unknown peers" turned on.
+			e.authorizations.Delete(ev.Data[event.RemotePeer])
+			e.deleteConnection(onion)
+		case event.DeleteGroup:
+			// TODO: There isn't a way here to determine if other Groups are using a server connection...
+		case event.SendMessageToGroup:
+			ciphertext, _ := base64.StdEncoding.DecodeString(ev.Data[event.Ciphertext])
+			signature, _ := base64.StdEncoding.DecodeString(ev.Data[event.Signature])
+			go e.sendMessageToGroup(ev.Data[event.GroupID], ev.Data[event.GroupServer], ciphertext, signature)
+		case event.SendMessageToPeer:
+			// TODO: remove this passthrough once the UI is integrated.
+			context, ok := ev.Data[event.EventContext]
+			if !ok {
+				context = event.ContextRaw
+			}
+			err := e.sendMessageToPeer(ev.EventID, ev.Data[event.RemotePeer], context, []byte(ev.Data[event.Data]))
+			if err != nil {
+				e.eventManager.Publish(event.NewEvent(event.SendMessageToPeerError, map[event.Field]string{event.RemotePeer: ev.Data[event.RemotePeer], event.EventID: ev.EventID, event.Error: "peer is offline or the connection has yet to finalize"}))
+			}
+		case event.SendGetValMessageToPeer:
+			e.sendGetValToPeer(ev.EventID, ev.Data[event.RemotePeer], ev.Data[event.Scope], ev.Data[event.Path])
+		case event.SendRetValMessageToPeer:
+			e.sendRetValToPeer(ev.EventID, ev.Data[event.RemotePeer], ev.Data[event.Data], ev.Data[event.Exists])
+		case event.SetPeerAuthorization:
+			auth := model.Authorization(ev.Data[event.Authorization])
+			e.authorizations.Store(ev.Data[event.RemotePeer], auth)
+			if auth == model.AuthBlocked {
+				connection, err := e.service.GetConnection(ev.Data[event.RemotePeer])
+				if connection != nil && err == nil {
+					connection.Close()
+				}
+				// Explicitly send a disconnected event (if we don't do this here then the UI can wait for a while before
+				// an ongoing Open() connection fails and so the user will see a blocked peer as still connecting (because
+				// there isn't an active connection and we are stuck waiting for tor to time out)
+				e.peerDisconnected(ev.Data[event.RemotePeer])
+			}
+		case event.AllowUnknownPeers:
+			log.Debugf("%v now allows unknown connections", e.identity.Hostname())
+			e.blockUnknownContacts = false
+		case event.BlockUnknownPeers:
+			log.Debugf("%v now forbids unknown connections", e.identity.Hostname())
+			e.blockUnknownContacts = true
 		case event.ProtocolEngineStartListen:
 			go e.listenFn()
 		default:
@@ -98,167 +186,330 @@ func (e *Engine) eventHandler() {
 	}
 }
 
-// GetPeerHandler is an external interface function that allows callers access to a CwtchPeerHandler
-// TODO: There is likely a slightly better way to encapsulate this behavior
-func (e *Engine) GetPeerHandler(remotePeerHostname string) *CwtchPeerHandler {
-	return &CwtchPeerHandler{Onion: remotePeerHostname, EventBus: e.eventManager}
+func (e *engine) isBlocked(onion string) bool {
+	authorization, known := e.authorizations.Load(onion)
+	if !known {
+		// if we block unknown peers we will block this contact
+		return e.blockUnknownContacts
+	}
+	return authorization.(model.Authorization) == model.AuthBlocked
+}
+
+func (e *engine) isAllowed(onion string) bool {
+	authorization, known := e.authorizations.Load(onion)
+	if !known {
+		log.Errorf("attempted to lookup authorization of onion not in map...that should never happen")
+		return false
+	}
+	if e.blockUnknownContacts {
+		return authorization.(model.Authorization) == model.AuthApproved
+	}
+	return authorization.(model.Authorization) != model.AuthBlocked
+}
+
+func (e *engine) createPeerTemplate() *PeerApp {
+	peerAppTemplate := new(PeerApp)
+	peerAppTemplate.IsBlocked = e.isBlocked
+	peerAppTemplate.IsAllowed = e.isAllowed
+	peerAppTemplate.MessageHandler = e.handlePeerMessage
+	peerAppTemplate.OnAcknowledgement = e.ignoreOnShutdown2(e.peerAck)
+	peerAppTemplate.OnAuth = e.ignoreOnShutdown(e.peerAuthed)
+	peerAppTemplate.OnConnecting = e.ignoreOnShutdown(e.peerConnecting)
+	peerAppTemplate.OnClose = e.ignoreOnShutdown(e.peerDisconnected)
+	peerAppTemplate.RetValHandler = e.handlePeerRetVal
+	return peerAppTemplate
 }
 
 // Listen sets up an onion listener to process incoming cwtch messages
-func (e *Engine) listenFn() {
-	ra := new(application.RicochetApplication)
-	onionService, err := e.ACN.Listen(e.privateKey, application.RicochetPort)
-	if err != nil /*&& fmt.Sprintf("%v", err) != "550 Unspecified Tor error: Onion address collision"*/ {
-		e.eventManager.Publish(event.NewEvent(event.ProtocolEngineStopped, map[string]string{"Identity": e.Identity.Hostname(), "Error": err.Error()}))
-		return
+func (e *engine) listenFn() {
+	err := e.service.Listen(e.createPeerTemplate())
+	if !e.shuttingDown {
+		e.eventManager.Publish(event.NewEvent(event.ProtocolEngineStopped, map[event.Field]string{event.Identity: e.identity.Hostname(), event.Error: err.Error()}))
 	}
-
-	af := application.ApplicationInstanceFactory{}
-	af.Init()
-	af.AddHandler("im.cwtch.peer", func(rai *application.ApplicationInstance) func() channels.Handler {
-		cpi := new(CwtchPeerInstance)
-		cpi.Init(rai, ra)
-		return func() channels.Handler {
-			cpc := new(peer.CwtchPeerChannel)
-			cpc.Handler = e.GetPeerHandler(rai.RemoteHostname)
-			return cpc
-		}
-	})
-
-	af.AddHandler("im.cwtch.peer.data", func(rai *application.ApplicationInstance) func() channels.Handler {
-		cpi := new(CwtchPeerInstance)
-		cpi.Init(rai, ra)
-		return func() channels.Handler {
-			cpc := new(peer.CwtchPeerDataChannel)
-			cpc.Handler = e.GetPeerHandler(rai.RemoteHostname)
-			return cpc
-		}
-	})
-
-	ra.Init(e.ACN, e.Identity.Name, e.Identity, af, e)
-	log.Infof("Running cwtch peer on %v", onionService.AddressFull())
-	e.started = true
-	e.app = ra
-	ra.Run(onionService)
-	e.eventManager.Publish(event.NewEvent(event.ProtocolEngineStopped, map[string]string{"Identity": e.Identity.Hostname()}))
-	return
-}
-
-// LookupContact is a V2 API Call, we want to reject all V2 Peers
-// TODO Deprecate
-func (e *Engine) LookupContact(hostname string, publicKey rsa.PublicKey) (allowed, known bool) {
-	return false, false
-}
-
-// ContactRequest is a V2 API Call needed to implement ContactRequestHandler Interface
-// TODO Deprecate
-func (e *Engine) ContactRequest(name string, message string) string {
-	return "Rejected"
-}
-
-// LookupContactV3 returns that a contact is known and allowed to communicate for all cases.
-func (e *Engine) LookupContactV3(hostname string, publicKey ed25519.PublicKey) (allowed, known bool) {
-	// TODO: We want to autoblock those that are blocked, The known parameter has no use anymore and should be
-	// disregarded by peers, so we set it to false.
-	if _, blocked := e.blocked.Load(hostname); blocked {
-		return false, false
-	}
-	return true, false
 }
 
 // Shutdown tears down the eventHandler goroutine
-func (e *Engine) Shutdown() {
-	e.connectionsManager.Shutdown()
-	e.app.Shutdown()
+func (e *engine) Shutdown() {
+	e.shuttingDown = true
+	e.service.Shutdown()
 	e.queue.Shutdown()
 }
 
-// PeerWithOnion is the entry point for cwtchPeer relationships
-func (e *Engine) PeerWithOnion(onion string) *PeerPeerConnection {
-	return e.connectionsManager.ManagePeerConnection(onion, e)
+// peerWithOnion is the entry point for cwtchPeer relationships
+// needs to be run in a goroutine as will block on Open.
+func (e *engine) peerWithOnion(onion string) {
+	log.Debugf("Called PeerWithOnion for %v", onion)
+	if !e.isBlocked(onion) {
+		e.ignoreOnShutdown(e.peerConnecting)(onion)
+		connected, err := e.service.Connect(onion, e.createPeerTemplate())
+
+		// If we are already connected...check if we are authed and issue an auth event
+		// (This allows the ui to be stateless)
+		if connected && err != nil {
+			conn, err := e.service.GetConnection(onion)
+			if err == nil {
+				if conn.HasCapability(cwtchCapability) {
+					e.ignoreOnShutdown(e.peerAuthed)(onion)
+					return
+				}
+			}
+		}
+
+		// Only issue a disconnected error if we are disconnected (Connect will fail if a connection already exists)
+		if !connected && err != nil {
+			e.ignoreOnShutdown(e.peerDisconnected)(onion)
+		}
+	}
 }
 
-// InviteOnionToGroup kicks off the invite process
-func (e *Engine) InviteOnionToGroup(onion string, invite []byte) error {
-	ppc := e.connectionsManager.GetPeerPeerConnectionForOnion(onion)
-	if ppc == nil {
-		return errors.New("peer connection not setup for onion. peers must be trusted before sending")
+// peerWithTokenServer is the entry point for cwtchPeer - server relationships
+// needs to be run in a goroutine as will block on Open.
+func (e *engine) peerWithTokenServer(onion string, tokenServerOnion string, tokenServerY string, lastKnownSignature []byte) {
+
+	service, exists := e.ephemeralServices.Load(onion)
+	if exists {
+		connection := service.(*tor.BaseOnionService)
+		if conn, err := connection.GetConnection(onion); err == nil {
+			// We are already peered and synced so return...
+			// This will only not-trigger it lastKnownSignature has been wiped, which only happens when ResyncServer is called
+			// in CwtchPeer.
+			if !conn.IsClosed() && len(lastKnownSignature) != 0 {
+				return
+			}
+			// Otherwise...we are going to rebuild the connection(which will result in a bandwidth heavy resync)...
+			e.leaveServer(onion)
+		}
+		// Otherwise...let's reconnect
 	}
-	if ppc.GetState() == AUTHENTICATED {
-		log.Infof("Got connection for group: %v - Sending Invite\n", ppc)
-		ppc.SendGroupInvite(invite)
-	} else {
-		return errors.New("cannot send invite to onion: peer connection is not ready")
+
+	log.Debugf("Peering with Token Server %v %v", onion, tokenServerOnion)
+	e.ignoreOnShutdown(e.serverConnecting)(onion)
+	// Create a new ephemeral service for this connection
+	ephemeralService := new(tor.BaseOnionService)
+	eid, epk := primitives.InitializeEphemeralIdentity()
+	ephemeralService.Init(e.acn, epk, &eid)
+
+	Y := ristretto255.NewElement()
+	Y.UnmarshalText([]byte(tokenServerY))
+	connected, err := ephemeralService.Connect(onion, NewTokenBoardClient(e.acn, Y, tokenServerOnion, lastKnownSignature, e.receiveGroupMessage, e.serverSynced, e.serverDisconnected))
+	e.ephemeralServices.Store(onion, ephemeralService)
+	// If we are already connected...check if we are authed and issue an auth event
+	// (This allows the ui to be stateless)
+	if connected && err != nil {
+		conn, err := ephemeralService.GetConnection(onion)
+		if err == nil {
+			if conn.HasCapability(groups.CwtchServerSyncedCapability) {
+				e.ignoreOnShutdown(e.serverConnected)(onion)
+				return
+			}
+		}
+	}
+
+	// Only issue a disconnected error if we are disconnected (Connect will fail if a connection already exists)
+	if !connected && err != nil {
+		e.ignoreOnShutdown(e.serverDisconnected)(onion)
 	}
-	return nil
 }
 
-// ReceiveGroupMessage is a callback function that processes GroupMessages from a given server
-func (e *Engine) ReceiveGroupMessage(server string, gm *protocol.GroupMessage) {
-	// Publish Event so that a Profile Engine can deal with it.
-	// Note: This technically means that *multiple* Profile Engines could listen to the same ProtocolEngine!
-	e.eventManager.Publish(event.NewEvent(event.EncryptedGroupMessage, map[string]string{"Ciphertext": string(gm.GetCiphertext()), "Signature": string(gm.GetSignature())}))
+func (e *engine) ignoreOnShutdown(f func(string)) func(string) {
+	return func(x string) {
+		if !e.shuttingDown {
+			f(x)
+		}
+	}
 }
 
-// JoinServer manages a new server connection with the given onion address
-func (e *Engine) JoinServer(onion string) {
-	e.connectionsManager.ManageServerConnection(onion, e.ReceiveGroupMessage)
+func (e *engine) ignoreOnShutdown2(f func(string, string)) func(string, string) {
+	return func(x, y string) {
+		if !e.shuttingDown {
+			f(x, y)
+		}
+	}
 }
 
-// SendMessageToGroup attemps to sent the given message to the given group id.
-func (e *Engine) SendMessageToGroup(server string, ct []byte, sig []byte) error {
-	psc := e.connectionsManager.GetPeerServerConnectionForOnion(server)
-	if psc == nil {
-		return errors.New("could not find server connection to send message to")
+func (e *engine) peerAuthed(onion string) {
+	_, known := e.authorizations.Load(onion)
+	if !known {
+		e.authorizations.Store(onion, model.AuthUnknown)
 	}
-	gm := &protocol.GroupMessage{
-		Ciphertext: ct,
-		Signature:  sig,
+	e.eventManager.Publish(event.NewEvent(event.PeerStateChange, map[event.Field]string{
+		event.RemotePeer:      string(onion),
+		event.ConnectionState: ConnectionStateName[AUTHENTICATED],
+	}))
+}
+
+func (e *engine) peerConnecting(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.PeerStateChange, map[event.Field]string{
+		event.RemotePeer:      string(onion),
+		event.ConnectionState: ConnectionStateName[CONNECTING],
+	}))
+}
+
+func (e *engine) serverConnecting(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.ServerStateChange, map[event.Field]string{
+		event.GroupServer:     string(onion),
+		event.ConnectionState: ConnectionStateName[CONNECTING],
+	}))
+}
+
+func (e *engine) serverConnected(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.ServerStateChange, map[event.Field]string{
+		event.GroupServer:     onion,
+		event.ConnectionState: ConnectionStateName[CONNECTED],
+	}))
+}
+
+func (e *engine) serverSynced(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.ServerStateChange, map[event.Field]string{
+		event.GroupServer:     onion,
+		event.ConnectionState: ConnectionStateName[SYNCED],
+	}))
+}
+
+func (e *engine) serverDisconnected(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.ServerStateChange, map[event.Field]string{
+		event.GroupServer:     onion,
+		event.ConnectionState: ConnectionStateName[DISCONNECTED],
+	}))
+}
+
+func (e *engine) peerAck(onion string, eventID string) {
+	e.eventManager.Publish(event.NewEvent(event.PeerAcknowledgement, map[event.Field]string{
+		event.EventID:    eventID,
+		event.RemotePeer: onion,
+	}))
+}
+
+func (e *engine) peerDisconnected(onion string) {
+	e.eventManager.Publish(event.NewEvent(event.PeerStateChange, map[event.Field]string{
+		event.RemotePeer:      string(onion),
+		event.ConnectionState: ConnectionStateName[DISCONNECTED],
+	}))
+}
+
+// sendMessageToPeer sends a message to a peer under a given context
+func (e *engine) sendMessageToPeer(eventID string, onion string, context string, message []byte) error {
+	conn, err := e.service.WaitForCapabilityOrClose(onion, cwtchCapability)
+	if err == nil {
+		peerApp, ok := (conn.App()).(*PeerApp)
+		if ok {
+			peerApp.SendMessage(PeerMessage{eventID, context, message})
+			return nil
+		}
+		return errors.New("failed type assertion conn.App != PeerApp")
 	}
-	err := psc.SendGroupMessage(gm)
 	return err
 }
 
-// GetPeers returns a list of peer connections.
-func (e *Engine) GetPeers() map[string]ConnectionState {
-	return e.connectionsManager.GetPeers()
+func (e *engine) sendGetValToPeer(eventID, onion, scope, path string) error {
+	log.Debugf("sendGetValMessage to peer %v %v%v\n", onion, scope, path)
+	getVal := peerGetVal{Scope: scope, Path: path}
+	message, err := json.Marshal(getVal)
+	if err != nil {
+		return err
+	}
+	return e.sendMessageToPeer(eventID, onion, event.ContextGetVal, message)
 }
 
-// GetServers returns a list of server connections
-func (e *Engine) GetServers() map[string]ConnectionState {
-	return e.connectionsManager.GetServers()
+func (e *engine) sendRetValToPeer(eventID, onion, val, existsStr string) error {
+	log.Debugf("sendRetValMessage to peer %v (%v) %v %v\n", onion, eventID, val, existsStr)
+	exists, _ := strconv.ParseBool(existsStr)
+	retVal := peerRetVal{Val: val, Exists: exists}
+	message, err := json.Marshal(retVal)
+	if err != nil {
+		return err
+	}
+	return e.sendMessageToPeer(eventID, onion, event.ContextRetVal, message)
 }
 
-// CwtchPeerInstance encapsulates incoming peer connections
-type CwtchPeerInstance struct {
-	rai *application.ApplicationInstance
-	ra  *application.RicochetApplication
-}
-
-// Init sets up a CwtchPeerInstance
-func (cpi *CwtchPeerInstance) Init(rai *application.ApplicationInstance, ra *application.RicochetApplication) {
-	cpi.rai = rai
-	cpi.ra = ra
-}
-
-// CwtchPeerHandler encapsulates handling of incoming CwtchPackets
-type CwtchPeerHandler struct {
-	Onion       string
-	EventBus    *event.Manager
-	DataHandler func(string, []byte) []byte
-}
-
-// HandleGroupInvite handles incoming GroupInvites
-func (cph *CwtchPeerHandler) HandleGroupInvite(gci *protocol.GroupChatInvite) {
-	log.Debugf("Received GroupID from %v %v\n", cph.Onion, gci.String())
-	marshal, err := proto.Marshal(gci)
+func (e *engine) deleteConnection(id string) {
+	conn, err := e.service.GetConnection(id)
 	if err == nil {
-		cph.EventBus.Publish(event.NewEvent(event.NewGroupInvite, map[string]string{"Onion": cph.Onion, "GroupInvite": string(marshal)}))
+		conn.Close()
 	}
 }
 
-// HandlePacket handles the Cwtch cwtchPeer Data Channel
-func (cph *CwtchPeerHandler) HandlePacket(data []byte) []byte {
-	cph.EventBus.Publish(event.NewEvent(event.NewMessageFromPeer, map[string]string{"Onion": cph.Onion, "Data": string(data)}))
-	return []byte{} // TODO remove this
+// receiveGroupMessage is a callback function that processes GroupMessages from a given server
+func (e *engine) receiveGroupMessage(server string, gm *groups.EncryptedGroupMessage) {
+	// Publish Event so that a Profile Engine can deal with it.
+	// Note: This technically means that *multiple* Profile Engines could listen to the same ProtocolEngine!
+	e.eventManager.Publish(event.NewEvent(event.EncryptedGroupMessage, map[event.Field]string{event.GroupServer: server, event.Ciphertext: base64.StdEncoding.EncodeToString(gm.Ciphertext), event.Signature: base64.StdEncoding.EncodeToString(gm.Signature)}))
+}
+
+// sendMessageToGroup attempts to sent the given message to the given group id.
+func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, sig []byte) {
+
+	es, ok := e.ephemeralServices.Load(server)
+	if es == nil || !ok {
+		e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: "server-not-found", event.Signature: base64.StdEncoding.EncodeToString(sig)}))
+		return
+	}
+	ephemeralService := es.(tapir.Service)
+
+	conn, err := ephemeralService.WaitForCapabilityOrClose(server, groups.CwtchServerSyncedCapability)
+	if err == nil {
+		tokenApp, ok := (conn.App()).(*TokenBoardClient)
+		if ok {
+			if spent, numtokens := tokenApp.Post(ct, sig); !spent {
+				// TODO: while this works for the spam guard, it won't work for other forms of payment...
+				// Make an -inline- payment, this will hold the goroutine
+				if err := tokenApp.MakePayment(); err == nil {
+					// This really shouldn't fail since we now know we have the required tokens...
+					if spent, _ := tokenApp.Post(ct, sig); !spent {
+						e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: err.Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)}))
+					}
+				} else {
+					// Broadast the token error
+					e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: err.Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)}))
+				}
+			} else if numtokens < 5 {
+				go tokenApp.MakePayment()
+			}
+			// regardless we return....
+			return
+		}
+	}
+	e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: "server-connection-not-valid", event.Signature: base64.StdEncoding.EncodeToString(sig)}))
+}
+
+func (e *engine) handlePeerMessage(hostname string, eventID string, context string, message []byte) {
+	log.Debugf("New message from peer: %v %v", hostname, context)
+	if context == event.ContextGetVal {
+		var getVal peerGetVal
+		err := json.Unmarshal(message, &getVal)
+		if err == nil {
+			ev := event.NewEventList(event.NewGetValMessageFromPeer, event.RemotePeer, hostname, event.Scope, getVal.Scope, event.Path, getVal.Path)
+			ev.EventID = eventID
+			e.eventManager.Publish(ev)
+		}
+	} else {
+		e.eventManager.Publish(event.NewEvent(event.NewMessageFromPeer, map[event.Field]string{event.TimestampReceived: time.Now().Format(time.RFC3339Nano), event.RemotePeer: hostname, event.Data: string(message)}))
+	}
+}
+
+func (e *engine) handlePeerRetVal(hostname string, getValData, retValData []byte) {
+	var getVal peerGetVal
+	var retVal peerRetVal
+
+	err := json.Unmarshal(getValData, &getVal)
+	if err != nil {
+		log.Errorf("Unmarshalling our own getVal request: %v\n", err)
+		return
+	}
+	err = json.Unmarshal(retValData, &retVal)
+	if err != nil {
+		log.Errorf("Unmarshalling peer response to getVal request")
+		return
+	}
+
+	e.eventManager.Publish(event.NewEventList(event.NewRetValMessageFromPeer, event.RemotePeer, hostname, event.Scope, getVal.Scope, event.Path, getVal.Path, event.Exists, strconv.FormatBool(retVal.Exists), event.Data, retVal.Val))
+}
+
+func (e *engine) leaveServer(server string) {
+	es, ok := e.ephemeralServices.Load(server)
+	if ok {
+		ephemeralService := es.(tapir.Service)
+		ephemeralService.Shutdown()
+		e.ephemeralServices.Delete(server)
+	}
 }

protocol/connections/fetch/peer_fetch_channel.go

@@ -1,103 +0,0 @@
-package fetch
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchPeerFetchChannel is the peer implementation of the im.cwtch.server.fetch
-// channel.
-type CwtchPeerFetchChannel struct {
-	channel *channels.Channel
-	Handler CwtchPeerFetchChannelHandler
-}
-
-// CwtchPeerFetchChannelHandler should be implemented by peers to receive new messages.
-type CwtchPeerFetchChannelHandler interface {
-	HandleGroupMessage(*protocol.GroupMessage)
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.server.fetch)
-func (cpfc *CwtchPeerFetchChannel) Type() string {
-	return "im.cwtch.server.fetch"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cpfc *CwtchPeerFetchChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch server channels only client can open
-func (cpfc *CwtchPeerFetchChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cpfc *CwtchPeerFetchChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cpfc *CwtchPeerFetchChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch server channels require no auth.
-func (cpfc *CwtchPeerFetchChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound - cwtch server peer implementations shouldnever respond to inbound requests
-func (cpfc *CwtchPeerFetchChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	return nil, errors.New("client does not receive inbound listen channels")
-}
-
-// OpenOutbound sets up a new cwtch fetch channel
-func (cpfc *CwtchPeerFetchChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	cpfc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.OpenChannel(channel.ID, cpfc.Type()), nil
-}
-
-// OpenOutboundResult confirms a previous open channel request
-func (cpfc *CwtchPeerFetchChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	if err == nil {
-		if crm.GetOpened() {
-			cpfc.channel.Pending = false
-			cpfc.FetchRequest()
-		}
-	}
-}
-
-// FetchRequest sends a FetchMessage to the Server.
-func (cpfc *CwtchPeerFetchChannel) FetchRequest() error {
-	if cpfc.channel.Pending == false {
-		fm := &protocol.FetchMessage{}
-		csp := &protocol.CwtchServerPacket{
-			FetchMessage: fm,
-		}
-		packet, _ := proto.Marshal(csp)
-		cpfc.channel.SendMessage(packet)
-	} else {
-		return errors.New("channel isn't set up yet")
-	}
-	return nil
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cpfc *CwtchPeerFetchChannel) Packet(data []byte) {
-	csp := &protocol.CwtchServerPacket{}
-	err := proto.Unmarshal(data, csp)
-	if err == nil {
-		if csp.GetGroupMessage() != nil {
-			gm := csp.GetGroupMessage()
-			// We create a new go routine here to avoid leaking any information about processing time
-			// TODO Server can probably try to use this to DoS a peer
-			go cpfc.Handler.HandleGroupMessage(gm)
-		}
-	}
-}

protocol/connections/fetch/peer_fetch_channel_test.go

@@ -1,92 +0,0 @@
-package fetch
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-	"time"
-)
-
-type TestHandler struct {
-	Received bool
-}
-
-func (th *TestHandler) HandleGroupMessage(m *protocol.GroupMessage) {
-	th.Received = true
-}
-
-func TestPeerFetchChannelAttributes(t *testing.T) {
-	cssc := new(CwtchPeerFetchChannel)
-	if cssc.Type() != "im.cwtch.server.fetch" {
-		t.Errorf("cwtch channel type is incorrect %v", cssc.Type())
-	}
-
-	if !cssc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.Fetch channel")
-	}
-
-	if cssc.Bidirectional() {
-		t.Errorf("im.cwtch.server.fetch should not be bidirectional")
-	}
-
-	if !cssc.Singleton() {
-		t.Errorf("im.cwtch.server.fetch should be a Singleton")
-	}
-
-	if cssc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cssc.RequiresAuthentication())
-	}
-
-}
-func TestPeerFetchChannelOpenInbound(t *testing.T) {
-	cssc := new(CwtchPeerFetchChannel)
-	channel := new(channels.Channel)
-	_, err := cssc.OpenInbound(channel, nil)
-	if err == nil {
-		t.Errorf("client implementation of im.cwtch.server.Fetch should never open an inbound channel")
-	}
-}
-
-func TestPeerFetchChannel(t *testing.T) {
-	pfc := new(CwtchPeerFetchChannel)
-	th := new(TestHandler)
-	pfc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 3
-	channel.SendMessage = func([]byte) {}
-	channel.CloseChannel = func() {}
-	result, err := pfc.OpenOutbound(channel)
-	if err != nil {
-		t.Errorf("expected result but also got non-nil error: result:%v, err: %v", result, err)
-	}
-
-	cr := &Protocol_Data_Control.ChannelResult{
-		ChannelIdentifier: proto.Int32(3),
-		Opened:            proto.Bool(true),
-	}
-
-	pfc.OpenOutboundResult(nil, cr)
-	if channel.Pending {
-		t.Errorf("once opened channel should no longer be pending")
-	}
-
-	csp := &protocol.CwtchServerPacket{
-		GroupMessage: &protocol.GroupMessage{
-			Ciphertext: []byte("hello"), Signature: []byte{}, Spamguard: []byte{},
-		},
-	}
-	packet, _ := proto.Marshal(csp)
-
-	pfc.Packet(packet)
-
-	time.Sleep(time.Second * 2)
-
-	if th.Received != true {
-		t.Errorf("group message should not have been received")
-	}
-
-	pfc.Closed(nil)
-
-}

protocol/connections/listen/peer_listen_channel.go

@@ -1,84 +0,0 @@
-package listen
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchPeerListenChannel is the peer implementation of im.cwtch.server.listen
-type CwtchPeerListenChannel struct {
-	channel *channels.Channel
-	Handler CwtchPeerSendChannelHandler
-}
-
-// CwtchPeerSendChannelHandler is implemented by peers who want to listen to new messages
-type CwtchPeerSendChannelHandler interface {
-	HandleGroupMessage(*protocol.GroupMessage)
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.server.listen".
-func (cplc *CwtchPeerListenChannel) Type() string {
-	return "im.cwtch.server.listen"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cplc *CwtchPeerListenChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch server channels can only be opened by peers
-func (cplc *CwtchPeerListenChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cplc *CwtchPeerListenChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cplc *CwtchPeerListenChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require no auth channels
-func (cplc *CwtchPeerListenChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound - peers should never respond to open inbound requests from servers
-func (cplc *CwtchPeerListenChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	return nil, errors.New("client does not receive inbound listen channels")
-}
-
-// OpenOutbound sets up a new server listen channel
-func (cplc *CwtchPeerListenChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	cplc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.OpenChannel(channel.ID, cplc.Type()), nil
-}
-
-// OpenOutboundResult confirms a previous open channel request
-func (cplc *CwtchPeerListenChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	if err == nil {
-		if crm.GetOpened() {
-			cplc.channel.Pending = false
-		}
-	}
-}
-
-// Packet is called for each server packet received on this channel.
-func (cplc *CwtchPeerListenChannel) Packet(data []byte) {
-	csp := &protocol.CwtchServerPacket{}
-	err := proto.Unmarshal(data, csp)
-	if err == nil {
-		if csp.GetGroupMessage() != nil {
-			gm := csp.GetGroupMessage()
-			cplc.Handler.HandleGroupMessage(gm)
-		}
-	}
-}

protocol/connections/listen/peer_listen_channel_test.go

@@ -1,89 +0,0 @@
-package listen
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-	"time"
-)
-
-type TestHandler struct {
-	Received bool
-}
-
-func (th *TestHandler) HandleGroupMessage(m *protocol.GroupMessage) {
-	th.Received = true
-}
-
-func TestPeerListenChannelAttributes(t *testing.T) {
-	cssc := new(CwtchPeerListenChannel)
-	if cssc.Type() != "im.cwtch.server.listen" {
-		t.Errorf("cwtch channel type is incorrect %v", cssc.Type())
-	}
-
-	if !cssc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.listen channel")
-	}
-
-	if cssc.Bidirectional() {
-		t.Errorf("im.cwtch.server.listen should not be bidirectional")
-	}
-
-	if !cssc.Singleton() {
-		t.Errorf("im.cwtch.server.listen should be a Singleton")
-	}
-
-	if cssc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cssc.RequiresAuthentication())
-	}
-
-}
-func TestPeerListenChannelOpenInbound(t *testing.T) {
-	cssc := new(CwtchPeerListenChannel)
-	channel := new(channels.Channel)
-	_, err := cssc.OpenInbound(channel, nil)
-	if err == nil {
-		t.Errorf("client implementation of im.cwtch.server.Listen should never open an inbound channel")
-	}
-}
-
-func TestPeerListenChannel(t *testing.T) {
-	pfc := new(CwtchPeerListenChannel)
-	th := new(TestHandler)
-	pfc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 3
-	result, err := pfc.OpenOutbound(channel)
-	if err != nil {
-		t.Errorf("expected result but also got non-nil error: result:%v, err: %v", result, err)
-	}
-
-	cr := &Protocol_Data_Control.ChannelResult{
-		ChannelIdentifier: proto.Int32(3),
-		Opened:            proto.Bool(true),
-	}
-
-	pfc.OpenOutboundResult(nil, cr)
-	if channel.Pending {
-		t.Errorf("once opened channel should no longer be pending")
-	}
-
-	csp := &protocol.CwtchServerPacket{
-		GroupMessage: &protocol.GroupMessage{Ciphertext: []byte("hello"), Signature: []byte{}, Spamguard: []byte{}},
-	}
-	packet, _ := proto.Marshal(csp)
-
-	pfc.Packet(packet)
-
-	// Wait for goroutine to run
-	time.Sleep(time.Second * 1)
-
-	if !th.Received {
-		t.Errorf("group message should have been received")
-	}
-
-	pfc.Closed(nil)
-
-}

protocol/connections/peer/peer_channel.go

@@ -1,106 +0,0 @@
-package peer
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchPeerChannel implements the ChannelHandler interface for a channel of
-// type "im.ricochet.Cwtch". The channel may be inbound or outbound.
-//
-// CwtchPeerChannel implements protocol-level sanity and state validation, but
-// does not handle or acknowledge Cwtch messages. The application must provide
-// a CwtchPeerChannelHandler implementation to handle Cwtch events.
-type CwtchPeerChannel struct {
-	// Methods of Handler are called for Cwtch events on this channel
-	Handler CwtchPeerChannelHandler
-	channel *channels.Channel
-}
-
-// CwtchPeerChannelHandler is implemented by an application type to receive
-// events from a CwtchPeerChannel.
-type CwtchPeerChannelHandler interface {
-	HandleGroupInvite(*protocol.GroupChatInvite)
-}
-
-// SendMessage sends a raw message on this channel
-func (cpc *CwtchPeerChannel) SendMessage(data []byte) {
-	cpc.channel.SendMessage(data)
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.Cwtch".
-func (cpc *CwtchPeerChannel) Type() string {
-	return "im.cwtch.peer"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cpc *CwtchPeerChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch channels any side can open
-func (cpc *CwtchPeerChannel) OnlyClientCanOpen() bool {
-	return false
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cpc *CwtchPeerChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cpc *CwtchPeerChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require hidden service auth
-func (cpc *CwtchPeerChannel) RequiresAuthentication() string {
-	return "im.ricochet.auth.3dh"
-}
-
-// OpenInbound is the first method called for an inbound channel request.
-// If an error is returned, the channel is rejected. If a RawMessage is
-// returned, it will be sent as the ChannelResult message.
-func (cpc *CwtchPeerChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	cpc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.AckOpenChannel(channel.ID), nil
-}
-
-// OpenOutbound is the first method called for an outbound channel request.
-// If an error is returned, the channel is not opened. If a RawMessage is
-// returned, it will be sent as the OpenChannel message.
-func (cpc *CwtchPeerChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	cpc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.OpenChannel(channel.ID, cpc.Type()), nil
-}
-
-// OpenOutboundResult is called when a response is received for an
-// outbound OpenChannel request. If `err` is non-nil, the channel was
-// rejected and Closed will be called immediately afterwards. `raw`
-// contains the raw protocol message including any extension data.
-func (cpc *CwtchPeerChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	if err == nil {
-		if crm.GetOpened() {
-			cpc.channel.Pending = false
-		}
-	}
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cpc *CwtchPeerChannel) Packet(data []byte) {
-	cpp := &protocol.CwtchPeerPacket{}
-	err := proto.Unmarshal(data, cpp)
-	if err == nil {
-		if cpp.GetGroupChatInvite() != nil {
-			cpc.Handler.HandleGroupInvite(cpp.GetGroupChatInvite())
-		}
-	} else {
-		log.Errorf("Error Receivng Packet %v\n", err)
-	}
-}

protocol/connections/peer/peer_channel_test.go

@@ -1,102 +0,0 @@
-package peer
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-func TestPeerChannelAttributes(t *testing.T) {
-	cssc := new(CwtchPeerChannel)
-	if cssc.Type() != "im.cwtch.peer" {
-		t.Errorf("cwtch channel type is incorrect %v", cssc.Type())
-	}
-
-	if cssc.OnlyClientCanOpen() {
-		t.Errorf("either side should be able to open im.cwtch.peer channel")
-	}
-
-	if cssc.Bidirectional() {
-		t.Errorf("im.cwtch.peer should not be bidirectional")
-	}
-
-	if !cssc.Singleton() {
-		t.Errorf("im.cwtch.server.listen should be a Singleton")
-	}
-
-	if cssc.RequiresAuthentication() != "im.ricochet.auth.3dh" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cssc.RequiresAuthentication())
-	}
-}
-
-type TestHandler struct {
-	Received            bool
-	ReceviedGroupInvite bool
-}
-
-func (th *TestHandler) ClientIdentity(ci *protocol.CwtchIdentity) {
-	if ci.GetName() == "hello" {
-		th.Received = true
-	}
-}
-
-func (th *TestHandler) HandleGroupInvite(ci *protocol.GroupChatInvite) {
-	///if ci.GetName() == "hello" {
-	th.ReceviedGroupInvite = true
-	//}
-}
-
-func (th *TestHandler) GetClientIdentityPacket() []byte {
-	return nil
-}
-
-func TestPeerChannel(t *testing.T) {
-	th := new(TestHandler)
-	cpc := new(CwtchPeerChannel)
-	cpc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 3
-	result, err := cpc.OpenOutbound(channel)
-	if err != nil {
-		t.Errorf("should have send open channel request instead %v, %v", result, err)
-	}
-
-	cpc2 := new(CwtchPeerChannel)
-	channel2 := new(channels.Channel)
-	channel2.ID = 3
-	sent := false
-	channel2.SendMessage = func(message []byte) {
-		sent = true
-	}
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(result[:], control)
-	ack, err := cpc2.OpenInbound(channel2, control.GetOpenChannel())
-	if err != nil {
-		t.Errorf("should have ack open channel request instead %v, %v", ack, err)
-	}
-
-	ackpacket := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(ack[:], ackpacket)
-	cpc.OpenOutboundResult(nil, ackpacket.GetChannelResult())
-	if channel.Pending != false {
-		t.Errorf("Channel should no longer be pending")
-	}
-
-	gci := &protocol.GroupChatInvite{
-		GroupName:      "hello",
-		GroupSharedKey: []byte{},
-		ServerHost:     "abc.onion",
-	}
-
-	cpp := &protocol.CwtchPeerPacket{
-		GroupChatInvite: gci,
-	}
-	packet, _ := proto.Marshal(cpp)
-	cpc.Packet(packet)
-	if sent && th.ReceviedGroupInvite == false {
-		t.Errorf("Should have sent invite packet to handler")
-	}
-}

protocol/connections/peer/peer_data_channel.go

@@ -1,98 +0,0 @@
-package peer
-
-import (
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-)
-
-// CwtchPeerDataChannel implements the ChannelHandler interface for a channel of
-// type "im.cwtch.peer.data The channel may be inbound or outbound.
-//
-// CwtchPeerChannel implements protocol-level sanity and state validation, but
-// does not handle or acknowledge Cwtch messages. The application must provide
-// a CwtchPeerChannelHandler implementation to handle Cwtch events.
-type CwtchPeerDataChannel struct {
-	// Methods of Handler are called for Cwtch events on this channel
-	Handler CwtchPeerDataHandler
-	channel *channels.Channel
-}
-
-// CwtchPeerDataHandler is implemented by an application type to receive
-// events from a CwtchPeerChannel.
-type CwtchPeerDataHandler interface {
-	HandlePacket([]byte) []byte
-}
-
-// SendMessage sends a raw message on this channel
-func (cpc *CwtchPeerDataChannel) SendMessage(data []byte) {
-	cpc.channel.SendMessage(data)
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.Cwtch".
-func (cpc *CwtchPeerDataChannel) Type() string {
-	return "im.cwtch.peer.data"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cpc *CwtchPeerDataChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch channels any side can open
-func (cpc *CwtchPeerDataChannel) OnlyClientCanOpen() bool {
-	return false
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cpc *CwtchPeerDataChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are bidirectional
-func (cpc *CwtchPeerDataChannel) Bidirectional() bool {
-	return true
-}
-
-// RequiresAuthentication - Cwtch channels require hidden service auth
-func (cpc *CwtchPeerDataChannel) RequiresAuthentication() string {
-	return "im.ricochet.auth.3dh"
-}
-
-// OpenInbound is the first method called for an inbound channel request.
-// If an error is returned, the channel is rejected. If a RawMessage is
-// returned, it will be sent as the ChannelResult message.
-func (cpc *CwtchPeerDataChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	cpc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.AckOpenChannel(channel.ID), nil
-}
-
-// OpenOutbound is the first method called for an outbound channel request.
-// If an error is returned, the channel is not opened. If a RawMessage is
-// returned, it will be sent as the OpenChannel message.
-func (cpc *CwtchPeerDataChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	cpc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.OpenChannel(channel.ID, cpc.Type()), nil
-}
-
-// OpenOutboundResult is called when a response is received for an
-// outbound OpenChannel request. If `err` is non-nil, the channel was
-// rejected and Closed will be called immediately afterwards. `raw`
-// contains the raw protocol message including any extension data.
-func (cpc *CwtchPeerDataChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	if err == nil {
-		if crm.GetOpened() {
-			cpc.channel.Pending = false
-		}
-	}
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cpc *CwtchPeerDataChannel) Packet(data []byte) {
-	ret := cpc.Handler.HandlePacket(data)
-	if len(ret) > 0 {
-		cpc.channel.SendMessage(ret)
-	}
-}

protocol/connections/peerapp.go

@@ -0,0 +1,126 @@
+package connections
+
+import (
+	"cwtch.im/cwtch/event"
+	"encoding/json"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/applications"
+	"git.openprivacy.ca/openprivacy/log"
+	"sync"
+)
+
+const cwtchCapability = tapir.Capability("cwtchCapability")
+
+// PeerApp encapsulates the behaviour of a Cwtch Peer
+type PeerApp struct {
+	applications.AuthApp
+	connection        tapir.Connection
+	MessageHandler    func(string, string, string, []byte)
+	RetValHandler     func(string, []byte, []byte)
+	IsBlocked         func(string) bool
+	IsAllowed         func(string) bool
+	OnAcknowledgement func(string, string)
+	OnAuth            func(string)
+	OnClose           func(string)
+	OnConnecting      func(string)
+
+	getValRequests sync.Map // [string]string eventID:Data
+}
+
+// PeerMessage is an encapsulation that can be used by higher level applications
+type PeerMessage struct {
+	ID      string // A unique Message ID (primarily used for acknowledgments)
+	Context string // A unique context identifier i.e. im.cwtch.chat
+	Data    []byte // The serialized data packet.
+}
+
+type peerGetVal struct {
+	Scope, Path string
+}
+
+type peerRetVal struct {
+	Val    string
+	Exists bool
+}
+
+// NewInstance should always return a new instantiation of the application.
+func (pa *PeerApp) NewInstance() tapir.Application {
+	newApp := new(PeerApp)
+	newApp.MessageHandler = pa.MessageHandler
+	newApp.IsBlocked = pa.IsBlocked
+	newApp.IsAllowed = pa.IsAllowed
+	newApp.OnAcknowledgement = pa.OnAcknowledgement
+	newApp.OnAuth = pa.OnAuth
+	newApp.OnClose = pa.OnClose
+	newApp.OnConnecting = pa.OnConnecting
+	newApp.RetValHandler = pa.RetValHandler
+	return newApp
+}
+
+// Init is run when the connection is first started.
+func (pa *PeerApp) Init(connection tapir.Connection) {
+	// First run the Authentication App
+	pa.AuthApp.Init(connection)
+
+	if connection.HasCapability(applications.AuthCapability) {
+
+		pa.connection = connection
+		connection.SetCapability(cwtchCapability)
+
+		if pa.IsBlocked(connection.Hostname()) {
+			pa.connection.Close()
+			pa.OnClose(connection.Hostname())
+		} else {
+			pa.OnAuth(connection.Hostname())
+			go pa.listen()
+		}
+	} else {
+		// The auth protocol wasn't completed, we can safely shutdown the connection
+		connection.Close()
+	}
+}
+
+func (pa *PeerApp) listen() {
+	for {
+		message := pa.connection.Expect()
+		if len(message) == 0 {
+			log.Debugf("0 byte read, socket has likely failed. Closing the listen goroutine")
+			pa.OnClose(pa.connection.Hostname())
+			return
+		}
+		var peerMessage PeerMessage
+		err := json.Unmarshal(message, &peerMessage)
+		if err == nil {
+			switch peerMessage.Context {
+			case event.ContextAck:
+				pa.OnAcknowledgement(pa.connection.Hostname(), peerMessage.ID)
+			case event.ContextRetVal:
+				req, ok := pa.getValRequests.Load(peerMessage.ID)
+				if ok {
+					reqStr := []byte(req.(string))
+					pa.RetValHandler(pa.connection.Hostname(), reqStr, peerMessage.Data)
+					pa.getValRequests.Delete(peerMessage.ID)
+				}
+			default:
+				if pa.IsAllowed(pa.connection.Hostname()) {
+					pa.MessageHandler(pa.connection.Hostname(), peerMessage.ID, peerMessage.Context, peerMessage.Data)
+
+					// Acknowledge the message
+					pa.SendMessage(PeerMessage{peerMessage.ID, event.ContextAck, []byte{}})
+				}
+			}
+		} else {
+			log.Errorf("Error unmarshalling PeerMessage package: %x %v", message, err)
+		}
+	}
+}
+
+// SendMessage sends the peer a preformatted message
+// NOTE: This is a stub, we will likely want to extend this to better reflect the desired protocol
+func (pa *PeerApp) SendMessage(message PeerMessage) {
+	if message.Context == event.ContextGetVal {
+		pa.getValRequests.Store(message.ID, string(message.Data))
+	}
+	serialized, _ := json.Marshal(message)
+	pa.connection.Send(serialized)
+}

protocol/connections/peerpeerconnection.go

@@ -1,112 +0,0 @@
-package connections
-
-import (
-	"cwtch.im/cwtch/protocol/connections/peer"
-	"git.openprivacy.ca/openprivacy/libricochet-go"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connection"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"time"
-)
-
-// PeerPeerConnection encapsulates a single outgoing cwtchPeer->cwtchPeer connection
-type PeerPeerConnection struct {
-	connection.AutoConnectionHandler
-	PeerHostname   string
-	state          ConnectionState
-	connection     *connection.Connection
-	protocolEngine *Engine
-}
-
-// NewPeerPeerConnection creates a new peer connection for the given hostname and profile.
-func NewPeerPeerConnection(peerhostname string, protocolEngine *Engine) *PeerPeerConnection {
-	ppc := new(PeerPeerConnection)
-	ppc.PeerHostname = peerhostname
-	ppc.protocolEngine = protocolEngine
-	ppc.Init()
-	return ppc
-}
-
-// GetState returns the current connection state
-func (ppc *PeerPeerConnection) GetState() ConnectionState {
-	return ppc.state
-}
-
-// SendPacket sends data packets on the optional data channel
-func (ppc *PeerPeerConnection) SendPacket(data []byte) {
-	ppc.WaitTilAuthenticated()
-	ppc.connection.Do(func() error {
-		channel := ppc.connection.Channel("im.cwtch.peer.data", channels.Outbound)
-		if channel != nil {
-			peerchannel, ok := channel.Handler.(*peer.CwtchPeerDataChannel)
-			if ok {
-				log.Debugf("Sending packet\n")
-				peerchannel.SendMessage(data)
-			}
-		}
-		return nil
-	})
-}
-
-// SendGroupInvite sends the given serialized invite packet to the Peer
-func (ppc *PeerPeerConnection) SendGroupInvite(invite []byte) {
-	ppc.WaitTilAuthenticated()
-	ppc.connection.Do(func() error {
-		channel := ppc.connection.Channel("im.cwtch.peer", channels.Outbound)
-		if channel != nil {
-			peerchannel, ok := channel.Handler.(*peer.CwtchPeerChannel)
-			if ok {
-				log.Debugf("Sending group invite packet\n")
-				peerchannel.SendMessage(invite)
-			}
-		}
-		return nil
-	})
-}
-
-// WaitTilAuthenticated waits until the underlying connection is authenticated
-func (ppc *PeerPeerConnection) WaitTilAuthenticated() {
-	for {
-		if ppc.GetState() == AUTHENTICATED {
-			break
-		}
-		time.Sleep(time.Second * 1)
-	}
-}
-
-// Run manages the setup and teardown of a peer->peer connection
-func (ppc *PeerPeerConnection) Run() error {
-	ppc.state = CONNECTING
-	rc, err := goricochet.Open(ppc.protocolEngine.ACN, ppc.PeerHostname)
-	if err == nil {
-		ppc.connection = rc
-		ppc.state = CONNECTED
-		_, err := connection.HandleOutboundConnection(ppc.connection).ProcessAuthAsV3Client(ppc.protocolEngine.Identity)
-		if err == nil {
-			ppc.state = AUTHENTICATED
-			go func() {
-				ppc.connection.Do(func() error {
-					ppc.connection.RequestOpenChannel("im.cwtch.peer", &peer.CwtchPeerChannel{Handler: ppc.protocolEngine.GetPeerHandler(ppc.PeerHostname)})
-					return nil
-				})
-
-				ppc.connection.Do(func() error {
-					ppc.connection.RequestOpenChannel("im.cwtch.peer.data", &peer.CwtchPeerDataChannel{Handler: ppc.protocolEngine.GetPeerHandler(ppc.PeerHostname)})
-					return nil
-				})
-			}()
-
-			ppc.connection.Process(ppc)
-		}
-	}
-	ppc.state = FAILED
-	return err
-}
-
-// Close closes the connection
-func (ppc *PeerPeerConnection) Close() {
-	ppc.state = KILLED
-	if ppc.connection != nil {
-		ppc.connection.Conn.Close()
-	}
-}

protocol/connections/peerpeerconnection_test.go

@@ -1,87 +0,0 @@
-package connections
-
-import (
-	"crypto/rand"
-	"cwtch.im/cwtch/model"
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/peer"
-	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connection"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"golang.org/x/crypto/ed25519"
-	"net"
-	"testing"
-	"time"
-)
-
-func PeerAuthValid(hostname string, key ed25519.PublicKey) (allowed, known bool) {
-	return true, true
-}
-
-func runtestpeer(t *testing.T, tp *TestPeer, identity identity.Identity, listenChan chan bool) {
-	ln, _ := net.Listen("tcp", "127.0.0.1:5452")
-	listenChan <- true
-	conn, _ := ln.Accept()
-	defer conn.Close()
-
-	rc, err := goricochet.NegotiateVersionInbound(conn)
-	if err != nil {
-		t.Errorf("Negotiate Version Error: %v", err)
-	}
-	err = connection.HandleInboundConnection(rc).ProcessAuthAsV3Server(identity, PeerAuthValid)
-	if err != nil {
-		t.Errorf("ServerAuth Error: %v", err)
-	}
-	tp.RegisterChannelHandler("im.cwtch.peer", func() channels.Handler {
-		cpc := new(peer.CwtchPeerChannel)
-		cpc.Handler = tp
-		return cpc
-	})
-
-	rc.Process(tp)
-}
-
-type TestPeer struct {
-	connection.AutoConnectionHandler
-	ReceivedIdentityPacket bool
-	ReceivedGroupInvite    bool
-}
-
-func (tp *TestPeer) HandleGroupInvite(gci *protocol.GroupChatInvite) {
-	tp.ReceivedGroupInvite = true
-}
-
-func TestPeerPeerConnection(t *testing.T) {
-	pub, priv, _ := ed25519.GenerateKey(rand.Reader)
-	identity := identity.InitializeV3("", &priv, &pub)
-
-	profile := model.GenerateNewProfile("alice")
-	hostname := identity.Hostname()
-	ppc := NewPeerPeerConnection("127.0.0.1:5452|"+hostname, &Engine{ACN: connectivity.LocalProvider(), Identity: identity})
-
-	tp := new(TestPeer)
-	tp.Init()
-	listenChan := make(chan bool)
-	go runtestpeer(t, tp, identity, listenChan)
-	<-listenChan
-	state := ppc.GetState()
-	if state != DISCONNECTED {
-		fmt.Println("ERROR state should be disconnected")
-		t.Errorf("new connections should start in disconnected state")
-	}
-	go ppc.Run()
-	time.Sleep(time.Second * 5)
-	state = ppc.GetState()
-	if state != AUTHENTICATED {
-		t.Errorf("connection state should be authenticated(3), was instead %v", state)
-	}
-	_, invite, _ := profile.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
-	ppc.SendGroupInvite(invite)
-	time.Sleep(time.Second * 3)
-	if tp.ReceivedGroupInvite == false {
-		t.Errorf("should have received an group invite packet")
-	}
-}

protocol/connections/peerserverconnection.go

@@ -1,140 +0,0 @@
-package connections
-
-import (
-	"crypto/rand"
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/fetch"
-	"cwtch.im/cwtch/protocol/connections/listen"
-	"cwtch.im/cwtch/protocol/connections/send"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connection"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"golang.org/x/crypto/ed25519"
-	"time"
-)
-
-// PeerServerConnection encapsulates a single Peer->Server connection
-type PeerServerConnection struct {
-	connection.AutoConnectionHandler
-	Server     string
-	state      ConnectionState
-	connection *connection.Connection
-	acn        connectivity.ACN
-
-	GroupMessageHandler func(string, *protocol.GroupMessage)
-}
-
-// NewPeerServerConnection creates a new Peer->Server outbound connection
-func NewPeerServerConnection(acn connectivity.ACN, serverhostname string) *PeerServerConnection {
-	psc := new(PeerServerConnection)
-	psc.acn = acn
-	psc.Server = serverhostname
-	psc.Init()
-	return psc
-}
-
-// GetState returns the current connection state
-func (psc *PeerServerConnection) GetState() ConnectionState {
-	return psc.state
-}
-
-// WaitTilAuthenticated waits until the underlying connection is authenticated
-func (psc *PeerServerConnection) WaitTilAuthenticated() {
-	for {
-		if psc.GetState() == AUTHENTICATED {
-			break
-		}
-		time.Sleep(time.Second * 1)
-	}
-}
-
-// Run manages the setup and teardown of a peer server connection
-func (psc *PeerServerConnection) Run() error {
-	log.Infof("Connecting to %v", psc.Server)
-	rc, err := goricochet.Open(psc.acn, psc.Server)
-	if err == nil {
-		psc.connection = rc
-		psc.state = CONNECTED
-		pub, priv, err := ed25519.GenerateKey(rand.Reader)
-		if err == nil {
-			_, err := connection.HandleOutboundConnection(psc.connection).ProcessAuthAsV3Client(identity.InitializeV3("cwtchpeer", &priv, &pub))
-			if err == nil {
-				psc.state = AUTHENTICATED
-
-				go func() {
-					psc.connection.Do(func() error {
-						psc.connection.RequestOpenChannel("im.cwtch.server.fetch", &fetch.CwtchPeerFetchChannel{Handler: psc})
-						return nil
-					})
-
-					psc.connection.Do(func() error {
-						psc.connection.RequestOpenChannel("im.cwtch.server.listen", &listen.CwtchPeerListenChannel{Handler: psc})
-						return nil
-					})
-				}()
-				psc.connection.Process(psc)
-			}
-		}
-	}
-	psc.state = FAILED
-	return err
-}
-
-// Break makes Run() return and prevents processing, but doesn't close the connection.
-func (psc *PeerServerConnection) Break() error {
-	return psc.connection.Break()
-}
-
-// SendGroupMessage sends the given protocol message to the Server.
-func (psc *PeerServerConnection) SendGroupMessage(gm *protocol.GroupMessage) error {
-	if psc.state != AUTHENTICATED {
-		return errors.New("peer is not yet connected & authenticated to server cannot send message")
-	}
-
-	err := psc.connection.Do(func() error {
-		psc.connection.RequestOpenChannel("im.cwtch.server.send", &send.CwtchPeerSendChannel{})
-		return nil
-	})
-
-	errCount := 0
-	for errCount < 5 {
-		time.Sleep(time.Second * 1)
-		err = psc.connection.Do(func() error {
-			channel := psc.connection.Channel("im.cwtch.server.send", channels.Outbound)
-			if channel == nil {
-				return errors.New("no channel found")
-			}
-			sendchannel, ok := channel.Handler.(*send.CwtchPeerSendChannel)
-			if ok {
-				return sendchannel.SendGroupMessage(gm)
-			}
-			return errors.New("channel is not a peer send channel (this should definitely not happen)")
-		})
-
-		if err != nil {
-			errCount++
-		} else {
-			return nil
-		}
-	}
-
-	return err
-}
-
-// Close shuts down the connection (freeing the handler goroutines)
-func (psc *PeerServerConnection) Close() {
-	psc.state = KILLED
-	if psc.connection != nil {
-		psc.connection.Conn.Close()
-	}
-}
-
-// HandleGroupMessage passes the given group message back to the profile.
-func (psc *PeerServerConnection) HandleGroupMessage(gm *protocol.GroupMessage) {
-	log.Debugf("Received Group Message")
-	psc.GroupMessageHandler(psc.Server, gm)
-}

protocol/connections/peerserverconnection_test.go

@@ -1,106 +0,0 @@
-package connections
-
-import (
-	"crypto/rand"
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/server/fetch"
-	"cwtch.im/cwtch/server/send"
-	"git.openprivacy.ca/openprivacy/libricochet-go"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connection"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"golang.org/x/crypto/ed25519"
-	"net"
-	"testing"
-	"time"
-)
-
-func ServerAuthValid(hostname string, key ed25519.PublicKey) (allowed, known bool) {
-	return true, true
-}
-
-type TestServer struct {
-	connection.AutoConnectionHandler
-	Received bool
-}
-
-func (ts *TestServer) HandleGroupMessage(gm *protocol.GroupMessage) {
-	ts.Received = true
-}
-
-func (ts *TestServer) HandleFetchRequest() []*protocol.GroupMessage {
-	return []*protocol.GroupMessage{{Ciphertext: []byte("hello"), Signature: []byte{}, Spamguard: []byte{}}, {Ciphertext: []byte("hello"), Signature: []byte{}, Spamguard: []byte{}}}
-}
-
-func runtestserver(t *testing.T, ts *TestServer, identity identity.Identity, listenChan chan bool) {
-	ln, _ := net.Listen("tcp", "127.0.0.1:5451")
-	listenChan <- true
-	conn, _ := ln.Accept()
-	defer conn.Close()
-
-	rc, err := goricochet.NegotiateVersionInbound(conn)
-	if err != nil {
-		t.Errorf("Negotiate Version Error: %v", err)
-	}
-	err = connection.HandleInboundConnection(rc).ProcessAuthAsV3Server(identity, ServerAuthValid)
-	if err != nil {
-		t.Errorf("ServerAuth Error: %v", err)
-	}
-
-	ts.RegisterChannelHandler("im.cwtch.server.send", func() channels.Handler {
-		server := new(send.CwtchServerSendChannel)
-		server.Handler = ts
-		return server
-	})
-
-	ts.RegisterChannelHandler("im.cwtch.server.fetch", func() channels.Handler {
-		server := new(fetch.CwtchServerFetchChannel)
-		server.Handler = ts
-		return server
-	})
-
-	rc.Process(ts)
-}
-
-func TestPeerServerConnection(t *testing.T) {
-	pub, priv, _ := ed25519.GenerateKey(rand.Reader)
-
-	identity := identity.InitializeV3("", &priv, &pub)
-
-	ts := new(TestServer)
-	ts.Init()
-	listenChan := make(chan bool)
-	go runtestserver(t, ts, identity, listenChan)
-	<-listenChan
-	onionAddr := identity.Hostname()
-
-	psc := NewPeerServerConnection(connectivity.LocalProvider(), "127.0.0.1:5451|"+onionAddr)
-	numcalls := 0
-	psc.GroupMessageHandler = func(s string, gm *protocol.GroupMessage) {
-		numcalls++
-	}
-	state := psc.GetState()
-	if state != DISCONNECTED {
-		t.Errorf("new connections should start in disconnected state")
-	}
-	time.Sleep(time.Second * 1)
-	go psc.Run()
-	time.Sleep(time.Second * 2)
-	state = psc.GetState()
-	if state != AUTHENTICATED {
-		t.Errorf("connection should now be authed(%v), instead was %v", AUTHENTICATED, state)
-	}
-
-	gm := &protocol.GroupMessage{Ciphertext: []byte("hello"), Signature: []byte{}}
-	psc.SendGroupMessage(gm)
-	time.Sleep(time.Second * 2)
-	if ts.Received == false {
-		t.Errorf("Should have received a group message in test server")
-	}
-
-	if numcalls != 2 {
-		t.Errorf("Should have received 2  calls from fetch request, instead received %v", numcalls)
-	}
-
-}

protocol/connections/send/peer_send_channel.go

@@ -1,95 +0,0 @@
-package send
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/spam"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchPeerSendChannel is the peer implementation of im.cwtch.server.send
-type CwtchPeerSendChannel struct {
-	channel   *channels.Channel
-	spamGuard spam.Guard
-	challenge []byte
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.server.send".
-func (cpsc *CwtchPeerSendChannel) Type() string {
-	return "im.cwtch.server.send"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cpsc *CwtchPeerSendChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch server channels only peers may open.
-func (cpsc *CwtchPeerSendChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cpsc *CwtchPeerSendChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cpsc *CwtchPeerSendChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require no auth
-func (cpsc *CwtchPeerSendChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound should never be called on peers.
-func (cpsc *CwtchPeerSendChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	return nil, errors.New("client does not receive inbound listen channels")
-}
-
-// OpenOutbound is used to set up a new send channel and initialize spamguard
-func (cpsc *CwtchPeerSendChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	cpsc.spamGuard.Difficulty = 2
-	cpsc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.OpenChannel(channel.ID, cpsc.Type()), nil
-}
-
-// OpenOutboundResult confirms the open channel request and sets the spamguard challenge
-func (cpsc *CwtchPeerSendChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	if err == nil {
-		if crm.GetOpened() {
-			ce, _ := proto.GetExtension(crm, protocol.E_ServerNonce)
-			cpsc.challenge = ce.([]byte)[:]
-			cpsc.channel.Pending = false
-		}
-	}
-}
-
-// SendGroupMessage performs the spamguard proof of work and sends a message.
-func (cpsc *CwtchPeerSendChannel) SendGroupMessage(gm *protocol.GroupMessage) error {
-	if cpsc.channel.Pending == false {
-		sgsolve := cpsc.spamGuard.SolveChallenge(cpsc.challenge, gm.GetCiphertext())
-		gm.Spamguard = sgsolve[:]
-		csp := &protocol.CwtchServerPacket{
-			GroupMessage: gm,
-		}
-		packet, _ := proto.Marshal(csp)
-		cpsc.channel.SendMessage(packet)
-		cpsc.channel.CloseChannel()
-	} else {
-		return errors.New("channel isn't set up yet")
-	}
-	return nil
-}
-
-// Packet should never be
-func (cpsc *CwtchPeerSendChannel) Packet(data []byte) {
-	// If we receive a packet on this channel, close the connection
-	cpsc.channel.CloseChannel()
-}

protocol/connections/send/peer_send_channel_test.go

@@ -1,108 +0,0 @@
-package send
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/spam"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-func TestPeerSendChannelAttributes(t *testing.T) {
-	cssc := new(CwtchPeerSendChannel)
-	if cssc.Type() != "im.cwtch.server.send" {
-		t.Errorf("cwtch channel type is incorrect %v", cssc.Type())
-	}
-
-	if !cssc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.send channel")
-	}
-
-	if cssc.Bidirectional() {
-		t.Errorf("im.cwtch.server.listen should not be bidirectional")
-	}
-
-	if !cssc.Singleton() {
-		t.Errorf("im.cwtch.server.listen should be a Singleton")
-	}
-
-	if cssc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cssc.RequiresAuthentication())
-	}
-
-}
-
-func TestPeerSendChannelOpenInbound(t *testing.T) {
-	cssc := new(CwtchPeerSendChannel)
-	channel := new(channels.Channel)
-	_, err := cssc.OpenInbound(channel, nil)
-	if err == nil {
-		t.Errorf("client implementation of im.cwtch.server.Listen should never open an inbound channel")
-	}
-}
-
-func TestPeerSendChannelClosesOnPacket(t *testing.T) {
-	pfc := new(CwtchPeerSendChannel)
-	channel := new(channels.Channel)
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-
-	pfc.OpenOutbound(channel)
-	pfc.Packet([]byte{})
-	if !closed {
-		t.Errorf("send channel should close if server attempts to send packets")
-	}
-}
-
-func TestPeerSendChannel(t *testing.T) {
-	pfc := new(CwtchPeerSendChannel)
-
-	channel := new(channels.Channel)
-	channel.ID = 3
-	success := false
-
-	var sg spam.Guard
-	sg.Difficulty = 2
-
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-
-	channel.SendMessage = func(message []byte) {
-		packet := new(protocol.CwtchServerPacket)
-		proto.Unmarshal(message[:], packet)
-		if packet.GetGroupMessage() != nil {
-			success = sg.ValidateChallenge(packet.GetGroupMessage().GetCiphertext(), packet.GetGroupMessage().GetSpamguard())
-		}
-	}
-	result, err := pfc.OpenOutbound(channel)
-	if err != nil {
-		t.Errorf("expected result but also got non-nil error: result:%v, err: %v", result, err)
-	}
-
-	challenge := sg.GenerateChallenge(3)
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(challenge[:], control)
-
-	pfc.OpenOutboundResult(nil, control.GetChannelResult())
-	if channel.Pending {
-		t.Errorf("once opened channel should no longer be pending")
-	}
-
-	gm := &protocol.GroupMessage{Ciphertext: []byte("hello")}
-	pfc.SendGroupMessage(gm)
-	if !success {
-		t.Errorf("send channel should have successfully sent a valid group message")
-	}
-
-	if !closed {
-		t.Errorf("send channel should have successfully closed after a valid group message")
-	}
-
-	pfc.Closed(nil)
-
-}

protocol/connections/spam/spamguard.go

@@ -1,100 +0,0 @@
-package spam
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"io"
-	//"fmt"
-)
-
-// Guard implements a spam protection mechanism for Cwtch Servers.
-type Guard struct {
-	Difficulty int
-	nonce      [24]byte
-}
-
-func getRandomness(arr *[24]byte) {
-	if _, err := io.ReadFull(rand.Reader, arr[:]); err != nil {
-		utils.CheckError(err)
-	}
-}
-
-//GenerateChallenge returns a channel result packet with a spamguard challenge nonce
-func (sg *Guard) GenerateChallenge(channelID int32) []byte {
-
-	cr := &Protocol_Data_Control.ChannelResult{
-		ChannelIdentifier: proto.Int32(channelID),
-		Opened:            proto.Bool(true),
-	}
-
-	var nonce [24]byte
-	getRandomness(&nonce)
-	sg.nonce = nonce
-	err := proto.SetExtension(cr, protocol.E_ServerNonce, sg.nonce[:])
-	utils.CheckError(err)
-
-	pc := &Protocol_Data_Control.Packet{
-		ChannelResult: cr,
-	}
-	ret, err := proto.Marshal(pc)
-	utils.CheckError(err)
-	return ret
-}
-
-// SolveChallenge takes in a challenge and a message and returns a solution
-// The solution is a 24 byte nonce which when hashed with the challenge and the message
-// produces a sha256 hash with Difficulty leading 0s
-func (sg *Guard) SolveChallenge(challenge []byte, message []byte) []byte {
-	solved := false
-	var spamguard [24]byte
-	sum := sha256.Sum256([]byte{})
-	solve := make([]byte, len(challenge)+len(message)+len(spamguard))
-	for !solved {
-
-		getRandomness(&spamguard)
-
-		copy(solve[0:], challenge[:])
-		copy(solve[len(challenge):], message[:])
-		copy(solve[len(challenge)+len(message):], spamguard[:])
-
-		sum = sha256.Sum256(solve)
-
-		solved = true
-		for i := 0; i < sg.Difficulty; i++ {
-			if sum[i] != 0x00 {
-				solved = false
-			}
-		}
-	}
-	//fmt.Printf("[SOLVED] %x\n",sha256.Sum256(solve))
-	return spamguard[:]
-}
-
-// ValidateChallenge returns true if the message and spamguard pass the challenge
-func (sg *Guard) ValidateChallenge(message []byte, spamguard []byte) bool {
-	if len(spamguard) != 24 {
-		return false
-	}
-
-	// If the message is too large just throw it away.
-	if len(message) > 2048 {
-		return false
-	}
-
-	solve := make([]byte, len(sg.nonce)+len(message)+len(spamguard))
-	copy(solve[0:], sg.nonce[:])
-	copy(solve[len(sg.nonce):], message[:])
-	copy(solve[len(sg.nonce)+len(message):], spamguard[:])
-	sum := sha256.Sum256(solve)
-
-	for i := 0; i < sg.Difficulty; i++ {
-		if sum[i] != 0x00 {
-			return false
-		}
-	}
-	return true
-}

protocol/connections/spam/spamguard_test.go

@@ -1,69 +0,0 @@
-package spam
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-func TestSpamGuard(t *testing.T) {
-	var spamGuard Guard
-	spamGuard.Difficulty = 2
-	challenge := spamGuard.GenerateChallenge(3)
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(challenge[:], control)
-
-	if control.GetChannelResult() != nil {
-		ce, _ := proto.GetExtension(control.GetChannelResult(), protocol.E_ServerNonce)
-		challenge := ce.([]byte)[:]
-
-		sgsolve := spamGuard.SolveChallenge(challenge, []byte("Hello"))
-		t.Logf("Solved: %v %v", challenge, sgsolve)
-		result := spamGuard.ValidateChallenge([]byte("Hello"), sgsolve)
-		if result != true {
-			t.Errorf("Validating Guard Failed")
-		}
-		return
-	}
-
-	t.Errorf("Failed SpamGaurd")
-
-}
-
-func TestSpamGuardBadLength(t *testing.T) {
-	var spamGuard Guard
-	spamGuard.Difficulty = 2
-	spamGuard.GenerateChallenge(3)
-	result := spamGuard.ValidateChallenge([]byte("test"), []byte{0x00, 0x00})
-	if result {
-		t.Errorf("Validating Guard should have failed")
-	}
-}
-
-func TestSpamGuardFail(t *testing.T) {
-	var spamGuard Guard
-	spamGuard.Difficulty = 2
-	challenge := spamGuard.GenerateChallenge(3)
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(challenge[:], control)
-
-	if control.GetChannelResult() != nil {
-		ce, _ := proto.GetExtension(control.GetChannelResult(), protocol.E_ServerNonce)
-		challenge := ce.([]byte)[:]
-
-		var spamGuard2 Guard
-		spamGuard2.Difficulty = 1
-		sgsolve := spamGuard2.SolveChallenge(challenge, []byte("Hello"))
-		t.Logf("Solved: %v %v", challenge, sgsolve)
-		result := spamGuard.ValidateChallenge([]byte("Hello"), sgsolve)
-		if result {
-			t.Errorf("Validating Guard successes")
-		}
-		return
-	}
-
-	t.Errorf("Failed SpamGaurd")
-}

protocol/connections/state.go

@@ -7,17 +7,25 @@ type ConnectionState int
 // DISCONNECTED - No existing connection has been made, or all attempts have failed
 // CONNECTING - We are in the process of attempting to connect to a given endpoint
 // CONNECTED - We have connected but not yet authenticated
-// AUTHENTICATED - im.ricochet.auth-hidden-server has succeeded on thec onnection.
+// AUTHENTICATED - im.ricochet.auth-hidden-server has succeeded on the connection.
+// SYNCED - we have pulled all the messages for groups from the server and are ready to send
 const (
 	DISCONNECTED ConnectionState = iota
 	CONNECTING
 	CONNECTED
 	AUTHENTICATED
+	SYNCED
 	FAILED
 	KILLED
 )
 
 var (
-	// ConnectionStateName allows conversaion of states to their string representations
-	ConnectionStateName = []string{"Disconnected", "Connecting", "Connected", "Authenticated", "Failed", "Killed"}
+	// ConnectionStateName allows conversion of states to their string representations
+	ConnectionStateName = []string{"Disconnected", "Connecting", "Connected", "Authenticated", "Synced", "Failed", "Killed"}
 )
+
+// ConnectionStateToType allows conversion of strings to their state type
+func ConnectionStateToType() map[string]ConnectionState {
+	return map[string]ConnectionState{"Disconnected": DISCONNECTED, "Connecting": CONNECTING,
+		"Connected": CONNECTED, "Authenticated": AUTHENTICATED, "Synced": SYNCED, "Failed": FAILED, "Killed": KILLED}
+}

protocol/connections/tokenboardclientapp.go

@@ -0,0 +1,209 @@
+package connections
+
+import (
+	"cwtch.im/cwtch/protocol/groups"
+	"encoding/json"
+	"errors"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/applications"
+	"git.openprivacy.ca/cwtch.im/tapir/networks/tor"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives/privacypass"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/log"
+	"github.com/gtank/ristretto255"
+	"sync"
+)
+
+// NewTokenBoardClient generates a new Client for Token Board
+func NewTokenBoardClient(acn connectivity.ACN, Y *ristretto255.Element, tokenServiceOnion string, lastKnownSignature []byte, groupMessageHandler func(server string, gm *groups.EncryptedGroupMessage), serverSyncedHandler func(server string), serverClosedHandler func(server string)) tapir.Application {
+	tba := new(TokenBoardClient)
+	tba.acn = acn
+	tba.tokenService = privacypass.NewTokenServer()
+	tba.tokenService.Y = Y
+	tba.tokenServiceOnion = tokenServiceOnion
+	tba.receiveGroupMessageHandler = groupMessageHandler
+	tba.serverSyncedHandler = serverSyncedHandler
+	tba.serverClosedHandler = serverClosedHandler
+	tba.lastKnownSignature = lastKnownSignature
+	return tba
+}
+
+// TokenBoardClient defines a client for the TokenBoard server
+type TokenBoardClient struct {
+	applications.AuthApp
+	connection                 tapir.Connection
+	receiveGroupMessageHandler func(server string, gm *groups.EncryptedGroupMessage)
+	serverSyncedHandler        func(server string)
+	serverClosedHandler        func(server string)
+
+	// Token service handling
+	acn                connectivity.ACN
+	tokens             []*privacypass.Token
+	tokenLock          sync.Mutex
+	tokenService       *privacypass.TokenServer
+	tokenServiceOnion  string
+	lastKnownSignature []byte
+}
+
+// NewInstance Client a new TokenBoardApp
+func (ta *TokenBoardClient) NewInstance() tapir.Application {
+	tba := new(TokenBoardClient)
+	tba.serverSyncedHandler = ta.serverSyncedHandler
+	tba.serverClosedHandler = ta.serverClosedHandler
+	tba.receiveGroupMessageHandler = ta.receiveGroupMessageHandler
+	tba.acn = ta.acn
+	tba.tokenService = ta.tokenService
+	tba.tokenServiceOnion = ta.tokenServiceOnion
+	tba.lastKnownSignature = ta.lastKnownSignature
+	return tba
+}
+
+// Init initializes the cryptographic TokenBoardApp
+func (ta *TokenBoardClient) Init(connection tapir.Connection) {
+	ta.AuthApp.Init(connection)
+	if connection.HasCapability(applications.AuthCapability) {
+		ta.connection = connection
+		ta.connection.SetCapability(groups.CwtchServerSyncedCapability)
+		log.Debugf("Successfully Initialized Connection")
+		go ta.Listen()
+		// Optimistically acquire many tokens for this server...
+		go ta.MakePayment()
+		go ta.MakePayment()
+		ta.Replay()
+	} else {
+		connection.Close()
+	}
+}
+
+// Listen processes the messages for this application
+func (ta *TokenBoardClient) Listen() {
+	for {
+		log.Debugf("Client waiting...")
+		data := ta.connection.Expect()
+		if len(data) == 0 {
+			log.Debugf("Server closed the connection...")
+			ta.serverClosedHandler(ta.connection.Hostname())
+			return // connection is closed
+		}
+
+		// We always expect the server to follow protocol, and the second it doesn't we close the connection
+		var message groups.Message
+		if err := json.Unmarshal(data, &message); err != nil {
+			log.Debugf("Server sent an unexpected message, closing the connection: %v", err)
+			ta.serverClosedHandler(ta.connection.Hostname())
+			ta.connection.Close()
+			return
+		}
+
+		switch message.MessageType {
+		case groups.NewMessageMessage:
+			if message.NewMessage != nil {
+				ta.receiveGroupMessageHandler(ta.connection.Hostname(), &message.NewMessage.EGM)
+			} else {
+				log.Debugf("Server sent an unexpected NewMessage, closing the connection: %s", data)
+				ta.serverClosedHandler(ta.connection.Hostname())
+				ta.connection.Close()
+				return
+			}
+		case groups.PostResultMessage:
+			// TODO handle failure
+		case groups.ReplayResultMessage:
+			if message.ReplayResult != nil {
+				log.Debugf("Replaying %v Messages...", message.ReplayResult.NumMessages)
+				for i := 0; i < message.ReplayResult.NumMessages; i++ {
+					data := ta.connection.Expect()
+
+					if len(data) == 0 {
+						log.Debugf("Server sent an unexpected EncryptedGroupMessage, closing the connection")
+						ta.serverClosedHandler(ta.connection.Hostname())
+						ta.connection.Close()
+						return
+					}
+
+					egm := &groups.EncryptedGroupMessage{}
+					if err := json.Unmarshal(data, egm); err == nil {
+						ta.receiveGroupMessageHandler(ta.connection.Hostname(), egm)
+						ta.lastKnownSignature = egm.Signature
+					} else {
+						log.Debugf("Server sent an unexpected EncryptedGroupMessage, closing the connection: %v", err)
+						ta.serverClosedHandler(ta.connection.Hostname())
+						ta.connection.Close()
+						return
+					}
+				}
+				ta.serverSyncedHandler(ta.connection.Hostname())
+			}
+		}
+	}
+}
+
+// Replay posts a Replay Message to the server.
+func (ta *TokenBoardClient) Replay() {
+	data, _ := json.Marshal(groups.Message{MessageType: groups.ReplayRequestMessage, ReplayRequest: &groups.ReplayRequest{LastCommit: ta.lastKnownSignature}})
+	ta.connection.Send(data)
+}
+
+// PurchaseTokens purchases the given number of tokens from the server (using the provided payment handler)
+func (ta *TokenBoardClient) PurchaseTokens() {
+	ta.MakePayment()
+}
+
+// Post sends a Post Request to the server
+func (ta *TokenBoardClient) Post(ct []byte, sig []byte) (bool, int) {
+	egm := groups.EncryptedGroupMessage{Ciphertext: ct, Signature: sig}
+	token, numTokens, err := ta.NextToken(egm.ToBytes(), ta.connection.Hostname())
+	if err == nil {
+		data, _ := json.Marshal(groups.Message{MessageType: groups.PostRequestMessage, PostRequest: &groups.PostRequest{EGM: egm, Token: token}})
+		log.Debugf("Message Length: %s %v", data, len(data))
+		ta.connection.Send(data)
+		return true, numTokens
+	}
+	log.Debugf("No Valid Tokens: %v", err)
+	return false, numTokens
+}
+
+// MakePayment uses the PoW based token protocol to obtain more tokens
+func (ta *TokenBoardClient) MakePayment() error {
+	log.Debugf("Making a Payment %v", ta)
+	id, sk := primitives.InitializeEphemeralIdentity()
+	client := new(tor.BaseOnionService)
+	client.Init(ta.acn, sk, &id)
+
+	tokenApplication := new(applications.TokenApplication)
+	tokenApplication.TokenService = ta.tokenService
+	powTokenApp := new(applications.ApplicationChain).
+		ChainApplication(new(applications.ProofOfWorkApplication), applications.SuccessfulProofOfWorkCapability).
+		ChainApplication(tokenApplication, applications.HasTokensCapability)
+	client.Connect(ta.tokenServiceOnion, powTokenApp)
+	log.Debugf("Waiting for successful PoW Auth...")
+	conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability)
+	if err == nil {
+		powtapp, _ := conn.App().(*applications.TokenApplication)
+		// Update tokens...we need a lock here to prevent SpendToken from modifying the tokens
+		// during this process..
+		log.Debugf("Updating Tokens")
+		ta.tokenLock.Lock()
+		ta.tokens = append(ta.tokens, powtapp.Tokens...)
+		ta.tokenLock.Unlock()
+		log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit())
+		conn.Close()
+		return nil
+	}
+	log.Debugf("Error making payment: to %v %v", ta.tokenServiceOnion, err)
+	return err
+}
+
+// NextToken retrieves the next token
+func (ta *TokenBoardClient) NextToken(data []byte, hostname string) (privacypass.SpentToken, int, error) {
+	// Taken the first new token, we need a lock here because tokens can be appended by MakePayment
+	// which could result in weird behaviour...
+	ta.tokenLock.Lock()
+	defer ta.tokenLock.Unlock()
+	if len(ta.tokens) == 0 {
+		return privacypass.SpentToken{}, len(ta.tokens), errors.New("no more tokens")
+	}
+	token := ta.tokens[0]
+	ta.tokens = ta.tokens[1:]
+	return token.SpendToken(append(data, hostname...)), len(ta.tokens), nil
+}

protocol/cwtch-profile.pb.go

@@ -1,135 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: cwtch-profile.proto
-
-/*
-Package protocol is a generated protocol buffer package.
-
-It is generated from these files:
-	cwtch-profile.proto
-
-It has these top-level messages:
-	CwtchPeerPacket
-	CwtchIdentity
-	GroupChatInvite
-*/
-package protocol
-
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
-
-type CwtchPeerPacket struct {
-	CwtchIdentify   *CwtchIdentity   `protobuf:"bytes,1,opt,name=cwtch_identify,json=cwtchIdentify" json:"cwtch_identify,omitempty"`
-	GroupChatInvite *GroupChatInvite `protobuf:"bytes,2,opt,name=group_chat_invite,json=groupChatInvite" json:"group_chat_invite,omitempty"`
-}
-
-func (m *CwtchPeerPacket) Reset()                    { *m = CwtchPeerPacket{} }
-func (m *CwtchPeerPacket) String() string            { return proto.CompactTextString(m) }
-func (*CwtchPeerPacket) ProtoMessage()               {}
-func (*CwtchPeerPacket) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
-
-func (m *CwtchPeerPacket) GetCwtchIdentify() *CwtchIdentity {
-	if m != nil {
-		return m.CwtchIdentify
-	}
-	return nil
-}
-
-func (m *CwtchPeerPacket) GetGroupChatInvite() *GroupChatInvite {
-	if m != nil {
-		return m.GroupChatInvite
-	}
-	return nil
-}
-
-type CwtchIdentity struct {
-	Name             string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
-	Ed25519PublicKey []byte `protobuf:"bytes,2,opt,name=ed25519_public_key,json=ed25519PublicKey,proto3" json:"ed25519_public_key,omitempty"`
-}
-
-func (m *CwtchIdentity) Reset()                    { *m = CwtchIdentity{} }
-func (m *CwtchIdentity) String() string            { return proto.CompactTextString(m) }
-func (*CwtchIdentity) ProtoMessage()               {}
-func (*CwtchIdentity) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
-
-func (m *CwtchIdentity) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-func (m *CwtchIdentity) GetEd25519PublicKey() []byte {
-	if m != nil {
-		return m.Ed25519PublicKey
-	}
-	return nil
-}
-
-// [name] has invited you to join a group chat: [message]
-type GroupChatInvite struct {
-	GroupName      string `protobuf:"bytes,1,opt,name=group_name,json=groupName" json:"group_name,omitempty"`
-	GroupSharedKey []byte `protobuf:"bytes,2,opt,name=group_shared_key,json=groupSharedKey,proto3" json:"group_shared_key,omitempty"`
-	ServerHost     string `protobuf:"bytes,3,opt,name=server_host,json=serverHost" json:"server_host,omitempty"`
-	SignedGroupId  []byte `protobuf:"bytes,4,opt,name=signed_group_id,json=signedGroupId,proto3" json:"signed_group_id,omitempty"`
-	InitialMessage []byte `protobuf:"bytes,5,opt,name=initial_message,json=initialMessage,proto3" json:"initial_message,omitempty"`
-}
-
-func (m *GroupChatInvite) Reset()                    { *m = GroupChatInvite{} }
-func (m *GroupChatInvite) String() string            { return proto.CompactTextString(m) }
-func (*GroupChatInvite) ProtoMessage()               {}
-func (*GroupChatInvite) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
-
-func (m *GroupChatInvite) GetGroupName() string {
-	if m != nil {
-		return m.GroupName
-	}
-	return ""
-}
-
-func (m *GroupChatInvite) GetGroupSharedKey() []byte {
-	if m != nil {
-		return m.GroupSharedKey
-	}
-	return nil
-}
-
-func (m *GroupChatInvite) GetServerHost() string {
-	if m != nil {
-		return m.ServerHost
-	}
-	return ""
-}
-
-func (m *GroupChatInvite) GetSignedGroupId() []byte {
-	if m != nil {
-		return m.SignedGroupId
-	}
-	return nil
-}
-
-func (m *GroupChatInvite) GetInitialMessage() []byte {
-	if m != nil {
-		return m.InitialMessage
-	}
-	return nil
-}
-
-func init() {
-	proto.RegisterType((*CwtchPeerPacket)(nil), "protocol.CwtchPeerPacket")
-	proto.RegisterType((*CwtchIdentity)(nil), "protocol.CwtchIdentity")
-	proto.RegisterType((*GroupChatInvite)(nil), "protocol.GroupChatInvite")
-}
-
-func init() { proto.RegisterFile("cwtch-profile.proto", fileDescriptor0) }

protocol/cwtch-profile.proto

@@ -1,21 +0,0 @@
-syntax = "proto3";
-package protocol;
-
-message CwtchPeerPacket {
-        CwtchIdentity cwtch_identify = 1;
-        GroupChatInvite group_chat_invite = 2;
-}
-
-message CwtchIdentity {
-        string name = 1;
-        bytes  ed25519_public_key = 2;
-}
-
-// [name] has invited you to join a group chat: [message]
-message GroupChatInvite {
-        string group_name = 1;
-        bytes  group_shared_key = 2;
-        string server_host = 3;
-        bytes  signed_group_id = 4;
-        bytes  initial_message = 5;
-}

protocol/group_message.pb.go

@@ -1,188 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: group_message.proto
-
-package protocol
-
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import control "git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-type CwtchServerPacket struct {
-	GroupMessage     *GroupMessage `protobuf:"bytes,1,opt,name=group_message,json=groupMessage" json:"group_message,omitempty"`
-	FetchMessage     *FetchMessage `protobuf:"bytes,2,opt,name=fetch_message,json=fetchMessage" json:"fetch_message,omitempty"`
-	XXX_unrecognized []byte        `json:"-"`
-}
-
-func (m *CwtchServerPacket) Reset()                    { *m = CwtchServerPacket{} }
-func (m *CwtchServerPacket) String() string            { return proto.CompactTextString(m) }
-func (*CwtchServerPacket) ProtoMessage()               {}
-func (*CwtchServerPacket) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} }
-
-func (m *CwtchServerPacket) GetGroupMessage() *GroupMessage {
-	if m != nil {
-		return m.GroupMessage
-	}
-	return nil
-}
-
-func (m *CwtchServerPacket) GetFetchMessage() *FetchMessage {
-	if m != nil {
-		return m.FetchMessage
-	}
-	return nil
-}
-
-type FetchMessage struct {
-	XXX_unrecognized []byte `json:"-"`
-}
-
-func (m *FetchMessage) Reset()                    { *m = FetchMessage{} }
-func (m *FetchMessage) String() string            { return proto.CompactTextString(m) }
-func (*FetchMessage) ProtoMessage()               {}
-func (*FetchMessage) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{1} }
-
-type GroupMessage struct {
-	Ciphertext       []byte `protobuf:"bytes,1,req,name=ciphertext" json:"ciphertext,omitempty"`
-	Spamguard        []byte `protobuf:"bytes,2,req,name=spamguard" json:"spamguard,omitempty"`
-	Signature        []byte `protobuf:"bytes,3,req,name=signature" json:"signature,omitempty"`
-	XXX_unrecognized []byte `json:"-"`
-}
-
-func (m *GroupMessage) Reset()                    { *m = GroupMessage{} }
-func (m *GroupMessage) String() string            { return proto.CompactTextString(m) }
-func (*GroupMessage) ProtoMessage()               {}
-func (*GroupMessage) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{2} }
-
-func (m *GroupMessage) GetCiphertext() []byte {
-	if m != nil {
-		return m.Ciphertext
-	}
-	return nil
-}
-
-func (m *GroupMessage) GetSpamguard() []byte {
-	if m != nil {
-		return m.Spamguard
-	}
-	return nil
-}
-
-func (m *GroupMessage) GetSignature() []byte {
-	if m != nil {
-		return m.Signature
-	}
-	return nil
-}
-
-// DecryptedGroupMessage is *never* sent in the clear on the wire
-// and is only ever sent when encrypted in the ciphertext parameter of
-// GroupMessage
-type DecryptedGroupMessage struct {
-	Onion              *string `protobuf:"bytes,1,req,name=onion" json:"onion,omitempty"`
-	Timestamp          *int32  `protobuf:"varint,2,req,name=timestamp" json:"timestamp,omitempty"`
-	Text               *string `protobuf:"bytes,3,req,name=text" json:"text,omitempty"`
-	SignedGroupId      []byte  `protobuf:"bytes,4,req,name=signed_group_id,json=signedGroupId" json:"signed_group_id,omitempty"`
-	PreviousMessageSig []byte  `protobuf:"bytes,5,req,name=previous_message_sig,json=previousMessageSig" json:"previous_message_sig,omitempty"`
-	// Used to prevent analysis on text length, length is 1024 - len(text)
-	Padding          []byte `protobuf:"bytes,6,req,name=padding" json:"padding,omitempty"`
-	XXX_unrecognized []byte `json:"-"`
-}
-
-func (m *DecryptedGroupMessage) Reset()                    { *m = DecryptedGroupMessage{} }
-func (m *DecryptedGroupMessage) String() string            { return proto.CompactTextString(m) }
-func (*DecryptedGroupMessage) ProtoMessage()               {}
-func (*DecryptedGroupMessage) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{3} }
-
-func (m *DecryptedGroupMessage) GetOnion() string {
-	if m != nil && m.Onion != nil {
-		return *m.Onion
-	}
-	return ""
-}
-
-func (m *DecryptedGroupMessage) GetTimestamp() int32 {
-	if m != nil && m.Timestamp != nil {
-		return *m.Timestamp
-	}
-	return 0
-}
-
-func (m *DecryptedGroupMessage) GetText() string {
-	if m != nil && m.Text != nil {
-		return *m.Text
-	}
-	return ""
-}
-
-func (m *DecryptedGroupMessage) GetSignedGroupId() []byte {
-	if m != nil {
-		return m.SignedGroupId
-	}
-	return nil
-}
-
-func (m *DecryptedGroupMessage) GetPreviousMessageSig() []byte {
-	if m != nil {
-		return m.PreviousMessageSig
-	}
-	return nil
-}
-
-func (m *DecryptedGroupMessage) GetPadding() []byte {
-	if m != nil {
-		return m.Padding
-	}
-	return nil
-}
-
-var E_ServerNonce = &proto.ExtensionDesc{
-	ExtendedType:  (*control.ChannelResult)(nil),
-	ExtensionType: ([]byte)(nil),
-	Field:         8200,
-	Name:          "protocol.server_nonce",
-	Tag:           "bytes,8200,opt,name=server_nonce,json=serverNonce",
-	Filename:      "group_message.proto",
-}
-
-func init() {
-	proto.RegisterType((*CwtchServerPacket)(nil), "protocol.CwtchServerPacket")
-	proto.RegisterType((*FetchMessage)(nil), "protocol.FetchMessage")
-	proto.RegisterType((*GroupMessage)(nil), "protocol.GroupMessage")
-	proto.RegisterType((*DecryptedGroupMessage)(nil), "protocol.DecryptedGroupMessage")
-	proto.RegisterExtension(E_ServerNonce)
-}
-
-func init() { proto.RegisterFile("group_message.proto", fileDescriptor2) }
-
-var fileDescriptor2 = []byte{
-	// 360 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0x4d, 0x6e, 0xdb, 0x30,
-	0x10, 0x85, 0x21, 0xff, 0xb4, 0xf5, 0x58, 0x6e, 0x51, 0xd6, 0x6d, 0x89, 0xa2, 0x28, 0x0c, 0x2d,
-	0x0a, 0xaf, 0x8c, 0x20, 0xcb, 0x78, 0x13, 0xc0, 0x41, 0x82, 0x2c, 0x12, 0x04, 0xf2, 0x01, 0x04,
-	0x42, 0x1a, 0x53, 0x4c, 0x24, 0x92, 0x20, 0x29, 0x27, 0xb9, 0x41, 0x36, 0x39, 0x5d, 0x2e, 0x14,
-	0x88, 0xb2, 0x6c, 0x39, 0x2b, 0x69, 0xde, 0x37, 0x6f, 0xde, 0x80, 0x24, 0xfc, 0xe0, 0x46, 0x55,
-	0x3a, 0x29, 0xd1, 0x5a, 0xc6, 0x71, 0xa1, 0x8d, 0x72, 0x8a, 0x7c, 0xf1, 0x9f, 0x54, 0x15, 0x7f,
-	0xa6, 0x2b, 0x25, 0x9d, 0x51, 0xc5, 0x2a, 0x67, 0x52, 0x62, 0xd1, 0xf0, 0xe8, 0x35, 0x80, 0xef,
-	0xab, 0x47, 0x97, 0xe6, 0x6b, 0x34, 0x5b, 0x34, 0x77, 0x2c, 0x7d, 0x40, 0x47, 0x96, 0x30, 0x39,
-	0x1a, 0x46, 0x83, 0x59, 0x30, 0x1f, 0x9f, 0xfe, 0x5a, 0xb4, 0xd3, 0x16, 0x57, 0x35, 0xbe, 0x69,
-	0x68, 0x1c, 0xf2, 0x4e, 0x55, 0x9b, 0x37, 0xe8, 0xd2, 0x7c, 0x6f, 0xee, 0x7d, 0x34, 0x5f, 0xd6,
-	0x78, 0x6f, 0xde, 0x74, 0xaa, 0xe8, 0x2b, 0x84, 0x5d, 0x1a, 0xdd, 0x43, 0xd8, 0x8d, 0x22, 0xff,
-	0x00, 0x52, 0xa1, 0x73, 0x34, 0x0e, 0x9f, 0x1c, 0x0d, 0x66, 0xbd, 0x79, 0x18, 0x77, 0x14, 0xf2,
-	0x17, 0x46, 0x56, 0xb3, 0x92, 0x57, 0xcc, 0x64, 0xb4, 0xe7, 0xf1, 0x41, 0xf0, 0x54, 0x70, 0xc9,
-	0x5c, 0x65, 0x90, 0xf6, 0x77, 0xb4, 0x15, 0xa2, 0xb7, 0x00, 0x7e, 0x5e, 0x60, 0x6a, 0x9e, 0xb5,
-	0xc3, 0xec, 0x28, 0x75, 0x0a, 0x43, 0x25, 0x85, 0x92, 0x3e, 0x70, 0x14, 0x37, 0x45, 0x3d, 0xcd,
-	0x89, 0x12, 0xad, 0x63, 0xa5, 0xf6, 0x59, 0xc3, 0xf8, 0x20, 0x10, 0x02, 0x03, 0xbf, 0x63, 0xdf,
-	0x5b, 0xfc, 0x3f, 0xf9, 0x0f, 0xdf, 0xea, 0x38, 0xcc, 0x92, 0xe6, 0x78, 0x45, 0x46, 0x07, 0x7e,
-	0x8b, 0x49, 0x23, 0xfb, 0xd0, 0xeb, 0x8c, 0x9c, 0xc0, 0x54, 0x1b, 0xdc, 0x0a, 0x55, 0xd9, 0xf6,
-	0x14, 0x13, 0x2b, 0x38, 0x1d, 0xfa, 0x66, 0xd2, 0xb2, 0xdd, 0x7a, 0x6b, 0xc1, 0x09, 0x85, 0xcf,
-	0x9a, 0x65, 0x99, 0x90, 0x9c, 0x7e, 0xf2, 0x4d, 0x6d, 0x79, 0xb6, 0x84, 0xd0, 0xfa, 0xbb, 0x4d,
-	0xa4, 0x92, 0x29, 0x92, 0xdf, 0x87, 0x7b, 0xd8, 0x3d, 0x85, 0x18, 0x6d, 0x55, 0x38, 0xfa, 0x72,
-	0x3e, 0x0b, 0xe6, 0x61, 0x3c, 0x6e, 0xba, 0x6f, 0xeb, 0xe6, 0xf7, 0x00, 0x00, 0x00, 0xff, 0xff,
-	0x33, 0x30, 0xca, 0x8b, 0x54, 0x02, 0x00, 0x00,
-}

protocol/group_message.proto

@@ -1,35 +0,0 @@
-syntax = "proto2";
-package protocol;
-
-import "ControlChannel.proto";
-
-message CwtchServerPacket {
-       optional GroupMessage group_message = 1;
-       optional FetchMessage fetch_message = 2;
-}
-
-extend protocol.ChannelResult {
-    optional bytes server_nonce = 8200;      // 32 random bytes
-}
-
-message FetchMessage {
-}
-
-message GroupMessage {
-        required bytes ciphertext = 1;
-        required bytes spamguard = 2;
-        required bytes signature = 3;
-}
-
-// DecryptedGroupMessage is *never* sent in the clear on the wire
-// and is only ever sent when encrypted in the ciphertext parameter of
-// GroupMessage
-message DecryptedGroupMessage {
-        required string onion = 1;
-        required int32 timestamp = 2;
-        required string text = 3;
-        required bytes signed_group_id = 4;
-        required bytes previous_message_sig =5;
-        // Used to prevent analysis on text length, length is 1024 - len(text)
-        required bytes padding = 6;
-}

protocol/groups/common.go

@@ -0,0 +1,95 @@
+package groups
+
+import (
+	"encoding/json"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives/privacypass"
+)
+
+// CwtchServerSyncedCapability is used to indicate that a given cwtch server is synced
+const CwtchServerSyncedCapability = tapir.Capability("CwtchServerSyncedCapability")
+
+// GroupInvite provides a structured type for communicating group information to peers
+type GroupInvite struct {
+	GroupID       string
+	GroupName     string
+	SignedGroupID []byte
+	Timestamp     uint64
+	SharedKey     []byte
+	ServerHost    string
+}
+
+// DecryptedGroupMessage is the main encapsulation of group message data
+type DecryptedGroupMessage struct {
+	Text      string
+	Onion     string
+	Timestamp uint64
+	// NOTE: SignedGroupID is now a misnomer, the only way this is signed is indirectly via the signed encrypted group messages
+	// We now treat GroupID as binding to a server/key rather than an "owner" - additional validation logic (to e.g.
+	// respect particular group constitutions) can be built on top of group messages, but the underlying groups are
+	// now agnostic to those models.
+	SignedGroupID      []byte
+	PreviousMessageSig []byte
+	Padding            []byte
+}
+
+// EncryptedGroupMessage provides an encapsulation of the encrypted group message stored on the server
+type EncryptedGroupMessage struct {
+	Ciphertext []byte
+	Signature  []byte
+}
+
+// ToBytes converts the encrypted group message to a set of bytes for serialization
+func (egm EncryptedGroupMessage) ToBytes() []byte {
+	data, _ := json.Marshal(egm)
+	return data
+}
+
+// MessageType defines the enum for TokenBoard messages
+type MessageType int
+
+// Message Types
+const (
+	ReplayRequestMessage MessageType = iota
+	ReplayResultMessage
+	PostRequestMessage
+	PostResultMessage
+	NewMessageMessage
+)
+
+// Message encapsulates the application protocol
+type Message struct {
+	MessageType   MessageType
+	PostRequest   *PostRequest   `json:",omitempty"`
+	PostResult    *PostResult    `json:",omitempty"`
+	NewMessage    *NewMessage    `json:",omitempty"`
+	ReplayRequest *ReplayRequest `json:",omitempty"`
+	ReplayResult  *ReplayResult  `json:",omitempty"`
+}
+
+// ReplayRequest requests a reply from the given Commit
+type ReplayRequest struct {
+	LastCommit []byte
+}
+
+// PostRequest requests to post the message to the board with the given token
+type PostRequest struct {
+	Token privacypass.SpentToken
+	EGM   EncryptedGroupMessage
+}
+
+// PostResult returns the success of a given post attempt
+type PostResult struct {
+	Success bool
+}
+
+// ReplayResult is sent by the server before a stream of replayed messages
+type ReplayResult struct {
+	NumMessages int
+}
+
+// NewMessage is used to send a new bulletin board message to interested peers.
+type NewMessage struct {
+	//Token   privacypass.SpentToken
+	EGM EncryptedGroupMessage
+}

server/app/main.go

@@ -1,11 +1,18 @@
 package main
 
 import (
+	"crypto/rand"
+	"cwtch.im/cwtch/model"
 	cwtchserver "cwtch.im/cwtch/server"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"encoding/base64"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
+	mrand "math/rand"
 	"os"
-	"path"
+	"os/signal"
+	"syscall"
+	"time"
 )
 
 const (
@@ -15,11 +22,44 @@ const (
 func main() {
 	log.AddEverythingFromPattern("server/app/main")
 	log.AddEverythingFromPattern("server/server")
+	log.SetLevel(log.LevelDebug)
 	configDir := os.Getenv("CWTCH_CONFIG_DIR")
 
+	if len(os.Args) == 2 && os.Args[1] == "gen1" {
+		config := new(cwtchserver.Config)
+		id, pk := primitives.InitializeEphemeralIdentity()
+		tid, tpk := primitives.InitializeEphemeralIdentity()
+		config.PrivateKey = pk
+		config.PublicKey = id.PublicKey()
+		config.TokenServerPrivateKey = tpk
+		config.TokenServerPublicKey = tid.PublicKey()
+		config.MaxBufferLines = 100000
+		config.ServerReporting = cwtchserver.Reporting{
+			LogMetricsToFile:    true,
+			ReportingGroupID:    "",
+			ReportingServerAddr: "",
+		}
+		config.Save(".", "serverConfig.json")
+		return
+	}
+
 	serverConfig := cwtchserver.LoadConfig(configDir, serverConfigFile)
 
-	acn, err := connectivity.StartTor(path.Join(configDir, "tor"), "")
+	// we don't need real randomness for the port, just to avoid a possible conflict...
+	mrand.Seed(int64(time.Now().Nanosecond()))
+	controlPort := mrand.Intn(1000) + 9052
+
+	// generate a random password
+	key := make([]byte, 64)
+	_, err := rand.Read(key)
+	if err != nil {
+		panic(err)
+	}
+
+	os.MkdirAll("tordir/tor", 0700)
+	tor.NewTorrc().WithHashedPassword(base64.StdEncoding.EncodeToString(key)).WithControlPort(controlPort).Build("./tordir/tor/torrc")
+	acn, err := tor.NewTorACNWithAuth("tordir", "", controlPort, tor.HashedPasswordAuthenticator{Password: base64.StdEncoding.EncodeToString(key)})
+
 	if err != nil {
 		log.Errorf("\nError connecting to Tor: %v\n", err)
 		os.Exit(1)
@@ -29,7 +69,32 @@ func main() {
 	server := new(cwtchserver.Server)
 	log.Infoln("starting cwtch server...")
 
-	// TODO load params from .cwtch/server.conf or command line flag
-	// TODO: respond to HUP so t.Close is gracefully called
-	server.Run(acn, serverConfig)
+	server.Setup(serverConfig)
+
+	// TODO create a random group for testing
+	group, _ := model.NewGroup(tor.GetTorV3Hostname(serverConfig.PublicKey))
+	invite, err := group.Invite()
+	if err != nil {
+		panic(err)
+	}
+
+	bundle := server.KeyBundle().Serialize()
+	log.Infof("Server Config: server:%s", base64.StdEncoding.EncodeToString(bundle))
+
+	log.Infof("Server Tofu Bundle: tofubundle:server:%s||%s", base64.StdEncoding.EncodeToString(bundle), invite)
+
+	// Graceful Shutdown
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		acn.Close()
+		server.Close()
+		os.Exit(1)
+	}()
+
+	server.Run(acn)
+	for {
+		time.Sleep(time.Second)
+	}
 }

server/fetch/server_fetch_channel.go

@@ -1,100 +0,0 @@
-package fetch
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchServerFetchChannel implements the ChannelHandler interface for a channel of
-// type "im.cwtch.server.fetch" - this implementation only handles server side logic.
-type CwtchServerFetchChannel struct {
-	Handler CwtchServerFetchHandler
-	channel *channels.Channel
-}
-
-// CwtchServerFetchHandler defines the interface for interacting with this Channel
-type CwtchServerFetchHandler interface {
-	HandleFetchRequest() []*protocol.GroupMessage
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.Cwtch".
-func (cc *CwtchServerFetchChannel) Type() string {
-	return "im.cwtch.server.fetch"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cc *CwtchServerFetchChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch channels any side can open
-func (cc *CwtchServerFetchChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cc *CwtchServerFetchChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cc *CwtchServerFetchChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require hidden service auth
-func (cc *CwtchServerFetchChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound is the first method called for an inbound channel request.
-// If an error is returned, the channel is rejected. If a RawMessage is
-// returned, it will be sent as the ChannelResult message.
-func (cc *CwtchServerFetchChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	cc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.AckOpenChannel(channel.ID), nil
-}
-
-// OpenOutbound is the first method called for an outbound channel request.
-// If an error is returned, the channel is not opened. If a RawMessage is
-// returned, it will be sent as the OpenChannel message.
-func (cc *CwtchServerFetchChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	return nil, errors.New("server does not open Fetch channels")
-}
-
-// OpenOutboundResult is called when a response is received for an
-// outbound OpenChannel request. If `err` is non-nil, the channel was
-// rejected and Closed will be called immediately afterwards. `raw`
-// contains the raw protocol message including any extension data.
-func (cc *CwtchServerFetchChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	// NOTE: Should never be called
-}
-
-// SendGroupMessages sends a series of group messages to the client.
-func (cc *CwtchServerFetchChannel) SendGroupMessages(gms []*protocol.GroupMessage) {
-	for _, gm := range gms {
-		csp := &protocol.CwtchServerPacket{
-			GroupMessage: gm,
-		}
-		packet, _ := proto.Marshal(csp)
-		cc.channel.SendMessage(packet)
-	}
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cc *CwtchServerFetchChannel) Packet(data []byte) {
-	csp := &protocol.CwtchServerPacket{}
-	err := proto.Unmarshal(data, csp)
-	if err == nil {
-		if csp.GetFetchMessage() != nil {
-			cc.SendGroupMessages(cc.Handler.HandleFetchRequest())
-		}
-	}
-	// If we receive a packet on this channel, close the connection
-	cc.channel.CloseChannel()
-}

server/fetch/server_fetch_channel_test.go

@@ -1,109 +0,0 @@
-package fetch
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-func TestServerFetchChannelAttributes(t *testing.T) {
-	cslc := new(CwtchServerFetchChannel)
-	if cslc.Type() != "im.cwtch.server.fetch" {
-		t.Errorf("cwtch channel type is incorrect %v", cslc.Type())
-	}
-
-	if !cslc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.fetch channel")
-	}
-
-	if cslc.Bidirectional() {
-		t.Errorf("im.cwtch.server.fetch should not be bidirectional")
-	}
-
-	if !cslc.Singleton() {
-		t.Errorf("im.cwtch.server.fetch should be a Singleton")
-	}
-
-	if cslc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cslc.RequiresAuthentication())
-	}
-
-}
-func TestServerFetchChannelOpenOutbound(t *testing.T) {
-	cslc := new(CwtchServerFetchChannel)
-	channel := new(channels.Channel)
-	_, err := cslc.OpenOutbound(channel)
-	if err == nil {
-		t.Errorf("server implementation of im.cwtch.server.fetch should never open an outbound channel")
-	}
-}
-
-type TestHandler struct {
-}
-
-func (th *TestHandler) HandleFetchRequest() []*protocol.GroupMessage {
-	gm := &protocol.GroupMessage{
-		Ciphertext: []byte("Hello"),
-		Spamguard:  []byte{},
-	}
-	return []*protocol.GroupMessage{gm}
-}
-
-func TestServerFetchChannel(t *testing.T) {
-	cslc := new(CwtchServerFetchChannel)
-	th := new(TestHandler)
-	cslc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 1
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-	gotgm := false
-	channel.SendMessage = func([]byte) {
-		gotgm = true
-	}
-
-	oc := &Protocol_Data_Control.OpenChannel{
-		ChannelIdentifier: proto.Int32(1),
-		ChannelType:       proto.String(cslc.Type()),
-	}
-
-	resp, err := cslc.OpenInbound(channel, oc)
-	if err != nil {
-		t.Errorf("OpenInbound for im.cwtch.server.Fetch should have succeeded, instead: %v", err)
-	}
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(resp[:], control)
-
-	if control.GetChannelResult() != nil {
-
-		fm := &protocol.FetchMessage{}
-
-		csp := &protocol.CwtchServerPacket{
-			FetchMessage: fm,
-		}
-
-		packet, _ := proto.Marshal(csp)
-		cslc.Packet(packet)
-
-		if !gotgm {
-			t.Errorf("Did not receive packet on wire as expected in Fetch channel")
-		}
-
-		if !closed {
-			t.Errorf("Fetch channel should be cosed")
-		}
-
-		if !closed {
-			t.Errorf("Fetch channel should be closed after incorrect packet received")
-		}
-
-	} else {
-		t.Errorf("Expected ChannelResult from im.cwtch.server.Fetch, instead: %v", control)
-	}
-
-}

server/listen/server_listen_channel.go

@@ -1,85 +0,0 @@
-package listen
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchServerListenChannel implements the ChannelHandler interface for a channel of
-// type "im.cwtch.server.listen" - this implementation only handles server side logic.
-type CwtchServerListenChannel struct {
-	channel *channels.Channel
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.Cwtch".
-func (cc *CwtchServerListenChannel) Type() string {
-	return "im.cwtch.server.listen"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cc *CwtchServerListenChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch channels any side can open
-func (cc *CwtchServerListenChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cc *CwtchServerListenChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cc *CwtchServerListenChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require hidden service auth
-func (cc *CwtchServerListenChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound is the first method called for an inbound channel request.
-// If an error is returned, the channel is rejected. If a RawMessage is
-// returned, it will be sent as the ChannelResult message.
-func (cc *CwtchServerListenChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	cc.channel = channel
-	messageBuilder := new(utils.MessageBuilder)
-	return messageBuilder.AckOpenChannel(channel.ID), nil
-}
-
-// OpenOutbound is the first method called for an outbound channel request.
-// If an error is returned, the channel is not opened. If a RawMessage is
-// returned, it will be sent as the OpenChannel message.
-func (cc *CwtchServerListenChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	return nil, errors.New("server does not open listen channels")
-}
-
-// OpenOutboundResult is called when a response is received for an
-// outbound OpenChannel request. If `err` is non-nil, the channel was
-// rejected and Closed will be called immediately afterwards. `raw`
-// contains the raw protocol message including any extension data.
-func (cc *CwtchServerListenChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	// NOTE: Should never be called
-}
-
-// SendGroupMessage sends a single group message to the peer
-func (cc *CwtchServerListenChannel) SendGroupMessage(gm *protocol.GroupMessage) {
-	csp := &protocol.CwtchServerPacket{
-		GroupMessage: gm,
-	}
-	packet, _ := proto.Marshal(csp)
-	cc.channel.SendMessage(packet)
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cc *CwtchServerListenChannel) Packet(data []byte) {
-	// If we receive a packet on this channel, close the connection
-	cc.channel.CloseChannel()
-}

server/listen/server_listen_channel_test.go

@@ -1,94 +0,0 @@
-package listen
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-func TestServerListenChannelAttributes(t *testing.T) {
-	cslc := new(CwtchServerListenChannel)
-	if cslc.Type() != "im.cwtch.server.listen" {
-		t.Errorf("cwtch channel type is incorrect %v", cslc.Type())
-	}
-
-	if !cslc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.listen channel")
-	}
-
-	if cslc.Bidirectional() {
-		t.Errorf("im.cwtch.server.listen should not be bidirectional")
-	}
-
-	if !cslc.Singleton() {
-		t.Errorf("im.cwtch.server.listen should be a Singleton")
-	}
-
-	if cslc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cslc.RequiresAuthentication())
-	}
-
-}
-func TestServerListenChannelOpenOutbound(t *testing.T) {
-	cslc := new(CwtchServerListenChannel)
-	channel := new(channels.Channel)
-	_, err := cslc.OpenOutbound(channel)
-	if err == nil {
-		t.Errorf("server implementation of im.cwtch.server.listen should never open an outbound channel")
-	}
-}
-
-func TestServerListenChannel(t *testing.T) {
-	cslc := new(CwtchServerListenChannel)
-	channel := new(channels.Channel)
-	channel.ID = 1
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-	gotgm := false
-	channel.SendMessage = func([]byte) {
-		gotgm = true
-	}
-
-	oc := &Protocol_Data_Control.OpenChannel{
-		ChannelIdentifier: proto.Int32(1),
-		ChannelType:       proto.String(cslc.Type()),
-	}
-
-	resp, err := cslc.OpenInbound(channel, oc)
-	if err != nil {
-		t.Errorf("OpenInbound for im.cwtch.server.listen should have succeeded, instead: %v", err)
-	}
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(resp[:], control)
-
-	if control.GetChannelResult() != nil {
-		gm := &protocol.GroupMessage{
-			Ciphertext: []byte("Hello"),
-			Signature:  []byte{},
-			Spamguard:  []byte{},
-		}
-		cslc.SendGroupMessage(gm)
-		if !gotgm {
-			t.Errorf("Did not receive packet on wire as expected in listen channel")
-		}
-
-		if closed {
-			t.Errorf("listen channel should not be cosed")
-		}
-
-		cslc.Packet(nil)
-
-		if !closed {
-			t.Errorf("listen channel should be closed after incorrect packet received")
-		}
-
-	} else {
-		t.Errorf("Expected ChannelResult from im.cwtch.server.listen, instead: %v", control)
-	}
-
-}

server/metrics/metrics.go

@@ -12,6 +12,7 @@ import (
 type counter struct {
 	startTime time.Time
 	count     uint64
+	total     uint64
 }
 
 // Counter providers a threadsafe counter to use for storing long running counts
@@ -25,7 +26,7 @@ type Counter interface {
 
 // NewCounter initializes a counter starting at time.Now() and a count of 0 and returns it
 func NewCounter() Counter {
-	c := &counter{startTime: time.Now(), count: 0}
+	c := &counter{startTime: time.Now(), count: 0, total: 0}
 	return c
 }
 
@@ -151,11 +152,13 @@ func (mh *monitorHistory) Months() []float64 {
 }
 
 func (mh *monitorHistory) Report(w *bufio.Writer) {
+	mh.lock.Lock()
 	fmt.Fprintln(w, "Minutes:", reportLine(mh.monitorType, mh.perMinutePerHour[:]))
 	fmt.Fprintln(w, "Hours:  ", reportLine(mh.monitorType, mh.perHourForDay[:]))
 	fmt.Fprintln(w, "Days:   ", reportLine(mh.monitorType, mh.perDayForWeek[:]))
 	fmt.Fprintln(w, "Weeks:  ", reportLine(mh.monitorType, mh.perWeekForMonth[:]))
 	fmt.Fprintln(w, "Months: ", reportLine(mh.monitorType, mh.perMonthForYear[:]))
+	mh.lock.Unlock()
 }
 
 func reportLine(t MonitorType, array []float64) string {

server/metrics/metrics_test.go

@@ -9,13 +9,20 @@ func TestCounter(t *testing.T) {
 	starttime := time.Now()
 	c := NewCounter()
 
-	for i := 0; i < 100; i++ {
+	max := 100
+	done := make(chan bool, max)
+
+	// slightly stress test atomic nature of metric by flooding with threads Add()ing
+	for i := 0; i < max; i++ {
 		go func() {
 			c.Add(1)
+			done <- true
 		}()
 	}
 
-	time.Sleep(2 * time.Second)
+	for i := 0; i < max; i++ {
+		<-done
+	}
 
 	val := c.Count()
 	if val != 100 {
@@ -25,6 +32,6 @@ func TestCounter(t *testing.T) {
 	counterStart := c.GetStarttime()
 
 	if counterStart.Sub(starttime) > time.Millisecond {
-		t.Error("counter's starttime was innaccurate")
+		t.Errorf("counter's starttime was innaccurate %v", counterStart.Sub(starttime))
 	}
 }

server/metrics/monitors.go

@@ -3,11 +3,12 @@ package metrics
 import (
 	"bufio"
 	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/application"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/openprivacy/log"
 	"github.com/struCoder/pidusage"
 	"os"
 	"path"
+	"sync"
 	"time"
 )
 
@@ -17,28 +18,51 @@ const (
 
 // Monitors is a package of metrics for a Cwtch Server including message count, CPU, Mem, and conns
 type Monitors struct {
-	MessageCounter Counter
-	Messages       MonitorHistory
-	CPU            MonitorHistory
-	Memory         MonitorHistory
-	ClientConns    MonitorHistory
-	starttime      time.Time
-	breakChannel   chan bool
-	log            bool
-	configDir      string
+	MessageCounter      Counter
+	TotalMessageCounter Counter
+	Messages            MonitorHistory
+	CPU                 MonitorHistory
+	Memory              MonitorHistory
+	ClientConns         MonitorHistory
+	starttime           time.Time
+	breakChannel        chan bool
+	log                 bool
+	configDir           string
 }
 
 // Start initializes a Monitors's monitors
-func (mp *Monitors) Start(ra *application.RicochetApplication, configDir string, log bool) {
+func (mp *Monitors) Start(ts tapir.Service, configDir string, log bool) {
 	mp.log = log
 	mp.configDir = configDir
 	mp.starttime = time.Now()
 	mp.breakChannel = make(chan bool)
 	mp.MessageCounter = NewCounter()
-	mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) { c = float64(mp.MessageCounter.Count()); mp.MessageCounter.Reset(); return })
-	mp.CPU = NewMonitorHistory(Percent, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.CPU) })
-	mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 { sysInfo, _ := pidusage.GetStat(os.Getpid()); return float64(sysInfo.Memory) })
-	mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ra.ConnectionCount()) })
+
+	// Maintain a count of total messages
+	mp.TotalMessageCounter = NewCounter()
+	mp.Messages = NewMonitorHistory(Count, Cumulative, func() (c float64) {
+		c = float64(mp.MessageCounter.Count())
+		mp.TotalMessageCounter.Add(int(c))
+		mp.MessageCounter.Reset()
+		return
+	})
+
+	var pidUsageLock sync.Mutex
+	mp.CPU = NewMonitorHistory(Percent, Average, func() float64 {
+		pidUsageLock.Lock()
+		defer pidUsageLock.Unlock()
+		sysInfo, _ := pidusage.GetStat(os.Getpid())
+		return float64(sysInfo.CPU)
+	})
+	mp.Memory = NewMonitorHistory(MegaBytes, Average, func() float64 {
+		pidUsageLock.Lock()
+		defer pidUsageLock.Unlock()
+		sysInfo, _ := pidusage.GetStat(os.Getpid())
+		return float64(sysInfo.Memory)
+	})
+
+	// TODO: replace with ts.
+	mp.ClientConns = NewMonitorHistory(Count, Average, func() float64 { return float64(ts.Metrics().ConnectionCount) })
 
 	if mp.log {
 		go mp.run()
@@ -68,7 +92,7 @@ func (mp *Monitors) report() {
 
 	fmt.Fprintf(w, "Uptime: %v\n\n", time.Now().Sub(mp.starttime))
 
-	fmt.Fprintln(w, "Messages:")
+	fmt.Fprintln(w, "messages:")
 	mp.Messages.Report(w)
 
 	fmt.Fprintln(w, "\nClient Connections:")

server/send/server_send_channel.go

@@ -1,99 +0,0 @@
-package send
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/spam"
-	"errors"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-)
-
-// CwtchServerSendChannel implements the ChannelHandler interface for a channel of
-// type "im.cwtch.server.send - this implementation only handles server-side logic.
-type CwtchServerSendChannel struct {
-	// Methods of Handler are called for Cwtch events on this channel
-	Handler   CwtchServerSendChannelHandler
-	channel   *channels.Channel
-	spamguard spam.Guard
-}
-
-// CwtchServerSendChannelHandler defines the interface needed to interact with this channel
-type CwtchServerSendChannelHandler interface {
-	HandleGroupMessage(*protocol.GroupMessage)
-}
-
-// Type returns the type string for this channel, e.g. "im.ricochet.Cwtch".
-func (cc *CwtchServerSendChannel) Type() string {
-	return "im.cwtch.server.send"
-}
-
-// Closed is called when the channel is closed for any reason.
-func (cc *CwtchServerSendChannel) Closed(err error) {
-
-}
-
-// OnlyClientCanOpen  - for Cwtch channels any side can open
-func (cc *CwtchServerSendChannel) OnlyClientCanOpen() bool {
-	return true
-}
-
-// Singleton - for Cwtch channels there can only be one instance per direction
-func (cc *CwtchServerSendChannel) Singleton() bool {
-	return true
-}
-
-// Bidirectional - for Cwtch channels are not bidrectional
-func (cc *CwtchServerSendChannel) Bidirectional() bool {
-	return false
-}
-
-// RequiresAuthentication - Cwtch channels require hidden service auth
-func (cc *CwtchServerSendChannel) RequiresAuthentication() string {
-	return "none"
-}
-
-// OpenInbound is the first method called for an inbound channel request.
-// If an error is returned, the channel is rejected. If a RawMessage is
-// returned, it will be sent as the ChannelResult message.
-func (cc *CwtchServerSendChannel) OpenInbound(channel *channels.Channel, raw *Protocol_Data_Control.OpenChannel) ([]byte, error) {
-	cc.channel = channel
-	cc.spamguard.Difficulty = 2
-	return cc.spamguard.GenerateChallenge(channel.ID), nil
-}
-
-// OpenOutbound is the first method called for an outbound channel request.
-// If an error is returned, the channel is not opened. If a RawMessage is
-// returned, it will be sent as the OpenChannel message.
-func (cc *CwtchServerSendChannel) OpenOutbound(channel *channels.Channel) ([]byte, error) {
-	return nil, errors.New("server does not open send channel")
-}
-
-// OpenOutboundResult is called when a response is received for an
-// outbound OpenChannel request. If `err` is non-nil, the channel was
-// rejected and Closed will be called immediately afterwards. `raw`
-// contains the raw protocol message including any extension data.
-func (cc *CwtchServerSendChannel) OpenOutboundResult(err error, crm *Protocol_Data_Control.ChannelResult) {
-	// NOTE: Should never be called
-}
-
-// Packet is called for each raw packet received on this channel.
-func (cc *CwtchServerSendChannel) Packet(data []byte) {
-	csp := &protocol.CwtchServerPacket{}
-	err := proto.Unmarshal(data, csp)
-	if err == nil {
-		if csp.GetGroupMessage() != nil {
-			gm := csp.GetGroupMessage()
-			ok := cc.spamguard.ValidateChallenge(gm.GetCiphertext(), gm.GetSpamguard())
-			if ok {
-				cc.Handler.HandleGroupMessage(gm)
-			} else {
-				log.Errorf("Failed to validate spamguard %v\n", gm)
-			}
-		}
-	} else {
-		log.Errorf("Failed to decode packet on SEND channel %v\n", err)
-	}
-	cc.channel.CloseChannel()
-}

server/send/server_send_channel_test.go

@@ -1,174 +0,0 @@
-package send
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/protocol/connections/spam"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/wire/control"
-	"github.com/golang/protobuf/proto"
-	"testing"
-)
-
-type TestHandler struct {
-	Received bool
-}
-
-func (th *TestHandler) HandleGroupMessage(m *protocol.GroupMessage) {
-	th.Received = true
-}
-
-func TestServerSendChannelAttributes(t *testing.T) {
-	cssc := new(CwtchServerSendChannel)
-	if cssc.Type() != "im.cwtch.server.send" {
-		t.Errorf("cwtch channel type is incorrect %v", cssc.Type())
-	}
-
-	if !cssc.OnlyClientCanOpen() {
-		t.Errorf("only clients should be able to open im.cwtch.server.send channel")
-	}
-
-	if cssc.Bidirectional() {
-		t.Errorf("im.cwtch.server.send should not be bidirectional")
-	}
-
-	if !cssc.Singleton() {
-		t.Errorf("im.cwtch.server.send should be a Singleton")
-	}
-
-	if cssc.RequiresAuthentication() != "none" {
-		t.Errorf("cwtch channel required auth is incorrect %v", cssc.RequiresAuthentication())
-	}
-
-}
-func TestServerSendChannelOpenOutbound(t *testing.T) {
-	cssc := new(CwtchServerSendChannel)
-	channel := new(channels.Channel)
-	_, err := cssc.OpenOutbound(channel)
-	if err == nil {
-		t.Errorf("server implementation of im.cwtch.server.send should never open an outbound channel")
-	}
-}
-
-func TestServerSendChannel(t *testing.T) {
-	cssc := new(CwtchServerSendChannel)
-	th := new(TestHandler)
-	cssc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 1
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-
-	oc := &Protocol_Data_Control.OpenChannel{
-		ChannelIdentifier: proto.Int32(1),
-		ChannelType:       proto.String(cssc.Type()),
-	}
-
-	resp, err := cssc.OpenInbound(channel, oc)
-	if err != nil {
-		t.Errorf("OpenInbound for im.cwtch.server.send should have succeeded, instead: %v", err)
-	}
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(resp[:], control)
-
-	if control.GetChannelResult() != nil {
-
-		var spamguard spam.Guard
-		spamguard.Difficulty = 2
-
-		ce, _ := proto.GetExtension(control.GetChannelResult(), protocol.E_ServerNonce)
-		challenge := ce.([]byte)[:]
-
-		sgsolve := spamguard.SolveChallenge(challenge, []byte("Hello"))
-		//t.Logf("Solved: %x",  sgsolve)
-
-		gm := &protocol.GroupMessage{
-			Ciphertext: []byte("Hello"),
-			Signature:  []byte{},
-			Spamguard:  sgsolve,
-		}
-
-		csp := &protocol.CwtchServerPacket{
-			GroupMessage: gm,
-		}
-		packet, _ := proto.Marshal(csp)
-
-		cssc.Packet(packet)
-
-		if !th.Received {
-			t.Errorf("group message should have been received")
-		}
-
-		if !closed {
-			t.Errorf("im.cwtch.server.send should have been closed after use")
-		}
-	} else {
-		t.Errorf("Expected ChannelResult from im.cwtch.server.send, instead: %v", control)
-	}
-
-}
-
-func TestServerSendChannelNoSpamGuard(t *testing.T) {
-	cssc := new(CwtchServerSendChannel)
-	th := new(TestHandler)
-	th.Received = false
-	cssc.Handler = th
-	channel := new(channels.Channel)
-	channel.ID = 1
-	closed := false
-	channel.CloseChannel = func() {
-		closed = true
-	}
-
-	oc := &Protocol_Data_Control.OpenChannel{
-		ChannelIdentifier: proto.Int32(1),
-		ChannelType:       proto.String(cssc.Type()),
-	}
-
-	resp, err := cssc.OpenInbound(channel, oc)
-	if err != nil {
-		t.Errorf("OpenInbound for im.cwtch.server.send should have succeeded, instead: %v", err)
-	}
-
-	control := new(Protocol_Data_Control.Packet)
-	proto.Unmarshal(resp[:], control)
-
-	if control.GetChannelResult() != nil {
-
-		var spamguard spam.Guard
-		spamguard.Difficulty = 2
-
-		ce, _ := proto.GetExtension(control.GetChannelResult(), protocol.E_ServerNonce)
-		challenge := ce.([]byte)[:]
-
-		sgsolve := spamguard.SolveChallenge(challenge, []byte("4234"))
-		//t.Logf("Solved: %x", sgsolve)
-
-		gm := &protocol.GroupMessage{
-			Ciphertext: []byte("hello"),
-			Signature:  []byte{},
-			Spamguard:  sgsolve,
-		}
-
-		csp := &protocol.CwtchServerPacket{
-			GroupMessage: gm,
-		}
-		packet, _ := proto.Marshal(csp)
-
-		cssc.Packet(packet)
-
-		if th.Received == true {
-			t.Errorf("group message should not have been received")
-		}
-
-		if !closed {
-			t.Errorf("im.cwtch.server.send should have been closed after use")
-		}
-
-	} else {
-		t.Errorf("Expected ChannelResult from im.cwtch.server.send, instead: %v", control)
-	}
-
-}

server/server.go

@@ -1,94 +1,161 @@
 package server
 
 import (
-	"cwtch.im/cwtch/server/fetch"
-	"cwtch.im/cwtch/server/listen"
+	"crypto/ed25519"
+	"cwtch.im/cwtch/model"
 	"cwtch.im/cwtch/server/metrics"
-	"cwtch.im/cwtch/server/send"
-	"cwtch.im/cwtch/storage"
-	"git.openprivacy.ca/openprivacy/libricochet-go/application"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-	"git.openprivacy.ca/openprivacy/libricochet-go/connectivity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/utils"
-
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"os"
-	"strconv"
-	"time"
+	"cwtch.im/cwtch/server/storage"
+	"fmt"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/applications"
+	tor2 "git.openprivacy.ca/cwtch.im/tapir/networks/tor"
+	"git.openprivacy.ca/cwtch.im/tapir/persistence"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives/privacypass"
+	"git.openprivacy.ca/openprivacy/connectivity"
+	"git.openprivacy.ca/openprivacy/connectivity/tor"
+	"git.openprivacy.ca/openprivacy/log"
+	"path"
+	"sync"
 )
 
 // Server encapsulates a complete, compliant Cwtch server.
 type Server struct {
-	app         *application.RicochetApplication
-	config      Config
-	metricsPack metrics.Monitors
+	service              tapir.Service
+	config               Config
+	metricsPack          metrics.Monitors
+	tokenTapirService    tapir.Service
+	tokenServer          *privacypass.TokenServer
+	tokenService         primitives.Identity
+	tokenServicePrivKey  ed25519.PrivateKey
+	tokenServiceStopped  bool
+	onionServiceStopped  bool
+	running              bool
+	existingMessageCount int
+	lock                 sync.RWMutex
+}
+
+// Setup initialized a server from a given configuration
+func (s *Server) Setup(serverConfig Config) {
+	s.config = serverConfig
+	bs := new(persistence.BoltPersistence)
+	bs.Open(path.Join(serverConfig.ConfigDir, "tokens.db"))
+	s.tokenServer = privacypass.NewTokenServerFromStore(&serverConfig.TokenServiceK, bs)
+	log.Infof("Y: %v", s.tokenServer.Y)
+	s.tokenService = s.config.TokenServiceIdentity()
+	s.tokenServicePrivKey = s.config.TokenServerPrivateKey
+}
+
+// Identity returns the main onion identity of the server
+func (s *Server) Identity() primitives.Identity {
+	return s.config.Identity()
 }
 
 // Run starts a server with the given privateKey
-// TODO: surface errors
-// TODO: handle HUP/KILL signals to exit and close Tor gracefully
-// TODO: handle user input to exit
-func (s *Server) Run(acn connectivity.ACN, serverConfig Config) {
-	s.config = serverConfig
-	cwtchserver := new(application.RicochetApplication)
-	s.metricsPack.Start(cwtchserver, serverConfig.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
+func (s *Server) Run(acn connectivity.ACN) error {
+	addressIdentity := tor.GetTorV3Hostname(s.config.PublicKey)
+	identity := primitives.InitializeIdentity("", &s.config.PrivateKey, &s.config.PublicKey)
+	var service tapir.Service
+	service = new(tor2.BaseOnionService)
+	service.Init(acn, s.config.PrivateKey, &identity)
+	s.service = service
+	log.Infof("cwtch server running on cwtch:%s\n", addressIdentity+".onion:")
+	s.metricsPack.Start(service, s.config.ConfigDir, s.config.ServerReporting.LogMetricsToFile)
 
-	af := application.ApplicationInstanceFactory{}
-	af.Init()
 	ms := new(storage.MessageStore)
-	err := ms.Init(serverConfig.ConfigDir, s.config.MaxBufferLines, s.metricsPack.MessageCounter)
+	err := ms.Init(s.config.ConfigDir, s.config.MaxBufferLines, s.metricsPack.MessageCounter)
 	if err != nil {
-		log.Errorln(err)
-		acn.Close()
-		os.Exit(1)
+		return err
 	}
-	af.AddHandler("im.cwtch.server.listen", func(rai *application.ApplicationInstance) func() channels.Handler {
-		return func() channels.Handler {
-			cslc := new(listen.CwtchServerListenChannel)
-			return cslc
-		}
-	})
 
-	af.AddHandler("im.cwtch.server.fetch", func(rai *application.ApplicationInstance) func() channels.Handler {
-		si := new(Instance)
-		si.Init(rai, cwtchserver, ms)
-		return func() channels.Handler {
-			cssc := new(fetch.CwtchServerFetchChannel)
-			cssc.Handler = si
-			return cssc
-		}
-	})
+	// Needed because we only collect metrics on a per-session basis
+	// TODO fix metrics so they persist across sessions?
+	s.existingMessageCount = len(ms.FetchMessages())
 
-	af.AddHandler("im.cwtch.server.send", func(rai *application.ApplicationInstance) func() channels.Handler {
-		si := new(Instance)
-		si.Init(rai, cwtchserver, ms)
-		return func() channels.Handler {
-			cssc := new(send.CwtchServerSendChannel)
-			cssc.Handler = si
-			return cssc
-		}
-	})
+	s.tokenTapirService = new(tor2.BaseOnionService)
+	s.tokenTapirService.Init(acn, s.tokenServicePrivKey, &s.tokenService)
+	tokenApplication := new(applications.TokenApplication)
+	tokenApplication.TokenService = s.tokenServer
+	powTokenApp := new(applications.ApplicationChain).
+		ChainApplication(new(applications.ProofOfWorkApplication), applications.SuccessfulProofOfWorkCapability).
+		ChainApplication(tokenApplication, applications.HasTokensCapability)
+	go func() {
+		s.tokenTapirService.Listen(powTokenApp)
+		s.tokenServiceStopped = true
+	}()
+	go func() {
+		s.service.Listen(NewTokenBoardServer(ms, s.tokenServer))
+		s.onionServiceStopped = true
+	}()
 
-	addressIdentity := utils.GetTorV3Hostname(s.config.PublicKey)
-	cwtchserver.Init(acn, "cwtch server for "+addressIdentity, s.config.Identity(), af, new(application.AcceptAllContactManager))
-	port := strconv.Itoa(application.RicochetPort)
-	log.Infof("cwtch server running on cwtch:%s\n", addressIdentity+".onion:"+port)
+	s.lock.Lock()
+	s.running = true
+	s.lock.Unlock()
+	return nil
+}
 
-	s.app = cwtchserver
+// KeyBundle provides the signed keybundle of the server
+func (s *Server) KeyBundle() *model.KeyBundle {
+	kb := model.NewKeyBundle()
+	identity := s.config.Identity()
+	kb.Keys[model.KeyTypeServerOnion] = model.Key(identity.Hostname())
+	kb.Keys[model.KeyTypeTokenOnion] = model.Key(s.tokenService.Hostname())
+	kb.Keys[model.KeyTypePrivacyPass] = model.Key(s.tokenServer.Y.String())
+	kb.Sign(identity)
+	return kb
+}
 
-	for true {
-		listenService, err := acn.Listen(s.config.PrivateKey, application.RicochetPort)
-		if err != nil {
-			log.Errorf("Listen() error setting up onion service: %v\n", err)
-		} else {
-			s.app.Run(listenService)
-		}
-		time.Sleep(5 * time.Second)
+// CheckStatus returns true if the server is running and/or an error if any part of the server needs to be restarted.
+func (s *Server) CheckStatus() (bool, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	if s.onionServiceStopped == true || s.tokenServiceStopped == true {
+		return s.running, fmt.Errorf("one of more server components are down: onion:%v token service: %v", s.onionServiceStopped, s.tokenServiceStopped)
 	}
+	return s.running, nil
 }
 
 // Shutdown kills the app closing all connections and freeing all goroutines
 func (s *Server) Shutdown() {
-	s.app.Shutdown()
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.service.Shutdown()
+	s.tokenTapirService.Shutdown()
 	s.metricsPack.Stop()
+	s.running = true
+
+}
+
+// Statistics is an encapsulation of information about the server that an operator might want to know at a glance.
+type Statistics struct {
+	TotalMessages int
+}
+
+// GetStatistics is a stub method for providing some high level information about
+// the server operation to bundling applications (e.g. the UI)
+func (s *Server) GetStatistics() Statistics {
+	// TODO Statistics from Metrics is very awkward. Metrics needs an overhaul to make safe
+	total := s.existingMessageCount
+	if s.metricsPack.TotalMessageCounter != nil {
+		total += s.metricsPack.TotalMessageCounter.Count()
+	}
+
+	return Statistics{
+		TotalMessages: total,
+	}
+}
+
+// ConfigureAutostart sets whether this server should autostart (in the Cwtch UI/bundling application)
+func (s *Server) ConfigureAutostart(autostart bool) {
+	s.config.AutoStart = autostart
+	s.config.Save(s.config.ConfigDir, s.config.FilePath)
+}
+
+// Close shuts down the cwtch server in a safe way.
+func (s *Server) Close() {
+	log.Infof("Shutting down server")
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	log.Infof("Closing Token Server Database...")
+	s.tokenServer.Close()
 }

server/serverConfig.go

@@ -3,8 +3,9 @@ package server
 import (
 	"crypto/rand"
 	"encoding/json"
-	"git.openprivacy.ca/openprivacy/libricochet-go/identity"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives"
+	"git.openprivacy.ca/openprivacy/log"
+	"github.com/gtank/ristretto255"
 	"golang.org/x/crypto/ed25519"
 	"io/ioutil"
 	"path"
@@ -19,16 +20,30 @@ type Reporting struct {
 
 // Config is a struct for storing basic server configuration
 type Config struct {
-	ConfigDir       string             `json:"-"`
-	MaxBufferLines  int                `json:"maxBufferLines"`
-	PublicKey       ed25519.PublicKey  `json:"publicKey"`
-	PrivateKey      ed25519.PrivateKey `json:"privateKey"`
-	ServerReporting Reporting          `json:"serverReporting"`
+	ConfigDir      string `json:"-"`
+	FilePath       string `json:"-"`
+	MaxBufferLines int    `json:"maxBufferLines"`
+
+	PublicKey  ed25519.PublicKey  `json:"publicKey"`
+	PrivateKey ed25519.PrivateKey `json:"privateKey"`
+
+	TokenServerPublicKey  ed25519.PublicKey  `json:"tokenServerPublicKey"`
+	TokenServerPrivateKey ed25519.PrivateKey `json:"tokenServerPrivateKey"`
+
+	TokenServiceK ristretto255.Scalar `json:"tokenServiceK"`
+
+	ServerReporting Reporting `json:"serverReporting"`
+	AutoStart       bool      `json:"autostart"`
 }
 
-// Identity returns an encapsulation of the servers keys for running ricochet
-func (config *Config) Identity() identity.Identity {
-	return identity.InitializeV3("", &config.PrivateKey, &config.PublicKey)
+// Identity returns an encapsulation of the servers keys
+func (config *Config) Identity() primitives.Identity {
+	return primitives.InitializeIdentity("", &config.PrivateKey, &config.PublicKey)
+}
+
+// TokenServiceIdentity returns an encapsulation of the servers token server (experimental)
+func (config *Config) TokenServiceIdentity() primitives.Identity {
+	return primitives.InitializeIdentity("", &config.TokenServerPrivateKey, &config.TokenServerPublicKey)
 }
 
 // Save dumps the latest version of the config to a json file given by filename
@@ -42,9 +57,33 @@ func (config *Config) Save(dir, filename string) {
 func LoadConfig(configDir, filename string) Config {
 	log.Infof("Loading config from %s\n", path.Join(configDir, filename))
 	config := Config{}
-	config.ConfigDir = configDir
+
+	id, pk := primitives.InitializeEphemeralIdentity()
+	tid, tpk := primitives.InitializeEphemeralIdentity()
+	config.PrivateKey = pk
+	config.PublicKey = id.PublicKey()
+	config.TokenServerPrivateKey = tpk
+	config.TokenServerPublicKey = tid.PublicKey()
 	config.MaxBufferLines = 100000
-	config.ServerReporting.LogMetricsToFile = false
+	config.ServerReporting = Reporting{
+		LogMetricsToFile:    true,
+		ReportingGroupID:    "",
+		ReportingServerAddr: "",
+	}
+	config.AutoStart = false
+	config.ConfigDir = configDir
+	config.FilePath = filename
+
+	k := new(ristretto255.Scalar)
+	b := make([]byte, 64)
+	_, err := rand.Read(b)
+	if err != nil {
+		// unable to generate secure random numbers
+		panic("unable to generate secure random numbers")
+	}
+	k.FromUniformBytes(b)
+	config.TokenServiceK = *k
+
 	raw, err := ioutil.ReadFile(path.Join(configDir, filename))
 	if err == nil {
 		err = json.Unmarshal(raw, &config)
@@ -53,11 +92,6 @@ func LoadConfig(configDir, filename string) Config {
 			log.Errorf("reading config: %v", err)
 		}
 	}
-
-	if config.PrivateKey == nil {
-		config.PublicKey, config.PrivateKey, _ = ed25519.GenerateKey(rand.Reader)
-	}
-
 	// Always save (first time generation, new version with new variables populated)
 	config.Save(configDir, filename)
 	return config

server/server_instance.go

@@ -1,45 +0,0 @@
-package server
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/server/listen"
-	"cwtch.im/cwtch/storage"
-	"git.openprivacy.ca/openprivacy/libricochet-go/application"
-	"git.openprivacy.ca/openprivacy/libricochet-go/channels"
-)
-
-// Instance encapsulates the Ricochet application.
-type Instance struct {
-	rai *application.ApplicationInstance
-	ra  *application.RicochetApplication
-	msi storage.MessageStoreInterface
-}
-
-// Init sets up a Server Instance
-func (si *Instance) Init(rai *application.ApplicationInstance, ra *application.RicochetApplication, msi storage.MessageStoreInterface) {
-	si.rai = rai
-	si.ra = ra
-	si.msi = msi
-}
-
-// HandleFetchRequest returns a list of all messages in the servers buffer
-func (si *Instance) HandleFetchRequest() []*protocol.GroupMessage {
-	return si.msi.FetchMessages()
-}
-
-// HandleGroupMessage takes in a group message and distributes it to all listening peers
-func (si *Instance) HandleGroupMessage(gm *protocol.GroupMessage) {
-	si.msi.AddMessage(*gm)
-	go si.ra.Broadcast(func(rai *application.ApplicationInstance) {
-		rai.Connection.Do(func() error {
-			channel := rai.Connection.Channel("im.cwtch.server.listen", channels.Inbound)
-			if channel != nil {
-				cslc, ok := channel.Handler.(*listen.CwtchServerListenChannel)
-				if ok {
-					cslc.SendGroupMessage(gm)
-				}
-			}
-			return nil
-		})
-	})
-}

server/server_instance_test.go

@@ -1,37 +0,0 @@
-package server
-
-import (
-	"cwtch.im/cwtch/protocol"
-	"cwtch.im/cwtch/server/metrics"
-	"cwtch.im/cwtch/storage"
-	"git.openprivacy.ca/openprivacy/libricochet-go/application"
-	"os"
-	"testing"
-	"time"
-)
-
-func TestServerInstance(t *testing.T) {
-	si := new(Instance)
-	ai := new(application.ApplicationInstance)
-	ra := new(application.RicochetApplication)
-	msi := new(storage.MessageStore)
-	os.RemoveAll("messages")
-	msi.Init(".", 5, metrics.NewCounter())
-	gm := protocol.GroupMessage{
-		Ciphertext: []byte("Hello this is a fairly average length message that we are writing here."),
-		Spamguard:  []byte{},
-	}
-
-	si.Init(ai, ra, msi)
-	msi.AddMessage(gm)
-	res := si.HandleFetchRequest()
-
-	if len(res) != 1 {
-		t.Errorf("Expected 1 Group Messages Instead got %v", res)
-	}
-
-	//	ra.HandleApplicationInstance(ai)
-	si.HandleGroupMessage(&gm)
-
-	time.Sleep(time.Second * 2)
-}

server/server_tokenboard.go

@@ -0,0 +1,118 @@
+package server
+
+import (
+	"cwtch.im/cwtch/protocol/groups"
+	"cwtch.im/cwtch/server/storage"
+	"encoding/json"
+	"git.openprivacy.ca/cwtch.im/tapir"
+	"git.openprivacy.ca/cwtch.im/tapir/applications"
+	"git.openprivacy.ca/cwtch.im/tapir/primitives/privacypass"
+	"git.openprivacy.ca/openprivacy/log"
+)
+
+// NewTokenBoardServer generates new Server for Token Board
+func NewTokenBoardServer(store storage.MessageStoreInterface, tokenService *privacypass.TokenServer) tapir.Application {
+	tba := new(TokenboardServer)
+	tba.TokenService = tokenService
+	tba.LegacyMessageStore = store
+	return tba
+}
+
+// TokenboardServer defines the token board server
+type TokenboardServer struct {
+	applications.AuthApp
+	connection         tapir.Connection
+	TokenService       *privacypass.TokenServer
+	LegacyMessageStore storage.MessageStoreInterface
+}
+
+// NewInstance creates a new TokenBoardApp
+func (ta *TokenboardServer) NewInstance() tapir.Application {
+	tba := new(TokenboardServer)
+	tba.TokenService = ta.TokenService
+	tba.LegacyMessageStore = ta.LegacyMessageStore
+	return tba
+}
+
+// Init initializes the cryptographic TokenBoardApp
+func (ta *TokenboardServer) Init(connection tapir.Connection) {
+	ta.AuthApp.Init(connection)
+	if connection.HasCapability(applications.AuthCapability) {
+		ta.connection = connection
+		go ta.Listen()
+	} else {
+		connection.Close()
+	}
+}
+
+// Listen processes the messages for this application
+func (ta *TokenboardServer) Listen() {
+	for {
+		data := ta.connection.Expect()
+		if len(data) == 0 {
+			log.Debugf("Server Closing Connection")
+			ta.connection.Close()
+			return // connection is closed
+		}
+
+		var message groups.Message
+		if err := json.Unmarshal(data, &message); err != nil {
+			log.Debugf("Server Closing Connection Because of Malformed Client Packet %v", err)
+			ta.connection.Close()
+			return // connection is closed
+		}
+
+		switch message.MessageType {
+		case groups.PostRequestMessage:
+			if message.PostRequest != nil {
+				postrequest := *message.PostRequest
+				log.Debugf("Received a Post Message Request: %v", ta.connection.Hostname())
+				ta.postMessageRequest(postrequest)
+			} else {
+				log.Debugf("Server Closing Connection Because of PostRequestMessage Client Packet")
+				ta.connection.Close()
+				return // connection is closed
+			}
+		case groups.ReplayRequestMessage:
+			if message.ReplayRequest != nil {
+				log.Debugf("Received Replay Request %v", message.ReplayRequest)
+				messages := ta.LegacyMessageStore.FetchMessages()
+				response, _ := json.Marshal(groups.Message{MessageType: groups.ReplayResultMessage, ReplayResult: &groups.ReplayResult{NumMessages: len(messages)}})
+				log.Debugf("Sending Replay Response %v", groups.ReplayResult{NumMessages: len(messages)})
+				ta.connection.Send(response)
+				for _, message := range messages {
+					data, _ = json.Marshal(message)
+					ta.connection.Send(data)
+				}
+				// Set sync and then send any new messages that might have happened while we were syncing
+				ta.connection.SetCapability(groups.CwtchServerSyncedCapability)
+				newMessages := ta.LegacyMessageStore.FetchMessages()
+				if len(newMessages) > len(messages) {
+					for _, message := range newMessages[len(messages):] {
+						data, _ = json.Marshal(groups.Message{MessageType: groups.NewMessageMessage, NewMessage: &groups.NewMessage{EGM: *message}})
+						ta.connection.Send(data)
+					}
+				}
+			} else {
+				log.Debugf("Server Closing Connection Because of Malformed ReplayRequestMessage Packet")
+				ta.connection.Close()
+				return // connection is closed
+			}
+		}
+	}
+}
+
+func (ta *TokenboardServer) postMessageRequest(pr groups.PostRequest) {
+	if err := ta.TokenService.SpendToken(pr.Token, append(pr.EGM.ToBytes(), ta.connection.ID().Hostname()...)); err == nil {
+		log.Debugf("Token is valid")
+		ta.LegacyMessageStore.AddMessage(pr.EGM)
+		data, _ := json.Marshal(groups.Message{MessageType: groups.PostResultMessage, PostResult: &groups.PostResult{Success: true}})
+		ta.connection.Send(data)
+		data, _ = json.Marshal(groups.Message{MessageType: groups.NewMessageMessage, NewMessage: &groups.NewMessage{EGM: pr.EGM}})
+		ta.connection.Broadcast(data, groups.CwtchServerSyncedCapability)
+	} else {
+		log.Debugf("Attempt to spend an invalid token: %v", err)
+		data, _ := json.Marshal(groups.Message{MessageType: groups.PostResultMessage, PostResult: &groups.PostResult{Success: false}})
+		ta.connection.Send(data)
+	}
+}

server/storage/message_store.go

@@ -2,11 +2,11 @@ package storage
 
 import (
 	"bufio"
-	"cwtch.im/cwtch/protocol"
+	"cwtch.im/cwtch/protocol/groups"
 	"cwtch.im/cwtch/server/metrics"
 	"encoding/json"
 	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
+	"git.openprivacy.ca/openprivacy/log"
 	"os"
 	"path"
 	"sync"
@@ -20,8 +20,8 @@ const (
 
 // MessageStoreInterface defines an interface to interact with a store of cwtch messages.
 type MessageStoreInterface interface {
-	AddMessage(protocol.GroupMessage)
-	FetchMessages() []*protocol.GroupMessage
+	AddMessage(groups.EncryptedGroupMessage)
+	FetchMessages() []*groups.EncryptedGroupMessage
 }
 
 // MessageStore is a file-backed implementation of MessageStoreInterface
@@ -30,7 +30,7 @@ type MessageStore struct {
 	filePos        int
 	storeDirectory string
 	lock           sync.Mutex
-	messages       []*protocol.GroupMessage
+	messages       []*groups.EncryptedGroupMessage
 	messageCounter metrics.Counter
 	maxBufferLines int
 	bufferPos      int
@@ -45,7 +45,7 @@ func (ms *MessageStore) Close() {
 	ms.lock.Unlock()
 }
 
-func (ms *MessageStore) updateBuffer(gm *protocol.GroupMessage) {
+func (ms *MessageStore) updateBuffer(gm *groups.EncryptedGroupMessage) {
 	ms.messages[ms.bufferPos] = gm
 	ms.bufferPos++
 	if ms.bufferPos == ms.maxBufferLines {
@@ -70,7 +70,7 @@ func (ms *MessageStore) initAndLoadFiles() error {
 		for scanner.Scan() {
 			gms := scanner.Text()
 			ms.filePos++
-			gm := &protocol.GroupMessage{}
+			gm := &groups.EncryptedGroupMessage{}
 			err := json.Unmarshal([]byte(gms), gm)
 			if err == nil {
 				ms.updateBuffer(gm)
@@ -83,7 +83,7 @@ func (ms *MessageStore) initAndLoadFiles() error {
 	return nil
 }
 
-func (ms *MessageStore) updateFile(gm *protocol.GroupMessage) {
+func (ms *MessageStore) updateFile(gm *groups.EncryptedGroupMessage) {
 	s, err := json.Marshal(gm)
 	if err != nil {
 		log.Errorf("Failed to unmarshal group message %v\n", err)
@@ -118,7 +118,7 @@ func (ms *MessageStore) Init(appDirectory string, maxBufferLines int, messageCou
 
 	ms.bufferPos = 0
 	ms.maxBufferLines = maxBufferLines
-	ms.messages = make([]*protocol.GroupMessage, maxBufferLines)
+	ms.messages = make([]*groups.EncryptedGroupMessage, maxBufferLines)
 	ms.bufferRotated = false
 	ms.messageCounter = messageCounter
 
@@ -127,13 +127,13 @@ func (ms *MessageStore) Init(appDirectory string, maxBufferLines int, messageCou
 }
 
 // FetchMessages returns all messages from the backing file.
-func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) {
+func (ms *MessageStore) FetchMessages() (messages []*groups.EncryptedGroupMessage) {
 	ms.lock.Lock()
 	if !ms.bufferRotated {
-		messages = make([]*protocol.GroupMessage, ms.bufferPos)
+		messages = make([]*groups.EncryptedGroupMessage, ms.bufferPos)
 		copy(messages, ms.messages[0:ms.bufferPos])
 	} else {
-		messages = make([]*protocol.GroupMessage, ms.maxBufferLines)
+		messages = make([]*groups.EncryptedGroupMessage, ms.maxBufferLines)
 		copy(messages, ms.messages[ms.bufferPos:ms.maxBufferLines])
 		copy(messages[ms.bufferPos:], ms.messages[0:ms.bufferPos])
 	}
@@ -142,7 +142,7 @@ func (ms *MessageStore) FetchMessages() (messages []*protocol.GroupMessage) {
 }
 
 // AddMessage adds a GroupMessage to the store
-func (ms *MessageStore) AddMessage(gm protocol.GroupMessage) {
+func (ms *MessageStore) AddMessage(gm groups.EncryptedGroupMessage) {
 	ms.messageCounter.Add(1)
 	ms.lock.Lock()
 	ms.updateBuffer(&gm)

server/storage/message_store_test.go

@@ -1,7 +1,7 @@
 package storage
 
 import (
-	"cwtch.im/cwtch/protocol"
+	"cwtch.im/cwtch/protocol/groups"
 	"cwtch.im/cwtch/server/metrics"
 	"os"
 	"strconv"
@@ -14,9 +14,8 @@ func TestMessageStore(t *testing.T) {
 	counter := metrics.NewCounter()
 	ms.Init("./", 1000, counter)
 	for i := 0; i < 499; i++ {
-		gm := protocol.GroupMessage{
+		gm := groups.EncryptedGroupMessage{
 			Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
-			Spamguard:  []byte{},
 		}
 		ms.AddMessage(gm)
 	}
@@ -33,9 +32,8 @@ func TestMessageStore(t *testing.T) {
 	counter.Reset()
 
 	for i := 0; i < 1000; i++ {
-		gm := protocol.GroupMessage{
+		gm := groups.EncryptedGroupMessage{
 			Ciphertext: []byte("Hello this is a fairly average length message that we are writing here. " + strconv.Itoa(i)),
-			Spamguard:  []byte{},
 		}
 		ms.AddMessage(gm)
 	}

storage/file_profile_store.go

@@ -1,109 +0,0 @@
-package storage
-
-import (
-	"crypto/rand"
-	"cwtch.im/cwtch/model"
-	"cwtch.im/cwtch/peer"
-	"encoding/json"
-	"fmt"
-	"git.openprivacy.ca/openprivacy/libricochet-go/log"
-	"golang.org/x/crypto/nacl/secretbox"
-	"golang.org/x/crypto/pbkdf2"
-	"golang.org/x/crypto/sha3"
-	"io"
-	"io/ioutil"
-)
-
-// fileProfileStore stores a cwtchPeer in an encrypted file
-type fileProfileStore struct {
-	profilefile string
-	password    string
-}
-
-// CreateFileProfileStore instantiates a fileProfileStore given a filename and a password
-func CreateFileProfileStore(profilefile string, password string) ProfileStore {
-	filestore := new(fileProfileStore)
-	filestore.password = password
-	filestore.profilefile = profilefile
-	return filestore
-}
-
-// Save serializes a cwtchPeer to a file
-func (fps *fileProfileStore) Save(cwtchPeer peer.CwtchPeer) error {
-
-	key, salt, _ := createKey(fps.password)
-	encryptedbytes, err := encryptProfile(cwtchPeer, key)
-	if err != nil {
-		return err
-	}
-
-	// the salt for the derived key is appended to the front of  the file
-	encryptedbytes = append(salt[:], encryptedbytes...)
-	err = ioutil.WriteFile(fps.profilefile, encryptedbytes, 0600)
-	return err
-
-}
-
-// createKey derives a key from a password
-func createKey(password string) ([32]byte, [128]byte, error) {
-	var salt [128]byte
-	if _, err := io.ReadFull(rand.Reader, salt[:]); err != nil {
-		log.Errorf("Cannot read from random: %v\n", err)
-		return [32]byte{}, salt, err
-	}
-	dk := pbkdf2.Key([]byte(password), salt[:], 4096, 32, sha3.New512)
-
-	var dkr [32]byte
-	copy(dkr[:], dk)
-	return dkr, salt, nil
-}
-
-//encryptProfile encrypts the cwtchPeer via the specified key.
-func encryptProfile(p peer.CwtchPeer, key [32]byte) ([]byte, error) {
-	var nonce [24]byte
-
-	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
-		log.Errorf("Cannot read from random: %v\n", err)
-		return nil, err
-	}
-
-	bytes, _ := json.Marshal(p.GetProfile())
-	encrypted := secretbox.Seal(nonce[:], []byte(bytes), &nonce, &key)
-	return encrypted, nil
-}
-
-//decryptProfile decrypts the passed ciphertext into a cwtchPeer via the specified key.
-func decryptProfile(ciphertext []byte, key [32]byte) (*model.Profile, error) {
-
-	var decryptNonce [24]byte
-	copy(decryptNonce[:], ciphertext[:24])
-	decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &key)
-	if ok {
-		cp := new(model.Profile)
-		err := json.Unmarshal(decrypted, &cp)
-		if err == nil {
-			return cp, nil
-		}
-		return nil, err
-	}
-	return nil, fmt.Errorf("Failed to decrypt")
-}
-
-// Load instantiates a cwtchPeer from the file store
-func (fps *fileProfileStore) Load() (peer.CwtchPeer, error) {
-	encryptedbytes, err := ioutil.ReadFile(fps.profilefile)
-	if err == nil {
-		var dkr [32]byte
-		//Separate the salt from the encrypted bytes, then generate the derived key
-		salt, encryptedbytes := encryptedbytes[0:128], encryptedbytes[128:]
-		dk := pbkdf2.Key([]byte(fps.password), salt, 4096, 32, sha3.New512)
-		copy(dkr[:], dk)
-
-		profile, err := decryptProfile(encryptedbytes, dkr)
-		if err == nil {
-			return peer.FromProfile(profile), nil
-		}
-		return nil, err
-	}
-	return nil, err
-}

storage/file_profile_store_test.go

@@ -1,22 +0,0 @@
-package storage
-
-import (
-	"cwtch.im/cwtch/peer"
-	"testing"
-)
-
-func TestFileProfileStore(t *testing.T) {
-	fileStore := CreateFileProfileStore(".test.json", "password")
-	alice := peer.NewCwtchPeer("alice")
-	fileStore.Save(alice)
-
-	aliceLoaded, err := fileStore.Load()
-
-	if err != nil {
-		t.Errorf("alice profile should have been loaded from store instead %v", err)
-	}
-
-	if aliceLoaded.GetProfile().Name != "alice" {
-		t.Errorf("alice profile should have been loaded from store instead %v", aliceLoaded)
-	}
-}

storage/profile_store.go

@@ -1,11 +1,93 @@
 package storage
 
 import (
-	"cwtch.im/cwtch/peer"
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
+	"cwtch.im/cwtch/storage/v0"
+	"cwtch.im/cwtch/storage/v1"
+	"git.openprivacy.ca/openprivacy/log"
+	"io/ioutil"
+	"path"
+	"strconv"
 )
 
+const profileFilename = "profile"
+const versionFile = "VERSION"
+const currentVersion = 1
+
 // ProfileStore is an interface to managing the storage of Cwtch Profiles
 type ProfileStore interface {
-	Save(cwtchPeer peer.CwtchPeer) error
-	Load() (peer.CwtchPeer, error)
+	Shutdown()
+	Delete()
+	GetProfileCopy(timeline bool) *model.Profile
+	GetNewPeerMessage() *event.Event
+	GetStatusMessages() []*event.Event
+}
+
+// CreateProfileWriterStore creates a profile store backed by a filestore listening for events and saving them
+// directory should be $appDir/profiles/$rand
+func CreateProfileWriterStore(eventManager event.Manager, directory, password string, profile *model.Profile) ProfileStore {
+	return v1.CreateProfileWriterStore(eventManager, directory, password, profile)
+}
+
+// LoadProfileWriterStore loads a profile store from filestore listening for events and saving them
+// directory should be $appDir/profiles/$rand
+func LoadProfileWriterStore(eventManager event.Manager, directory, password string) (ProfileStore, error) {
+	versionCheckUpgrade(directory, password)
+
+	return v1.LoadProfileWriterStore(eventManager, directory, password)
+}
+
+// ReadProfile reads a profile from storage and returns the profile
+// Should only be called for cache refresh of the profile after a ProfileWriterStore has opened
+//   (and upgraded) the store, and thus supplied the key/salt
+func ReadProfile(directory string, key [32]byte, salt [128]byte) (*model.Profile, error) {
+	return v1.ReadProfile(directory, key, salt)
+}
+
+// NewProfile creates a new profile for use in the profile store.
+func NewProfile(name string) *model.Profile {
+	profile := model.GenerateNewProfile(name)
+	return profile
+}
+
+// ********* Versioning and upgrade **********
+
+func detectVersion(directory string) int {
+	vnumberStr, err := ioutil.ReadFile(path.Join(directory, versionFile))
+	if err != nil {
+		return 0
+	}
+	vnumber, err := strconv.Atoi(string(vnumberStr))
+	if err != nil {
+		log.Errorf("Could not parse VERSION file contents: '%v' - %v\n", vnumber, err)
+		return -1
+	}
+	return vnumber
+}
+
+func upgradeV0ToV1(directory, password string) error {
+	log.Debugln("Attempting storage v0 to v1: Reading v0 profile...")
+	profile, err := v0.ReadProfile(directory, password)
+	if err != nil {
+		return err
+	}
+
+	log.Debugln("Attempting storage v0 to v1: Writing v1 profile...")
+	return v1.UpgradeV0Profile(profile, directory, password)
+}
+
+func versionCheckUpgrade(directory, password string) {
+	version := detectVersion(directory)
+	log.Debugf("versionCheck: %v\n", version)
+	if version == -1 {
+		return
+	}
+	if version == 0 {
+		err := upgradeV0ToV1(directory, password)
+		if err != nil {
+			return
+		}
+		//version = 1
+	}
 }

storage/profile_store_test.go

@@ -0,0 +1,76 @@
+// Known race issue with event bus channel closure
+
+package storage
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
+	"cwtch.im/cwtch/storage/v0"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/log"
+	"os"
+	"testing"
+	"time"
+)
+
+const testingDir = "./testing"
+const filenameBase = "testStream"
+const password = "asdfqwer"
+const line1 = "Hello from storage!"
+const testProfileName = "Alice"
+const testKey = "key"
+const testVal = "value"
+const testInitialMessage = "howdy"
+const testMessage = "Hello from storage"
+
+func TestProfileStoreUpgradeV0toV1(t *testing.T) {
+	log.SetLevel(log.LevelDebug)
+	os.RemoveAll(testingDir)
+	eventBus := event.NewEventManager()
+
+	queue := event.NewQueue()
+	eventBus.Subscribe(event.ChangePasswordSuccess, queue)
+
+	fmt.Println("Creating and initializing v0 profile and store...")
+	profile := NewProfile(testProfileName)
+	profile.AddContact("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd", &model.PublicProfile{Attributes: map[string]string{string(model.KeyTypeServerOnion): "2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd"}})
+
+	ps1 := v0.NewProfileWriterStore(eventBus, testingDir, password, profile)
+
+	groupid, invite, err := profile.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+	if err != nil {
+		t.Errorf("Creating group: %v\n", err)
+	}
+	if err != nil {
+		t.Errorf("Creating group invite: %v\n", err)
+	}
+
+	ps1.AddGroup(invite)
+
+	fmt.Println("Sending 200 messages...")
+
+	for i := 0; i < 200; i++ {
+		ps1.AddGroupMessage(groupid, time.Now().Format(time.RFC3339Nano), time.Now().Format(time.RFC3339Nano), profile.Onion, testMessage)
+	}
+
+	fmt.Println("Shutdown v0 profile store...")
+	ps1.Shutdown()
+
+	fmt.Println("New v1 Profile store...")
+	ps2, err := LoadProfileWriterStore(eventBus, testingDir, password)
+	if err != nil {
+		t.Errorf("Error createing new profileStore with new password: %v\n", err)
+		return
+	}
+
+	profile2 := ps2.GetProfileCopy(true)
+
+	if profile2.Groups[groupid] == nil {
+		t.Errorf("Failed to load group %v\n", groupid)
+		return
+	}
+
+	if len(profile2.Groups[groupid].Timeline.Messages) != 200 {
+		t.Errorf("Failed to load group's 200 messages, instead got %v\n", len(profile2.Groups[groupid].Timeline.Messages))
+	}
+}

storage/v0/file_enc.go

@@ -0,0 +1,70 @@
+package v0
+
+import (
+	"crypto/rand"
+	"errors"
+	"git.openprivacy.ca/openprivacy/log"
+	"golang.org/x/crypto/nacl/secretbox"
+	"golang.org/x/crypto/pbkdf2"
+	"golang.org/x/crypto/sha3"
+	"io"
+	"io/ioutil"
+	"path"
+)
+
+// createKey derives a key from a password
+func createKey(password string) ([32]byte, [128]byte, error) {
+	var salt [128]byte
+	if _, err := io.ReadFull(rand.Reader, salt[:]); err != nil {
+		log.Errorf("Cannot read from random: %v\n", err)
+		return [32]byte{}, salt, err
+	}
+	dk := pbkdf2.Key([]byte(password), salt[:], 4096, 32, sha3.New512)
+
+	var dkr [32]byte
+	copy(dkr[:], dk)
+	return dkr, salt, nil
+}
+
+//encryptFileData encrypts the cwtchPeer via the specified key.
+func encryptFileData(data []byte, key [32]byte) ([]byte, error) {
+	var nonce [24]byte
+
+	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
+		log.Errorf("Cannot read from random: %v\n", err)
+		return nil, err
+	}
+
+	encrypted := secretbox.Seal(nonce[:], data, &nonce, &key)
+	return encrypted, nil
+}
+
+//decryptFile decrypts the passed ciphertext into a cwtchPeer via the specified key.
+func decryptFile(ciphertext []byte, key [32]byte) ([]byte, error) {
+	var decryptNonce [24]byte
+	copy(decryptNonce[:], ciphertext[:24])
+	decrypted, ok := secretbox.Open(nil, ciphertext[24:], &decryptNonce, &key)
+	if ok {
+		return decrypted, nil
+	}
+	return nil, errors.New("Failed to decrypt")
+}
+
+// Load instantiates a cwtchPeer from the file store
+func readEncryptedFile(directory, filename, password string) ([]byte, error) {
+	encryptedbytes, err := ioutil.ReadFile(path.Join(directory, filename))
+	if err == nil && len(encryptedbytes) > 128 {
+		var dkr [32]byte
+		//Separate the salt from the encrypted bytes, then generate the derived key
+		salt, encryptedbytes := encryptedbytes[0:128], encryptedbytes[128:]
+		dk := pbkdf2.Key([]byte(password), salt, 4096, 32, sha3.New512)
+		copy(dkr[:], dk)
+
+		data, err := decryptFile(encryptedbytes, dkr)
+		if err == nil {
+			return data, nil
+		}
+		return nil, err
+	}
+	return nil, err
+}

storage/v0/file_store.go

@@ -0,0 +1,46 @@
+package v0
+
+import (
+	"io/ioutil"
+	"path"
+)
+
+// fileStore stores a cwtchPeer in an encrypted file
+type fileStore struct {
+	directory string
+	filename  string
+	password  string
+}
+
+// FileStore is a primitive around storing encrypted files
+type FileStore interface {
+	Read() ([]byte, error)
+	Write(data []byte) error
+}
+
+// NewFileStore instantiates a fileStore given a filename and a password
+func NewFileStore(directory string, filename string, password string) FileStore {
+	filestore := new(fileStore)
+	filestore.password = password
+	filestore.filename = filename
+	filestore.directory = directory
+	return filestore
+}
+
+func (fps *fileStore) Read() ([]byte, error) {
+	return readEncryptedFile(fps.directory, fps.filename, fps.password)
+}
+
+// write serializes a cwtchPeer to a file
+func (fps *fileStore) Write(data []byte) error {
+	key, salt, _ := createKey(fps.password)
+	encryptedbytes, err := encryptFileData(data, key)
+	if err != nil {
+		return err
+	}
+
+	// the salt for the derived key is appended to the front of  the file
+	encryptedbytes = append(salt[:], encryptedbytes...)
+	err = ioutil.WriteFile(path.Join(fps.directory, fps.filename), encryptedbytes, 0600)
+	return err
+}

storage/v0/profile_store.go

@@ -0,0 +1,120 @@
+package v0
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+)
+
+const groupIDLen = 32
+const peerIDLen = 56
+const profileFilename = "profile"
+
+// ProfileStoreV0 is a legacy profile store used now for upgrading legacy profile stores to newer versions
+type ProfileStoreV0 struct {
+	fs           FileStore
+	streamStores map[string]StreamStore // map [groupId|onion] StreamStore
+	directory    string
+	password     string
+	profile      *model.Profile
+}
+
+// NewProfileWriterStore returns a profile store backed by a filestore listening for events and saving them
+// directory should be $appDir/profiles/$rand
+func NewProfileWriterStore(eventManager event.Manager, directory, password string, profile *model.Profile) *ProfileStoreV0 {
+	os.Mkdir(directory, 0700)
+	ps := &ProfileStoreV0{fs: NewFileStore(directory, profileFilename, password), password: password, directory: directory, profile: profile, streamStores: map[string]StreamStore{}}
+	if profile != nil {
+		ps.save()
+	}
+
+	return ps
+}
+
+// ReadProfile reads a profile from storqage and returns the profile
+// directory should be $appDir/profiles/$rand
+func ReadProfile(directory, password string) (*model.Profile, error) {
+	os.Mkdir(directory, 0700)
+	ps := &ProfileStoreV0{fs: NewFileStore(directory, profileFilename, password), password: password, directory: directory, profile: nil, streamStores: map[string]StreamStore{}}
+
+	err := ps.Load()
+	if err != nil {
+		return nil, err
+	}
+
+	profile := ps.getProfileCopy(true)
+
+	return profile, nil
+}
+
+/********************************************************************************************/
+
+// AddGroup For testing, adds a group to the profile (and starts a stream store)
+func (ps *ProfileStoreV0) AddGroup(invite string) {
+	gid, err := ps.profile.ProcessInvite(invite)
+	if err == nil {
+		ps.save()
+		group := ps.profile.Groups[gid]
+		ps.streamStores[group.GroupID] = NewStreamStore(ps.directory, group.LocalID, ps.password)
+	}
+}
+
+// AddGroupMessage for testing, adds a group message
+func (ps *ProfileStoreV0) AddGroupMessage(groupid string, timeSent, timeRecvied string, remotePeer, data string) {
+	received, _ := time.Parse(time.RFC3339Nano, timeRecvied)
+	sent, _ := time.Parse(time.RFC3339Nano, timeSent)
+	message := model.Message{Received: received, Timestamp: sent, Message: data, PeerID: remotePeer, Signature: []byte("signature"), PreviousMessageSig: []byte("PreviousSignature")}
+	ss, exists := ps.streamStores[groupid]
+	if exists {
+		ss.Write(message)
+	} else {
+		fmt.Println("ERROR")
+	}
+}
+
+// GetNewPeerMessage is for AppService to call on Reload events, to reseed the AppClient with the loaded peers
+func (ps *ProfileStoreV0) GetNewPeerMessage() *event.Event {
+	message := event.NewEventList(event.NewPeer, event.Identity, ps.profile.LocalID, event.Password, ps.password, event.Status, "running")
+	return &message
+}
+
+// Load instantiates a cwtchPeer from the file store
+func (ps *ProfileStoreV0) Load() error {
+	decrypted, err := ps.fs.Read()
+	if err != nil {
+		return err
+	}
+	cp := new(model.Profile)
+	err = json.Unmarshal(decrypted, &cp)
+	if err == nil {
+		ps.profile = cp
+
+		for gid, group := range cp.Groups {
+			ss := NewStreamStore(ps.directory, group.LocalID, ps.password)
+
+			cp.Groups[gid].Timeline.SetMessages(ss.Read())
+			ps.streamStores[group.GroupID] = ss
+		}
+	}
+
+	return err
+}
+
+func (ps *ProfileStoreV0) getProfileCopy(timeline bool) *model.Profile {
+	return ps.profile.GetCopy(timeline)
+}
+
+// Shutdown saves the storage system
+func (ps *ProfileStoreV0) Shutdown() {
+	ps.save()
+}
+
+/************* Writing *************/
+
+func (ps *ProfileStoreV0) save() error {
+	bytes, _ := json.Marshal(ps.profile)
+	return ps.fs.Write(bytes)
+}

storage/v0/profile_store_test.go

@@ -0,0 +1,70 @@
+// Known race issue with event bus channel closure
+
+package v0
+
+import (
+	"cwtch.im/cwtch/event"
+	"cwtch.im/cwtch/model"
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+const testProfileName = "Alice"
+const testKey = "key"
+const testVal = "value"
+const testInitialMessage = "howdy"
+const testMessage = "Hello from storage"
+
+// NewProfile creates a new profile for use in the profile store.
+func NewProfile(name string) *model.Profile {
+	profile := model.GenerateNewProfile(name)
+	return profile
+}
+
+func TestProfileStoreWriteRead(t *testing.T) {
+	log.Println("profile store test!")
+	os.RemoveAll(testingDir)
+	eventBus := event.NewEventManager()
+	profile := NewProfile(testProfileName)
+	ps1 := NewProfileWriterStore(eventBus, testingDir, password, profile)
+
+	profile.SetAttribute(testKey, testVal)
+
+	groupid, invite, err := profile.StartGroup("2c3kmoobnyghj2zw6pwv7d57yzld753auo3ugauezzpvfak3ahc4bdyd")
+	if err != nil {
+		t.Errorf("Creating group: %v\n", err)
+	}
+	if err != nil {
+		t.Errorf("Creating group invite: %v\n", err)
+	}
+
+	ps1.AddGroup(invite)
+
+	ps1.AddGroupMessage(groupid, time.Now().Format(time.RFC3339Nano), time.Now().Format(time.RFC3339Nano), ps1.getProfileCopy(true).Onion, testMessage)
+
+	ps1.Shutdown()
+
+	ps2 := NewProfileWriterStore(eventBus, testingDir, password, nil)
+	err = ps2.Load()
+	if err != nil {
+		t.Errorf("Error createing ProfileStoreV0: %v\n", err)
+	}
+
+	profile = ps2.getProfileCopy(true)
+	if profile.Name != testProfileName {
+		t.Errorf("Profile name from loaded profile incorrect. Expected: '%v' Actual: '%v'\n", testProfileName, profile.Name)
+	}
+
+	v, _ := profile.GetAttribute(testKey)
+	if v != testVal {
+		t.Errorf("Profile attribute '%v' incorrect. Expected: '%v' Actual: '%v'\n", testKey, testVal, v)
+	}
+
+	group2 := ps2.getProfileCopy(true).Groups[groupid]
+	if group2 == nil {
+		t.Errorf("Group not loaded\n")
+	}
+
+}

storage/v0/stream_store.go

@@ -0,0 +1,145 @@
+package v0
+
+import (
+	"cwtch.im/cwtch/model"
+	"encoding/json"
+	"fmt"
+	"git.openprivacy.ca/openprivacy/log"
+	"io/ioutil"
+	"os"
+	"path"
+	"sync"
+)
+
+const (
+	fileStorePartitions = 16
+	bytesPerFile        = 15 * 1024
+)
+
+// streamStore is a file-backed implementation of StreamStore using an in memory buffer of ~16KB and a rotating set of files
+type streamStore struct {
+	password string
+
+	storeDirectory string
+	filenameBase   string
+
+	lock sync.Mutex
+
+	// Buffer is used just for current file to write to
+	messages        []model.Message
+	bufferByteCount int
+}
+
+// StreamStore provides a stream like interface to encrypted storage
+type StreamStore interface {
+	Read() []model.Message
+	Write(m model.Message)
+}
+
+// NewStreamStore returns an initialized StreamStore ready for reading and writing
+func NewStreamStore(directory string, filenameBase string, password string) (store StreamStore) {
+	ss := &streamStore{storeDirectory: directory, filenameBase: filenameBase, password: password}
+	os.Mkdir(ss.storeDirectory, 0700)
+
+	ss.initBuffer()
+
+	return ss
+}
+
+// Read returns all messages from the backing file (not the buffer, for writing to the current file)
+func (ss *streamStore) Read() (messages []model.Message) {
+	ss.lock.Lock()
+	defer ss.lock.Unlock()
+
+	resp := []model.Message{}
+
+	for i := fileStorePartitions - 1; i >= 0; i-- {
+		filename := fmt.Sprintf("%s.%d", ss.filenameBase, i)
+
+		bytes, err := readEncryptedFile(ss.storeDirectory, filename, ss.password)
+		if err != nil {
+			continue
+		}
+
+		msgs := []model.Message{}
+		json.Unmarshal([]byte(bytes), &msgs)
+		resp = append(resp, msgs...)
+	}
+
+	// 2019.10.10 "Acknowledged" & "ReceivedByServer" are added to the struct, populate it as true for old ones without
+	for i := 0; i < len(resp) && (resp[i].Acknowledged == false && resp[i].ReceivedByServer == false); i++ {
+		resp[i].Acknowledged = true
+		resp[i].ReceivedByServer = true
+	}
+
+	return resp
+}
+
+// ****** Writing *******/
+
+func (ss *streamStore) WriteN(messages []model.Message) {
+	ss.lock.Lock()
+	defer ss.lock.Unlock()
+
+	for _, m := range messages {
+		ss.updateBuffer(m)
+
+		if ss.bufferByteCount > bytesPerFile {
+			ss.updateFile()
+			log.Debugf("rotating log file")
+			ss.rotateFileStore()
+			ss.initBuffer()
+		}
+	}
+}
+
+// Write adds a GroupMessage to the store
+func (ss *streamStore) Write(m model.Message) {
+	ss.lock.Lock()
+	defer ss.lock.Unlock()
+	ss.updateBuffer(m)
+	ss.updateFile()
+
+	if ss.bufferByteCount > bytesPerFile {
+		log.Debugf("rotating log file")
+		ss.rotateFileStore()
+		ss.initBuffer()
+	}
+}
+
+func (ss *streamStore) initBuffer() {
+	ss.messages = []model.Message{}
+	ss.bufferByteCount = 0
+}
+
+func (ss *streamStore) updateBuffer(m model.Message) {
+	ss.messages = append(ss.messages, m)
+	ss.bufferByteCount += (model.MessageBaseSize * 1.5) + len(m.Message)
+}
+
+func (ss *streamStore) updateFile() error {
+	msgs, err := json.Marshal(ss.messages)
+	if err != nil {
+		log.Errorf("Failed to marshal group messages %v\n", err)
+	}
+
+	// ENCRYPT
+	key, salt, _ := createKey(ss.password)
+	encryptedMsgs, err := encryptFileData(msgs, key)
+	if err != nil {
+		log.Errorf("Failed to encrypt messages: %v\n", err)
+		return err
+	}
+	encryptedMsgs = append(salt[:], encryptedMsgs...)
+
+	ioutil.WriteFile(path.Join(ss.storeDirectory, fmt.Sprintf("%s.%d", ss.filenameBase, 0)), encryptedMsgs, 0700)
+	return nil
+}
+
+func (ss *streamStore) rotateFileStore() {
+	os.Remove(path.Join(ss.storeDirectory, fmt.Sprintf("%s.%d", ss.filenameBase, fileStorePartitions-1)))
+
+	for i := fileStorePartitions - 2; i >= 0; i-- {
+		os.Rename(path.Join(ss.storeDirectory, fmt.Sprintf("%s.%d", ss.filenameBase, i)), path.Join(ss.storeDirectory, fmt.Sprintf("%s.%d", ss.filenameBase, i+1)))
+	}
+}