From 271ecdb8eccc307a3d045129fa91a790a480f3c6 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 12:05:14 -0700 Subject: [PATCH 01/15] Fixup Token Requests --- peer/cwtch_peer.go | 1 - protocol/connections/engine.go | 10 +++---- protocol/connections/peerapp.go | 6 ++-- protocol/connections/tokenboardclientapp.go | 33 ++++++++++++--------- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/peer/cwtch_peer.go b/peer/cwtch_peer.go index 34260cc..f9e1d65 100644 --- a/peer/cwtch_peer.go +++ b/peer/cwtch_peer.go @@ -159,7 +159,6 @@ type ModifyServers interface { // SendMessages enables a caller to sender messages to a contact type SendMessages interface { - SendMessage(handle string, message string) error SendGetValToPeer(string, string, string) diff --git a/protocol/connections/engine.go b/protocol/connections/engine.go index c1f27c9..453df99 100644 --- a/protocol/connections/engine.go +++ b/protocol/connections/engine.go @@ -450,7 +450,7 @@ func (e *engine) sendGetValToPeer(eventID, onion, scope, path string) error { return err } - e.getValRequests.Store(onion + eventID, message) + e.getValRequests.Store(onion+eventID, message) return e.sendPeerMessage(onion, model3.PeerMessage{ID: eventID, Context: event.ContextGetVal, Data: message}) } @@ -519,16 +519,16 @@ func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, si func (e *engine) handlePeerMessage(hostname string, eventID string, context string, message []byte) { log.Debugf("New message from peer: %v %v", hostname, context) - if context == event.ContextAck { + if context == event.ContextAck { e.peerAck(hostname, eventID) } else if context == event.ContextRetVal { - req, ok := e.getValRequests.Load(hostname + eventID) + req, ok := e.getValRequests.Load(hostname + eventID) if ok { reqStr := req.([]byte) e.handlePeerRetVal(hostname, reqStr, message) - e.getValRequests.Delete(hostname + eventID) + e.getValRequests.Delete(hostname + eventID) } else { - log.Errorf("could not find val request for %v %s",hostname, eventID) + log.Errorf("could not find val request for %v %s", hostname, eventID) } } else if context == event.ContextGetVal { var getVal peerGetVal diff --git a/protocol/connections/peerapp.go b/protocol/connections/peerapp.go index 4f707b8..ec3deb5 100644 --- a/protocol/connections/peerapp.go +++ b/protocol/connections/peerapp.go @@ -79,9 +79,9 @@ func (pa *PeerApp) listen() { var peerMessage model2.PeerMessage err := json.Unmarshal(message, &peerMessage) if err == nil { - if pa.IsAllowed(pa.connection.Hostname()) { - pa.MessageHandler(pa.connection.Hostname(), peerMessage.ID, peerMessage.Context, peerMessage.Data) - } + if pa.IsAllowed(pa.connection.Hostname()) { + pa.MessageHandler(pa.connection.Hostname(), peerMessage.ID, peerMessage.Context, peerMessage.Data) + } } else { log.Errorf("Error unmarshalling PeerMessage package: %x %v", message, err) } diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index cb36cc8..8488ae9 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -72,7 +72,6 @@ func (ta *TokenBoardClient) Init(connection tapir.Connection) { go ta.Listen() // Optimistically acquire many tokens for this server... go ta.MakePayment() - go ta.MakePayment() ta.Replay() } else { connection.Close() @@ -179,20 +178,26 @@ func (ta *TokenBoardClient) MakePayment() error { powTokenApp := new(applications.ApplicationChain). ChainApplication(new(applications.ProofOfWorkApplication), applications.SuccessfulProofOfWorkCapability). ChainApplication(tokenApplication, applications.HasTokensCapability) - client.Connect(ta.tokenServiceOnion, powTokenApp) + log.Debugf("Waiting for successful PoW Auth...") - conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) - if err == nil { - powtapp, _ := conn.App().(*applications.TokenApplication) - // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens - // during this process.. - log.Debugf("Updating Tokens") - ta.tokenLock.Lock() - ta.tokens = append(ta.tokens, powtapp.Tokens...) - ta.tokenLock.Unlock() - log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) - conn.Close() - return nil + + connected, err := client.Connect(ta.tokenServiceOnion, powTokenApp) + if connected == true && err == nil { + conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) + if err == nil { + powtapp, ok := conn.App().(*applications.TokenApplication) + if ok { + // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens + // during this process.. + log.Debugf("Updating Tokens") + ta.tokenLock.Lock() + ta.tokens = append(ta.tokens, powtapp.Tokens...) + ta.tokenLock.Unlock() + log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) + conn.Close() + return nil + } + } } log.Debugf("Error making payment: to %v %v", ta.tokenServiceOnion, err) return err From 9254c9bc071daf003233785915b59c4adc594e0d Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 14:50:17 -0700 Subject: [PATCH 02/15] Upgrade Log to 1.0.3 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 31672ef..df99c63 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( git.openprivacy.ca/cwtch.im/tapir v0.4.5 git.openprivacy.ca/openprivacy/connectivity v1.4.5 - git.openprivacy.ca/openprivacy/log v1.0.2 + git.openprivacy.ca/openprivacy/log v1.0.3 github.com/gtank/ristretto255 v0.1.2 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/struCoder/pidusage v0.1.3 diff --git a/go.sum b/go.sum index 4c313c5..3bb35f4 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ git.openprivacy.ca/openprivacy/connectivity v1.4.5/go.mod h1:JVRCIdL+lAG6ohBFWiK git.openprivacy.ca/openprivacy/log v1.0.1/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw= git.openprivacy.ca/openprivacy/log v1.0.2 h1:HLP4wsw4ljczFAelYnbObIs821z+jgMPCe8uODPnGQM= git.openprivacy.ca/openprivacy/log v1.0.2/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw= +git.openprivacy.ca/openprivacy/log v1.0.3 h1:E/PMm4LY+Q9s3aDpfySfEDq/vYQontlvNj/scrPaga0= +git.openprivacy.ca/openprivacy/log v1.0.3/go.mod h1:gGYK8xHtndRLDymFtmjkG26GaMQNgyhioNS82m812Iw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 0f09a5ddcf842c75bc3ae5f20b2ba7f1f234164a Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 15:06:23 -0700 Subject: [PATCH 03/15] Maintain Tokens --- protocol/connections/tokenboardclientapp.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index 8488ae9..2f5c7e0 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -189,12 +189,15 @@ func (ta *TokenBoardClient) MakePayment() error { if ok { // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens // during this process.. + log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) + conn.Close() log.Debugf("Updating Tokens") ta.tokenLock.Lock() ta.tokens = append(ta.tokens, powtapp.Tokens...) + if len(ta.tokens) < 5 { + go ta.MakePayment() + } ta.tokenLock.Unlock() - log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) - conn.Close() return nil } } From 32c57a07aea0da07b1a2f07c231f7d0693cce516 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 15:15:17 -0700 Subject: [PATCH 04/15] Remove explicit MakePayment from SendMessageToGroup --- protocol/connections/engine.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/protocol/connections/engine.go b/protocol/connections/engine.go index 453df99..334a3f6 100644 --- a/protocol/connections/engine.go +++ b/protocol/connections/engine.go @@ -493,20 +493,18 @@ func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, si if err == nil { tokenApp, ok := (conn.App()).(*TokenBoardClient) if ok { - if spent, numtokens := tokenApp.Post(ct, sig); !spent { + if spent, _ := tokenApp.Post(ct, sig); !spent { // TODO: while this works for the spam guard, it won't work for other forms of payment... // Make an -inline- payment, this will hold the goroutine if err := tokenApp.MakePayment(); err == nil { // This really shouldn't fail since we now know we have the required tokens... - if spent, _ := tokenApp.Post(ct, sig); !spent { - e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: err.Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)})) + if spent, numTokens := tokenApp.Post(ct, sig); !spent { + e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: fmt.Errorf("could not post %v", numTokens).Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)})) } } else { // Broadcast the token error e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: err.Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)})) } - } else if numtokens < 5 { - go tokenApp.MakePayment() } // regardless we return.... return From 2dc4325e1f46708b3c024270c5f9e87d059d81ce Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 15:23:11 -0700 Subject: [PATCH 05/15] MakePayment inline to avoid connection contention --- protocol/connections/tokenboardclientapp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index 2f5c7e0..4244db7 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -71,7 +71,7 @@ func (ta *TokenBoardClient) Init(connection tapir.Connection) { log.Debugf("Successfully Initialized Connection") go ta.Listen() // Optimistically acquire many tokens for this server... - go ta.MakePayment() + ta.MakePayment() ta.Replay() } else { connection.Close() From 83bf6559a2657833ddd5b9d0ef722717f2b2e053 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 15:42:31 -0700 Subject: [PATCH 06/15] Use a different check for PoWApp casting --- protocol/connections/tokenboardclientapp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index 4244db7..df85e2b 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -185,8 +185,8 @@ func (ta *TokenBoardClient) MakePayment() error { if connected == true && err == nil { conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) if err == nil { - powtapp, ok := conn.App().(*applications.TokenApplication) - if ok { + powtapp := conn.App().(*applications.TokenApplication) + if powtapp != nil { // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens // during this process.. log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) From 76823517532f7c2f86387617a14587c1e77d0407 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 16:27:05 -0700 Subject: [PATCH 07/15] Check for Duplicates... --- protocol/connections/tokenboardclientapp.go | 44 +++++++++++---------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index df85e2b..36e120a 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -71,7 +71,7 @@ func (ta *TokenBoardClient) Init(connection tapir.Connection) { log.Debugf("Successfully Initialized Connection") go ta.Listen() // Optimistically acquire many tokens for this server... - ta.MakePayment() + go ta.MakePayment() ta.Replay() } else { connection.Close() @@ -181,25 +181,29 @@ func (ta *TokenBoardClient) MakePayment() error { log.Debugf("Waiting for successful PoW Auth...") - connected, err := client.Connect(ta.tokenServiceOnion, powTokenApp) - if connected == true && err == nil { - conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) - if err == nil { - powtapp := conn.App().(*applications.TokenApplication) - if powtapp != nil { - // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens - // during this process.. - log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) - conn.Close() - log.Debugf("Updating Tokens") - ta.tokenLock.Lock() - ta.tokens = append(ta.tokens, powtapp.Tokens...) - if len(ta.tokens) < 5 { - go ta.MakePayment() - } - ta.tokenLock.Unlock() - return nil - } + // We can't have duplicate outbounds... + conn, _ := client.GetConnection(ta.tokenServiceOnion) + for { + if conn == nil { + break + } + conn, _ = client.GetConnection(ta.tokenServiceOnion) + } + + client.Connect(ta.tokenServiceOnion, powTokenApp) + conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) + if err == nil { + powtapp, ok := conn.App().(*applications.TokenApplication) + if ok { + // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens + // during this process.. + log.Debugf("Updating Tokens") + ta.tokenLock.Lock() + ta.tokens = append(ta.tokens, powtapp.Tokens...) + ta.tokenLock.Unlock() + log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) + conn.Close() + return nil } } log.Debugf("Error making payment: to %v %v", ta.tokenServiceOnion, err) From b389a3421a4183a4f8ff145bd98b8bf7150028cf Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Thu, 9 Sep 2021 16:41:28 -0700 Subject: [PATCH 08/15] Fetch more Tokens --- protocol/connections/tokenboardclientapp.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index 36e120a..c57a953 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -71,7 +71,7 @@ func (ta *TokenBoardClient) Init(connection tapir.Connection) { log.Debugf("Successfully Initialized Connection") go ta.Listen() // Optimistically acquire many tokens for this server... - go ta.MakePayment() + ta.MakePayment() ta.Replay() } else { connection.Close() @@ -200,6 +200,9 @@ func (ta *TokenBoardClient) MakePayment() error { log.Debugf("Updating Tokens") ta.tokenLock.Lock() ta.tokens = append(ta.tokens, powtapp.Tokens...) + if len(ta.tokens) < 10 { + go ta.MakePayment() + } ta.tokenLock.Unlock() log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) conn.Close() From 38dd63abd981a022876c060fff91f90ac65742de Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 09:11:43 -0700 Subject: [PATCH 09/15] Don't automatically run file sharing integration test (for now) We either need to reduce the time these tests take or bump up drone timeouts --- .../file_sharing_integration_test.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) rename testing/{ => file_sharing}/file_sharing_integration_test.go (81%) diff --git a/testing/file_sharing_integration_test.go b/testing/file_sharing/file_sharing_integration_test.go similarity index 81% rename from testing/file_sharing_integration_test.go rename to testing/file_sharing/file_sharing_integration_test.go index 5d9e1d3..aca0473 100644 --- a/testing/file_sharing_integration_test.go +++ b/testing/file_sharing/file_sharing_integration_test.go @@ -1,4 +1,4 @@ -package testing +package file_sharing import ( "crypto/rand" @@ -7,6 +7,9 @@ import ( "cwtch.im/cwtch/event" "cwtch.im/cwtch/functionality/filesharing" "cwtch.im/cwtch/model" + "cwtch.im/cwtch/model/attr" + "cwtch.im/cwtch/peer" + "cwtch.im/cwtch/protocol/connections" "cwtch.im/cwtch/protocol/files" "encoding/base64" "encoding/hex" @@ -24,6 +27,29 @@ import ( "time" ) +func waitForPeerPeerConnection(t *testing.T, peera peer.CwtchPeer, peerb peer.CwtchPeer) { + for { + state, ok := peera.GetPeerState(peerb.GetOnion()) + if ok { + //log.Infof("Waiting for Peer %v to peer with peer: %v - state: %v\n", peera.GetProfile().Name, peerb.GetProfile().Name, state) + if state == connections.FAILED { + t.Fatalf("%v could not connect to %v", peera.GetOnion(), peerb.GetOnion()) + } + if state != connections.AUTHENTICATED { + fmt.Printf("peer %v waiting connect to peer %v, currently: %v\n", peera.GetOnion(), peerb.GetOnion(), connections.ConnectionStateName[state]) + time.Sleep(time.Second * 5) + continue + } else { + peerAName, _ := peera.GetAttribute(attr.GetLocalScope("name")) + peerBName, _ := peerb.GetAttribute(attr.GetLocalScope("name")) + fmt.Printf("%v CONNECTED and AUTHED to %v\n", peerAName, peerBName) + break + } + } + } + return +} + func TestFileSharing(t *testing.T) { numGoRoutinesStart := runtime.NumGoroutine() From b2646d0f8ec008e2632de766c11a4bab9cd19554 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 09:18:23 -0700 Subject: [PATCH 10/15] fmt --- .../file_sharing_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename testing/{file_sharing => filesharing}/file_sharing_integration_test.go (99%) diff --git a/testing/file_sharing/file_sharing_integration_test.go b/testing/filesharing/file_sharing_integration_test.go similarity index 99% rename from testing/file_sharing/file_sharing_integration_test.go rename to testing/filesharing/file_sharing_integration_test.go index aca0473..e02d3be 100644 --- a/testing/file_sharing/file_sharing_integration_test.go +++ b/testing/filesharing/file_sharing_integration_test.go @@ -1,4 +1,4 @@ -package file_sharing +package filesharing import ( "crypto/rand" From 0f71da3fb6a6a7b2c646d152b6a1a2e6abc051ff Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 09:52:53 -0700 Subject: [PATCH 11/15] Allow SendMessageToGroups to silently retry prior to failing --- protocol/connections/engine.go | 35 +++++++++++-------- protocol/connections/tokenboardclientapp.go | 37 ++++++++++++--------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/protocol/connections/engine.go b/protocol/connections/engine.go index 334a3f6..774e210 100644 --- a/protocol/connections/engine.go +++ b/protocol/connections/engine.go @@ -160,7 +160,7 @@ func (e *engine) eventHandler() { case event.SendMessageToGroup: ciphertext, _ := base64.StdEncoding.DecodeString(ev.Data[event.Ciphertext]) signature, _ := base64.StdEncoding.DecodeString(ev.Data[event.Signature]) - go e.sendMessageToGroup(ev.Data[event.GroupID], ev.Data[event.GroupServer], ciphertext, signature) + go e.sendMessageToGroup(ev.Data[event.GroupID], ev.Data[event.GroupServer], ciphertext, signature, 0) case event.SendMessageToPeer: // TODO: remove this passthrough once the UI is integrated. context, ok := ev.Data[event.EventContext] @@ -480,7 +480,17 @@ func (e *engine) receiveGroupMessage(server string, gm *groups.EncryptedGroupMes } // sendMessageToGroup attempts to sent the given message to the given group id. -func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, sig []byte) { +func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, sig []byte, attempts int) { + + // sending to groups can fail for a few reasons (slow server, not enough tokens, etc.) + // rather than trying to keep all that logic in method we simply back-off and try again + // but if we fail more than 5 times then we report back to the client so they can investigate other options. + // Note: This flow only applies to online-and-connected servers (this method will return faster if the server is not + // online) + if attempts >= 5 { + e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: "could not make payment to server", event.Signature: base64.StdEncoding.EncodeToString(sig)})) + return + } es, ok := e.ephemeralServices.Load(server) if es == nil || !ok { @@ -493,17 +503,16 @@ func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, si if err == nil { tokenApp, ok := (conn.App()).(*TokenBoardClient) if ok { - if spent, _ := tokenApp.Post(ct, sig); !spent { - // TODO: while this works for the spam guard, it won't work for other forms of payment... - // Make an -inline- payment, this will hold the goroutine - if err := tokenApp.MakePayment(); err == nil { - // This really shouldn't fail since we now know we have the required tokens... - if spent, numTokens := tokenApp.Post(ct, sig); !spent { - e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: fmt.Errorf("could not post %v", numTokens).Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)})) - } - } else { - // Broadcast the token error - e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: err.Error(), event.Signature: base64.StdEncoding.EncodeToString(sig)})) + if spent, numtokens := tokenApp.Post(ct, sig); !spent { + // we failed to post, probably because we ran out of tokens... so make a payment + go tokenApp.MakePayment() + // backoff + time.Sleep(time.Second) + // try again + e.sendMessageToGroup(groupID, server, ct, sig, attempts+1) + } else { + if numtokens < 5 { + go tokenApp.MakePayment() } } // regardless we return.... diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index c57a953..dfe6810 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -13,6 +13,7 @@ import ( "git.openprivacy.ca/openprivacy/log" "github.com/gtank/ristretto255" "sync" + "time" ) // NewTokenBoardClient generates a new Client for Token Board @@ -190,24 +191,28 @@ func (ta *TokenBoardClient) MakePayment() error { conn, _ = client.GetConnection(ta.tokenServiceOnion) } - client.Connect(ta.tokenServiceOnion, powTokenApp) - conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) - if err == nil { - powtapp, ok := conn.App().(*applications.TokenApplication) - if ok { - // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens - // during this process.. - log.Debugf("Updating Tokens") - ta.tokenLock.Lock() - ta.tokens = append(ta.tokens, powtapp.Tokens...) - if len(ta.tokens) < 10 { - go ta.MakePayment() + connected, err := client.Connect(ta.tokenServiceOnion, powTokenApp) + if connected == true && err == nil { + conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) + if err == nil { + powtapp, ok := conn.App().(*applications.TokenApplication) + if ok { + // Update tokens...we need a lock here to prevent SpendToken from modifying the tokens + // during this process.. + log.Debugf("Updating Tokens") + ta.tokenLock.Lock() + ta.tokens = append(ta.tokens, powtapp.Tokens...) + ta.tokenLock.Unlock() + log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) + conn.Close() + return nil } - ta.tokenLock.Unlock() - log.Debugf("Transcript: %v", powtapp.Transcript().OutputTranscriptToAudit()) - conn.Close() - return nil } + } else { + log.Debugf("failed to make a connection. trying again...") + // it doesn't actually take that long to make a payment, so waiting a small amount of time should suffice + time.Sleep(time.Second) + return ta.MakePayment() } log.Debugf("Error making payment: to %v %v", ta.tokenServiceOnion, err) return err From e2204a52dce4082d345c6841d9689af312608dc0 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 10:07:01 -0700 Subject: [PATCH 12/15] Simplify MakePayment --- protocol/connections/engine.go | 1 + protocol/connections/tokenboardclientapp.go | 12 ++---------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/protocol/connections/engine.go b/protocol/connections/engine.go index 774e210..a80c936 100644 --- a/protocol/connections/engine.go +++ b/protocol/connections/engine.go @@ -488,6 +488,7 @@ func (e *engine) sendMessageToGroup(groupID string, server string, ct []byte, si // Note: This flow only applies to online-and-connected servers (this method will return faster if the server is not // online) if attempts >= 5 { + log.Errorf("failed to post a message to a group after %v attempts", attempts) e.eventManager.Publish(event.NewEvent(event.SendMessageToGroupError, map[event.Field]string{event.GroupID: groupID, event.GroupServer: server, event.Error: "could not make payment to server", event.Signature: base64.StdEncoding.EncodeToString(sig)})) return } diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index dfe6810..8f8f4d3 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -72,7 +72,8 @@ func (ta *TokenBoardClient) Init(connection tapir.Connection) { log.Debugf("Successfully Initialized Connection") go ta.Listen() // Optimistically acquire many tokens for this server... - ta.MakePayment() + go ta.MakePayment() + go ta.MakePayment() ta.Replay() } else { connection.Close() @@ -182,15 +183,6 @@ func (ta *TokenBoardClient) MakePayment() error { log.Debugf("Waiting for successful PoW Auth...") - // We can't have duplicate outbounds... - conn, _ := client.GetConnection(ta.tokenServiceOnion) - for { - if conn == nil { - break - } - conn, _ = client.GetConnection(ta.tokenServiceOnion) - } - connected, err := client.Connect(ta.tokenServiceOnion, powTokenApp) if connected == true && err == nil { conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) From e967f151ee27e57e28f314e506fcf9a30b5378d0 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 10:18:54 -0700 Subject: [PATCH 13/15] increased error reporting --- protocol/connections/tokenboardclientapp.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index 8f8f4d3..c365365 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -199,15 +199,16 @@ func (ta *TokenBoardClient) MakePayment() error { conn.Close() return nil } + log.Errorf("invalid case of powapp. this should not happen") + return errors.New("invalid cast of powapp. this should never happen") } - } else { - log.Debugf("failed to make a connection. trying again...") - // it doesn't actually take that long to make a payment, so waiting a small amount of time should suffice - time.Sleep(time.Second) + log.Debugf("could not connect to payment server %v..trying again") return ta.MakePayment() } - log.Debugf("Error making payment: to %v %v", ta.tokenServiceOnion, err) - return err + log.Debugf("failed to make a connection. trying again...") + // it doesn't actually take that long to make a payment, so waiting a small amount of time should suffice + time.Sleep(time.Second) + return ta.MakePayment() } // NextToken retrieves the next token From d20d4b18c2c047d524e728392623592d35236717 Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 11:02:25 -0700 Subject: [PATCH 14/15] deep logging --- protocol/connections/tokenboardclientapp.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/protocol/connections/tokenboardclientapp.go b/protocol/connections/tokenboardclientapp.go index c365365..1ca69fe 100644 --- a/protocol/connections/tokenboardclientapp.go +++ b/protocol/connections/tokenboardclientapp.go @@ -12,6 +12,7 @@ import ( "git.openprivacy.ca/openprivacy/connectivity" "git.openprivacy.ca/openprivacy/log" "github.com/gtank/ristretto255" + "reflect" "sync" "time" ) @@ -185,6 +186,7 @@ func (ta *TokenBoardClient) MakePayment() error { connected, err := client.Connect(ta.tokenServiceOnion, powTokenApp) if connected == true && err == nil { + log.Debugf("Waiting for successful Token Acquisition...") conn, err := client.WaitForCapabilityOrClose(ta.tokenServiceOnion, applications.HasTokensCapability) if err == nil { powtapp, ok := conn.App().(*applications.TokenApplication) @@ -199,11 +201,13 @@ func (ta *TokenBoardClient) MakePayment() error { conn.Close() return nil } - log.Errorf("invalid case of powapp. this should not happen") + log.Errorf("invalid cast of powapp. this should not happen %v %v", powtapp, reflect.TypeOf(conn.App())) return errors.New("invalid cast of powapp. this should never happen") } log.Debugf("could not connect to payment server %v..trying again") return ta.MakePayment() + } else if connected && err != nil { + log.Debugf("inexplicable error: %v", err) } log.Debugf("failed to make a connection. trying again...") // it doesn't actually take that long to make a payment, so waiting a small amount of time should suffice From e4aac53622775ef3b5487d969360ae92350c42ee Mon Sep 17 00:00:00 2001 From: Sarah Jamie Lewis Date: Fri, 10 Sep 2021 11:23:45 -0700 Subject: [PATCH 15/15] Upgrade Tapir to 0.4.6 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index df99c63..43dc8b8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module cwtch.im/cwtch go 1.14 require ( - git.openprivacy.ca/cwtch.im/tapir v0.4.5 + git.openprivacy.ca/cwtch.im/tapir v0.4.6 git.openprivacy.ca/openprivacy/connectivity v1.4.5 git.openprivacy.ca/openprivacy/log v1.0.3 github.com/gtank/ristretto255 v0.1.2 diff --git a/go.sum b/go.sum index 3bb35f4..7ed9cbc 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ git.openprivacy.ca/cwtch.im/tapir v0.4.4 h1:KyuTVmr9GYptTCeR7JDODjmhBBbnIBf9V3NS git.openprivacy.ca/cwtch.im/tapir v0.4.4/go.mod h1:qMFTdmDZITc1BLP1jSW0gVpLmvpg+Zjsh5ek8StwbFE= git.openprivacy.ca/cwtch.im/tapir v0.4.5 h1:CTMFdH2qBt4dkiyzNbxuQpK52iKerZmFnG2SfLLSmS8= git.openprivacy.ca/cwtch.im/tapir v0.4.5/go.mod h1:qMFTdmDZITc1BLP1jSW0gVpLmvpg+Zjsh5ek8StwbFE= +git.openprivacy.ca/cwtch.im/tapir v0.4.6 h1:hoKIXNfsWjTiLry9NYTBc9VJYSST3WydHv+Wrt+bn8s= +git.openprivacy.ca/cwtch.im/tapir v0.4.6/go.mod h1:qMFTdmDZITc1BLP1jSW0gVpLmvpg+Zjsh5ek8StwbFE= git.openprivacy.ca/openprivacy/bine v0.0.4 h1:CO7EkGyz+jegZ4ap8g5NWRuDHA/56KKvGySR6OBPW+c= git.openprivacy.ca/openprivacy/bine v0.0.4/go.mod h1:13ZqhKyqakDsN/ZkQkIGNULsmLyqtXc46XBcnuXm/mU= git.openprivacy.ca/openprivacy/connectivity v1.4.0 h1:c7AANUCrlA4hIqXxIGDOWMtSe8CpDleD1877PShScbM=