From 5959981fe4dea3c5bda90fdb13bdbac5cd7e0b8a Mon Sep 17 00:00:00 2001 From: Dan Ballard Date: Tue, 19 Apr 2022 17:00:19 -0700 Subject: [PATCH 1/3] a bunch of cache logic fixes and futher support for reconnect on android --- lib/models/message.dart | 39 ++++++++++++++++++++++++------------ lib/models/messagecache.dart | 32 +++++++++++++++++++++++++++-- lib/models/profile.dart | 7 +++++++ lib/views/messageview.dart | 2 +- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/lib/models/message.dart b/lib/models/message.dart index f8dcc232..0ab95a07 100644 --- a/lib/models/message.dart +++ b/lib/models/message.dart @@ -77,38 +77,51 @@ class ByIndex implements CacheHandler { } Future get(Cwtch cwtch, String profileOnion, int conversationIdentifier, MessageCache cache) async { - // if in cache, get - if (index < cache.cacheByIndex.length) { + // if in cache, get. But if the cache has unsynced or not in cache, we'll have to do a fetch + if (cache.indexUnsynced == 0 && index < cache.cacheByIndex.length) { return cache.getByIndex(index); } // otherwise we are going to fetch, so we'll fetch a chunk of messages // observationally flutter future builder seemed to be reaching for 20-40 message on pane load, so we start trying to load up to that many messages in one request - var chunk = 40; + var amount = 40; + var start = index; + // we have to keep the indexed cache contiguous so reach back to the end of it and start the fetch from there + if (index > cache.cacheByIndex.length) { + start = cache.cacheByIndex.length; + amount += index - start; + } + + // on android we may have recieved messages on the backend that we didn't process in the UI, get them + // override the index chunk setting, the index math is wrong will we fetch these and these are all that should be missing + if (cache.indexUnsynced > 0) { + start = 0; + amount = cache.indexUnsynced; + } + // check that we aren't asking for messages beyond stored messages - if (index + chunk >= cache.storageMessageCount) { - chunk = cache.storageMessageCount - index; - if (chunk <= 0) { + if (start + amount >= cache.storageMessageCount) { + amount = cache.storageMessageCount - start; + if (amount <= 0) { return Future.value(null); } } - cache.lockIndexes(index, index + chunk); - var msgs = await cwtch.GetMessages(profileOnion, conversationIdentifier, index, chunk); + cache.lockIndexes(start, start + amount); + var msgs = await cwtch.GetMessages(profileOnion, conversationIdentifier, start, amount); int i = 0; // i used to loop through returned messages. if doesn't reach the requested count, we will use it in the finally stanza to error out the remaining asked for messages in the cache try { List messagesWrapper = jsonDecode(msgs); for (; i < messagesWrapper.length; i++) { var messageInfo = messageWrapperToInfo(profileOnion, conversationIdentifier, messagesWrapper[i]); - cache.addIndexed(messageInfo, index + i); + cache.addIndexed(messageInfo, start + i); } - //messageWrapperToInfo } catch (e, stacktrace) { - EnvironmentConfig.debugLog("Error: Getting indexed messages $index to ${index + chunk} failed parsing: " + e.toString() + " " + stacktrace.toString()); + EnvironmentConfig.debugLog("Error: Getting indexed messages $start to ${start + amount} failed parsing: " + e.toString() + " " + stacktrace.toString()); } finally { - if (i != chunk) { - cache.malformIndexes(index + i, index + chunk); + if (i != amount) { + cache.malformIndexes(start + i, start + amount); } } return cache.getByIndex(index); diff --git a/lib/models/messagecache.dart b/lib/models/messagecache.dart index 1ad2ec59..8e67b81a 100644 --- a/lib/models/messagecache.dart +++ b/lib/models/messagecache.dart @@ -74,6 +74,8 @@ class MessageCache extends ChangeNotifier { // local index to MessageId late List cacheByIndex; + // index unsynced is used on android on reconnect to tell us new messages are in the backend that should be at the front of the index cache + int _indexUnsynced = 0; // map of content hash to MessageId late Map cacheByHash; @@ -87,13 +89,36 @@ class MessageCache extends ChangeNotifier { this._storageMessageCount = storageMessageCount; } - int get indexedLength => cacheByIndex.length; - int get storageMessageCount => _storageMessageCount; set storageMessageCount(int newval) { this._storageMessageCount = newval; } + // On android reconnect, get unread message cound and the last seen Id + // sync this data with what we have cached to determine if/how many messages are now at the front of the index that we don't have cached + void addFrontIndexGap(int count, int lastSeenId) { + // scan across indexed message the unread count amount (that's the last time UI/BE acked a message) + // if we find the last seen ID, the diff of unread count is what's unsynced + for(var i = 0; i < (count+1) && i < cacheByIndex.length; i++) { + if (this.cacheByIndex[i].messageId == lastSeenId) { + // we have + this._indexUnsynced = count - i; + notifyListeners(); + return; + } + } + // we did not find a matching index, diff to the back end is too great, reset index cache + resetIndexCache(); + } + + int get indexUnsynced => _indexUnsynced; + + void resetIndexCache() { + this._indexUnsynced = 0; + cacheByIndex = List.empty(growable: true); + notifyListeners(); + } + MessageInfo? getById(int id) => cache[id]; Future getByIndex(int index) async { @@ -124,6 +149,9 @@ class MessageCache extends ChangeNotifier { void lockIndexes(int start, int end) { for (var i = start; i < end; i++) { this.cacheByIndex.insert(i, LocalIndexMessage(null, isLoading: true)); + if (this._indexUnsynced > 0) { + this._indexUnsynced--; + } } } diff --git a/lib/models/profile.dart b/lib/models/profile.dart index 1848c2f9..4d23ef1e 100644 --- a/lib/models/profile.dart +++ b/lib/models/profile.dart @@ -177,6 +177,13 @@ class ProfileInfoState extends ChangeNotifier { profileContact.status = contact["status"]; profileContact.totalMessages = contact["numMessages"]; profileContact.unreadMessages = contact["numUnread"]; + // we only count up to 100 unread messages, if more than that we can't accuratly resync message cache, just reset + if (contact["numUnread"] > 100 || (contact["numUnread"] > 0 && contact["lastSeenMessageId"] == -1)) { + profileContact.messageCache.resetIndexCache(); + } else if (contact["numUnread"] > 0) { + print("contact ${contact["name"]} with unread ${contact["numUnread"]} so addFrontIndexGap"); + profileContact.messageCache.addFrontIndexGap(contact["numUnread"], contact["lastSeenMessageId"]); + } profileContact.lastMessageTime = DateTime.fromMillisecondsSinceEpoch(1000 * int.parse(contact["lastMsgTime"])); } else { this._contacts.add(ContactInfoState( diff --git a/lib/views/messageview.dart b/lib/views/messageview.dart index 42c504d1..2716da4f 100644 --- a/lib/views/messageview.dart +++ b/lib/views/messageview.dart @@ -128,7 +128,7 @@ class _MessageViewState extends State { backgroundColor: Provider.of(context).theme.backgroundMainColor, floatingActionButton: appState.unreadMessagesBelow ? FloatingActionButton( - child: Icon(Icons.arrow_downward), + child: Icon(Icons.arrow_downward, color: Provider.of(context).current().defaultButtonTextColor), onPressed: () { Provider.of(context, listen: false).initialScrollIndex = 0; Provider.of(context, listen: false).unreadMessagesBelow = false; From 82d1bf873f23b78316f526fcf8acbc920c42705c Mon Sep 17 00:00:00 2001 From: Dan Ballard Date: Tue, 19 Apr 2022 17:42:48 -0700 Subject: [PATCH 2/3] lcg bump --- LIBCWTCH-GO-MACOS.version | 2 +- LIBCWTCH-GO.version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LIBCWTCH-GO-MACOS.version b/LIBCWTCH-GO-MACOS.version index 4fc8a725..69f4effe 100644 --- a/LIBCWTCH-GO-MACOS.version +++ b/LIBCWTCH-GO-MACOS.version @@ -1 +1 @@ -2022-04-14-18-14-v.1.7.0-2-g9901e08 \ No newline at end of file +2022-04-19-20-25-v.1.7.0-6-gd8ed0bf \ No newline at end of file diff --git a/LIBCWTCH-GO.version b/LIBCWTCH-GO.version index 27059d83..d1bad76a 100644 --- a/LIBCWTCH-GO.version +++ b/LIBCWTCH-GO.version @@ -1 +1 @@ -2022-04-14-22-15-v.1.7.0-2-g9901e08 \ No newline at end of file +2022-04-20-00-25-v.1.7.0-6-gd8ed0bf \ No newline at end of file From 4bd92d854f9e153dceac1747cd9c9512f67fa740 Mon Sep 17 00:00:00 2001 From: Dan Ballard Date: Tue, 19 Apr 2022 20:46:37 -0700 Subject: [PATCH 3/3] comments --- lib/models/messagecache.dart | 8 +++++++- lib/models/profile.dart | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/models/messagecache.dart b/lib/models/messagecache.dart index 8e67b81a..eefc6206 100644 --- a/lib/models/messagecache.dart +++ b/lib/models/messagecache.dart @@ -5,6 +5,10 @@ import 'package:flutter/foundation.dart'; import 'message.dart'; +// we only count up to 100 unread messages, if more than that we can't accurately resync message cache, just reset +// https://git.openprivacy.ca/cwtch.im/libcwtch-go/src/branch/trunk/utils/eventHandler.go#L210 +const MaxUnreadBeforeCacheReset = 100; + class MessageInfo { late MessageMetadata metadata; late String wrapper; @@ -101,7 +105,7 @@ class MessageCache extends ChangeNotifier { // if we find the last seen ID, the diff of unread count is what's unsynced for(var i = 0; i < (count+1) && i < cacheByIndex.length; i++) { if (this.cacheByIndex[i].messageId == lastSeenId) { - // we have + // we have found the matching lastSeenId so we can calculate the unsynced as the unread messages before it this._indexUnsynced = count - i; notifyListeners(); return; @@ -149,6 +153,8 @@ class MessageCache extends ChangeNotifier { void lockIndexes(int start, int end) { for (var i = start; i < end; i++) { this.cacheByIndex.insert(i, LocalIndexMessage(null, isLoading: true)); + // if there are unsynced messages on the index cache it means there are messages at the front, and by the logic in message/ByIndex/get() we will be loading those + // there for we can decrement the count as this will be one of them if (this._indexUnsynced > 0) { this._indexUnsynced--; } diff --git a/lib/models/profile.dart b/lib/models/profile.dart index 4d23ef1e..a7b746da 100644 --- a/lib/models/profile.dart +++ b/lib/models/profile.dart @@ -6,6 +6,7 @@ import 'package:flutter/widgets.dart'; import 'contact.dart'; import 'contactlist.dart'; import 'filedownloadprogress.dart'; +import 'messagecache.dart'; import 'profileservers.dart'; class ProfileInfoState extends ChangeNotifier { @@ -177,11 +178,10 @@ class ProfileInfoState extends ChangeNotifier { profileContact.status = contact["status"]; profileContact.totalMessages = contact["numMessages"]; profileContact.unreadMessages = contact["numUnread"]; - // we only count up to 100 unread messages, if more than that we can't accuratly resync message cache, just reset - if (contact["numUnread"] > 100 || (contact["numUnread"] > 0 && contact["lastSeenMessageId"] == -1)) { + + if (contact["numUnread"] > MaxUnreadBeforeCacheReset || (contact["numUnread"] > 0 && contact["lastSeenMessageId"] == -1)) { profileContact.messageCache.resetIndexCache(); } else if (contact["numUnread"] > 0) { - print("contact ${contact["name"]} with unread ${contact["numUnread"]} so addFrontIndexGap"); profileContact.messageCache.addFrontIndexGap(contact["numUnread"], contact["lastSeenMessageId"]); } profileContact.lastMessageTime = DateTime.fromMillisecondsSinceEpoch(1000 * int.parse(contact["lastMsgTime"]));