Merge pull request 'a bunch of cache logic fixes and futher support for reconnect on android' (#431) from cachefixes into trunk

Reviewed-on: #431
Reviewed-by: Sarah Jamie Lewis <sarah@openprivacy.ca>

LIBCWTCH-GO-MACOS.version

@@ -1 +1 @@
-2022-04-14-18-14-v.1.7.0-2-g9901e08
+2022-04-19-20-25-v.1.7.0-6-gd8ed0bf

LIBCWTCH-GO.version

@@ -1 +1 @@
-2022-04-14-22-15-v.1.7.0-2-g9901e08
+2022-04-20-00-25-v.1.7.0-6-gd8ed0bf

lib/models/message.dart

@@ -77,38 +77,51 @@ class ByIndex implements CacheHandler {
   }
 
   Future<MessageInfo?> get(Cwtch cwtch, String profileOnion, int conversationIdentifier, MessageCache cache) async {
-    // if in cache, get
-    if (index < cache.cacheByIndex.length) {
+    // if in cache, get. But if the cache has unsynced or not in cache, we'll have to do a fetch
+    if (cache.indexUnsynced == 0 && index < cache.cacheByIndex.length) {
       return cache.getByIndex(index);
     }
 
     // otherwise we are going to fetch, so we'll fetch a chunk of messages
     // observationally flutter future builder seemed to be reaching for 20-40 message on pane load, so we start trying to load up to that many messages in one request
-    var chunk = 40;
+    var amount = 40;
+    var start = index;
+    // we have to keep the indexed cache contiguous so reach back to the end of it and start the fetch from there
+    if (index > cache.cacheByIndex.length) {
+      start = cache.cacheByIndex.length;
+      amount += index - start;
+    }
+
+    // on android we may have recieved messages on the backend that we didn't process in the UI, get them
+    // override the index chunk setting, the index math is wrong will we fetch these and these are all that should be missing
+    if (cache.indexUnsynced > 0) {
+      start = 0;
+      amount = cache.indexUnsynced;
+    }
+
     // check that we aren't asking for messages beyond stored messages
-    if (index + chunk >= cache.storageMessageCount) {
-      chunk = cache.storageMessageCount - index;
-      if (chunk <= 0) {
+    if (start + amount >= cache.storageMessageCount) {
+      amount = cache.storageMessageCount - start;
+      if (amount <= 0) {
         return Future.value(null);
       }
     }
 
-    cache.lockIndexes(index, index + chunk);
-    var msgs = await cwtch.GetMessages(profileOnion, conversationIdentifier, index, chunk);
+    cache.lockIndexes(start, start + amount);
+    var msgs = await cwtch.GetMessages(profileOnion, conversationIdentifier, start, amount);
     int i = 0; // i used to loop through returned messages. if doesn't reach the requested count, we will use it in the finally stanza to error out the remaining asked for messages in the cache
     try {
       List<dynamic> messagesWrapper = jsonDecode(msgs);
 
       for (; i < messagesWrapper.length; i++) {
         var messageInfo = messageWrapperToInfo(profileOnion, conversationIdentifier, messagesWrapper[i]);
-        cache.addIndexed(messageInfo, index + i);
+        cache.addIndexed(messageInfo, start + i);
       }
-      //messageWrapperToInfo
     } catch (e, stacktrace) {
-      EnvironmentConfig.debugLog("Error: Getting indexed messages $index to ${index + chunk} failed parsing: " + e.toString() + " " + stacktrace.toString());
+      EnvironmentConfig.debugLog("Error: Getting indexed messages $start to ${start + amount} failed parsing: " + e.toString() + " " + stacktrace.toString());
     } finally {
-      if (i != chunk) {
-        cache.malformIndexes(index + i, index + chunk);
+      if (i != amount) {
+        cache.malformIndexes(start + i, start + amount);
       }
     }
     return cache.getByIndex(index);

lib/models/messagecache.dart

@@ -5,6 +5,10 @@ import 'package:flutter/foundation.dart';
 
 import 'message.dart';
 
+// we only count up to 100 unread messages, if more than that we can't accurately resync message cache, just reset
+// https://git.openprivacy.ca/cwtch.im/libcwtch-go/src/branch/trunk/utils/eventHandler.go#L210
+const MaxUnreadBeforeCacheReset = 100;
+
 class MessageInfo {
   late MessageMetadata metadata;
   late String wrapper;
@@ -74,6 +78,8 @@ class MessageCache extends ChangeNotifier {
 
   // local index to MessageId
   late List<LocalIndexMessage> cacheByIndex;
+  // index unsynced is used on android on reconnect to tell us new messages are in the backend that should be at the front of the index cache
+  int _indexUnsynced = 0;
 
   // map of content hash to MessageId
   late Map<String, int> cacheByHash;
@@ -87,13 +93,36 @@ class MessageCache extends ChangeNotifier {
     this._storageMessageCount = storageMessageCount;
   }
 
-  int get indexedLength => cacheByIndex.length;
-
   int get storageMessageCount => _storageMessageCount;
   set storageMessageCount(int newval) {
     this._storageMessageCount = newval;
   }
 
+  // On android reconnect, get unread message cound and the last seen Id
+  // sync this data with what we have cached to determine if/how many messages are now at the front of the index that we don't have cached
+  void addFrontIndexGap(int count, int lastSeenId) {
+    // scan across indexed message the unread count amount (that's the last time UI/BE acked a message)
+    // if we find the last seen ID, the diff of unread count is what's unsynced
+    for(var i = 0; i < (count+1) && i < cacheByIndex.length; i++) {
+      if (this.cacheByIndex[i].messageId == lastSeenId) {
+        // we have found the matching lastSeenId so we can calculate the unsynced as the unread messages before it
+        this._indexUnsynced = count - i;
+        notifyListeners();
+        return;
+      }
+    }
+    // we did not find a matching index, diff to the back end is too great, reset index cache
+    resetIndexCache();
+  }
+  
+  int get indexUnsynced => _indexUnsynced;
+
+  void resetIndexCache() {
+    this._indexUnsynced = 0;
+    cacheByIndex = List.empty(growable: true);
+    notifyListeners();
+  }
+
   MessageInfo? getById(int id) => cache[id];
 
   Future<MessageInfo?> getByIndex(int index) async {
@@ -124,6 +153,11 @@ class MessageCache extends ChangeNotifier {
   void lockIndexes(int start, int end) {
     for (var i = start; i < end; i++) {
       this.cacheByIndex.insert(i, LocalIndexMessage(null, isLoading: true));
+      // if there are unsynced messages on the index cache it means there are messages at the front, and by the logic in message/ByIndex/get() we will be loading those
+      // there for we can decrement the count as this will be one of them
+      if (this._indexUnsynced > 0) {
+        this._indexUnsynced--;
+      }
     }
   }
 

lib/models/profile.dart

@@ -6,6 +6,7 @@ import 'package:flutter/widgets.dart';
 import 'contact.dart';
 import 'contactlist.dart';
 import 'filedownloadprogress.dart';
+import 'messagecache.dart';
 import 'profileservers.dart';
 
 class ProfileInfoState extends ChangeNotifier {
@@ -177,6 +178,12 @@ class ProfileInfoState extends ChangeNotifier {
           profileContact.status = contact["status"];
           profileContact.totalMessages = contact["numMessages"];
           profileContact.unreadMessages = contact["numUnread"];
+
+          if (contact["numUnread"] > MaxUnreadBeforeCacheReset || (contact["numUnread"] > 0 && contact["lastSeenMessageId"] == -1)) {
+            profileContact.messageCache.resetIndexCache();
+          } else if (contact["numUnread"] > 0) {
+            profileContact.messageCache.addFrontIndexGap(contact["numUnread"], contact["lastSeenMessageId"]);
+          }
           profileContact.lastMessageTime = DateTime.fromMillisecondsSinceEpoch(1000 * int.parse(contact["lastMsgTime"]));
         } else {
           this._contacts.add(ContactInfoState(

lib/views/messageview.dart

@@ -128,7 +128,7 @@ class _MessageViewState extends State<MessageView> {
           backgroundColor: Provider.of<Settings>(context).theme.backgroundMainColor,
           floatingActionButton: appState.unreadMessagesBelow
               ? FloatingActionButton(
-                  child: Icon(Icons.arrow_downward),
+                  child: Icon(Icons.arrow_downward, color: Provider.of<Settings>(context).current().defaultButtonTextColor),
                   onPressed: () {
                     Provider.of<AppState>(context, listen: false).initialScrollIndex = 0;
                     Provider.of<AppState>(context, listen: false).unreadMessagesBelow = false;