Merge pull request 'caching fixes for stability and android' (#450) from cache3.0 into trunk

Reviewed-on: #450
Reviewed-by: Sarah Jamie Lewis <sarah@openprivacy.ca>

lib/models/contact.dart

@@ -42,8 +42,7 @@ class ContactInfoState extends ChangeNotifier {
   late int _totalMessages = 0;
   late DateTime _lastMessageTime;
   late Map<String, GlobalKey<MessageRowState>> keys;
-  int _newMarkerMsgId = -1;
-  DateTime _newMarkerClearAt = DateTime.now();
+  int _newMarkerMsgIndex = -1;
   late MessageCache messageCache;
 
   // todo: a nicer way to model contacts, groups and other "entities"
@@ -145,25 +144,24 @@ class ContactInfoState extends ChangeNotifier {
     notifyListeners();
   }
 
+  void selected() {
+    this._newMarkerMsgIndex = this._unreadMessages-1;
+    this._unreadMessages = 0;
+  }
+
+  void unselected() {
+    this._newMarkerMsgIndex = -1;
+  }
+
   int get unreadMessages => this._unreadMessages;
 
   set unreadMessages(int newVal) {
-    if (newVal == 0 && this._unreadMessages != 0) {
-      // conversation has been selected, start the countdown for the New Messager marker to be reset
-      this._newMarkerClearAt = DateTime.now().add(const Duration(minutes: 2));
-    }
     this._unreadMessages = newVal;
     notifyListeners();
   }
 
-  int get newMarkerMsgId {
-    if (DateTime.now().isAfter(this._newMarkerClearAt)) {
-      // perform heresy
-      this._newMarkerMsgId = -1;
-      // no need to notifyListeners() because presumably this getter is
-      // being called from a renderer anyway
-    }
-    return this._newMarkerMsgId;
+  int get newMarkerMsgIndex {
+    return this._newMarkerMsgIndex;
   }
 
   int get totalMessages => this._totalMessages;
@@ -243,8 +241,10 @@ class ContactInfoState extends ChangeNotifier {
     if (!selectedConversation) {
       unreadMessages++;
     }
-    if (_newMarkerMsgId == -1) {
-      _newMarkerMsgId = messageID;
+    if (_newMarkerMsgIndex == -1) {
+      _newMarkerMsgIndex = 0;
+    } else {
+      _newMarkerMsgIndex++;
     }
 
     this._lastMessageTime = timestamp;

lib/models/message.dart

@@ -32,33 +32,34 @@ const GroupConversationHandleLength = 32;
 abstract class Message {
   MessageMetadata getMetadata();
 
-  Widget getWidget(BuildContext context, Key key);
+  Widget getWidget(BuildContext context, Key key, int index);
 
   Widget getPreviewWidget(BuildContext context);
 }
 
-Message compileOverlay(MessageMetadata metadata, String messageData) {
-  try {
-    dynamic message = jsonDecode(messageData);
+Message compileOverlay(MessageInfo messageInfo) {
+
+    try {
+    dynamic message = jsonDecode(messageInfo.wrapper);
     var content = message['d'] as dynamic;
     var overlay = int.parse(message['o'].toString());
 
     switch (overlay) {
       case TextMessageOverlay:
-        return TextMessage(metadata, content);
+        return TextMessage(messageInfo.metadata, content);
       case SuggestContactOverlay:
       case InviteGroupOverlay:
-        return InviteMessage(overlay, metadata, content);
+        return InviteMessage(overlay, messageInfo.metadata, content);
       case QuotedMessageOverlay:
-        return QuotedMessage(metadata, content);
+        return QuotedMessage(messageInfo.metadata, content);
       case FileShareOverlay:
-        return FileMessage(metadata, content);
+        return FileMessage(messageInfo.metadata, content);
       default:
         // Metadata is valid, content is not..
-        return MalformedMessage(metadata);
+        return MalformedMessage(messageInfo.metadata);
     }
   } catch (e) {
-    return MalformedMessage(metadata);
+    return MalformedMessage(messageInfo.metadata);
   }
 }
 
@@ -78,7 +79,7 @@ class ByIndex implements CacheHandler {
 
   Future<MessageInfo?> get(Cwtch cwtch, String profileOnion, int conversationIdentifier, MessageCache cache) async {
     // if in cache, get. But if the cache has unsynced or not in cache, we'll have to do a fetch
-    if (cache.indexUnsynced == 0 && index < cache.cacheByIndex.length) {
+    if (index < cache.cacheByIndex.length) {
       return cache.getByIndex(index);
     }
 
@@ -92,12 +93,6 @@ class ByIndex implements CacheHandler {
       amount += index - start;
     }
 
-    // on android we may have recieved messages on the backend that we didn't process in the UI, get them
-    // override the index chunk setting, the index math is wrong will we fetch these and these are all that should be missing
-    if (cache.indexUnsynced > 0) {
-      start = 0;
-      amount = cache.indexUnsynced;
-    }
 
     // check that we aren't asking for messages beyond stored messages
     if (start + amount >= cache.storageMessageCount) {
@@ -108,6 +103,27 @@ class ByIndex implements CacheHandler {
     }
 
     cache.lockIndexes(start, start + amount);
+    await fetchAndProcess(start, amount, cwtch, profileOnion, conversationIdentifier, cache);
+
+    return cache.getByIndex(index);
+  }
+
+  void loadUnsynced(Cwtch cwtch, String profileOnion, int conversationIdentifier, MessageCache cache) {
+    // return if inadvertently called when no unsynced messages
+    if (cache.indexUnsynced == 0) {
+      return;
+    }
+
+    // otherwise we are going to fetch, so we'll fetch a chunk of messages
+    var start = 0;
+    var amount = cache.indexUnsynced;
+
+    cache.lockIndexes(start, start + amount);
+    fetchAndProcess(start, amount, cwtch, profileOnion, conversationIdentifier, cache);
+    return;
+  }
+
+  Future<void> fetchAndProcess(int start, int amount, Cwtch cwtch, String profileOnion, int conversationIdentifier, MessageCache cache) async {
     var msgs = await cwtch.GetMessages(profileOnion, conversationIdentifier, start, amount);
     int i = 0; // i used to loop through returned messages. if doesn't reach the requested count, we will use it in the finally stanza to error out the remaining asked for messages in the cache
     try {
@@ -124,7 +140,6 @@ class ByIndex implements CacheHandler {
         cache.malformIndexes(start + i, start + amount);
       }
     }
-    return cache.getByIndex(index);
   }
 
   void add(MessageCache cache, MessageInfo messageInfo) {
@@ -208,7 +223,7 @@ Future<Message> messageHandler(BuildContext context, String profileOnion, int co
   MessageInfo? messageInfo = await cacheHandler.get(cwtch, profileOnion, conversationIdentifier, cache);
 
   if (messageInfo != null) {
-    return compileOverlay(messageInfo.metadata, messageInfo.wrapper);
+    return compileOverlay(messageInfo);
   } else {
     return MalformedMessage(malformedMetadata);
   }

lib/models/messagecache.dart

@@ -98,31 +98,13 @@ class MessageCache extends ChangeNotifier {
     this._storageMessageCount = newval;
   }
 
-  // On android reconnect, get unread message cound and the last seen Id
-  // sync this data with what we have cached to determine if/how many messages are now at the front of the index that we don't have cached
-  void addFrontIndexGap(int count, int lastSeenId) {
-    // scan across indexed message the unread count amount (that's the last time UI/BE acked a message)
-    // if we find the last seen ID, the diff of unread count is what's unsynced
-    for (var i = 0; i < (count + 1) && i < cacheByIndex.length; i++) {
-      if (this.cacheByIndex[i].messageId == lastSeenId) {
-        // we have found the matching lastSeenId so we can calculate the unsynced as the unread messages before it
-        this._indexUnsynced = count - i;
-        notifyListeners();
-        return;
-      }
-    }
-    // we did not find a matching index, diff to the back end is too great, reset index cache
-    resetIndexCache();
+  // On android reconnect, if backend supplied message count > UI message count, add the differnce to the front of the index
+  void addFrontIndexGap(int count) {
+    this._indexUnsynced = count;
   }
 
   int get indexUnsynced => _indexUnsynced;
 
-  void resetIndexCache() {
-    this._indexUnsynced = 0;
-    cacheByIndex = List.empty(growable: true);
-    notifyListeners();
-  }
-
   MessageInfo? getById(int id) => cache[id];
 
   Future<MessageInfo?> getByIndex(int index) async {
@@ -144,7 +126,6 @@ class MessageCache extends ChangeNotifier {
     if (contenthash != null && contenthash != "") {
       this.cacheByHash[contenthash] = messageID;
     }
-    notifyListeners();
   }
 
   // inserts place holder values into the index cache that will block on .get() until .finishLoad() is called on them with message contents
@@ -175,7 +156,6 @@ class MessageCache extends ChangeNotifier {
       this.cacheByIndex.insert(index, LocalIndexMessage(messageInfo.metadata.messageID));
     }
     this.cacheByHash[messageInfo.metadata.contenthash] = messageInfo.metadata.messageID;
-    notifyListeners();
   }
 
   void addUnindexed(MessageInfo messageInfo) {
@@ -183,7 +163,6 @@ class MessageCache extends ChangeNotifier {
     if (messageInfo.metadata.contenthash != "") {
       this.cacheByHash[messageInfo.metadata.contenthash] = messageInfo.metadata.messageID;
     }
-    notifyListeners();
   }
 
   void ackCache(int messageID) {

lib/models/messages/filemessage.dart

@@ -18,13 +18,13 @@ class FileMessage extends Message {
   FileMessage(this.metadata, this.content);
 
   @override
-  Widget getWidget(BuildContext context, Key key) {
+  Widget getWidget(BuildContext context, Key key, int index) {
     return ChangeNotifierProvider.value(
         value: this.metadata,
         builder: (bcontext, child) {
           dynamic shareObj = jsonDecode(this.content);
           if (shareObj == null) {
-            return MessageRow(MalformedBubble());
+            return MessageRow(MalformedBubble(), index);
           }
           String nameSuggestion = shareObj['f'] as String;
           String rootHash = shareObj['h'] as String;
@@ -39,10 +39,10 @@ class FileMessage extends Message {
           }
 
           if (!validHash(rootHash, nonce)) {
-            return MessageRow(MalformedBubble());
+            return MessageRow(MalformedBubble(), index);
           }
 
-          return MessageRow(FileBubble(nameSuggestion, rootHash, nonce, fileSize, isAuto: metadata.isAuto), key: key);
+          return MessageRow(FileBubble(nameSuggestion, rootHash, nonce, fileSize, isAuto: metadata.isAuto), index, key: key);
         });
   }
 
@@ -53,14 +53,14 @@ class FileMessage extends Message {
         builder: (bcontext, child) {
           dynamic shareObj = jsonDecode(this.content);
           if (shareObj == null) {
-            return MessageRow(MalformedBubble());
+            return MessageRow(MalformedBubble(), 0);
           }
           String nameSuggestion = shareObj['n'] as String;
           String rootHash = shareObj['h'] as String;
           String nonce = shareObj['n'] as String;
           int fileSize = shareObj['s'] as int;
           if (!validHash(rootHash, nonce)) {
-            return MessageRow(MalformedBubble());
+            return MessageRow(MalformedBubble(), 0);
           }
           return Container(
               alignment: Alignment.center,

lib/models/messages/invitemessage.dart

@@ -17,7 +17,7 @@ class InviteMessage extends Message {
   InviteMessage(this.overlay, this.metadata, this.content);
 
   @override
-  Widget getWidget(BuildContext context, Key key) {
+  Widget getWidget(BuildContext context, Key key, int index) {
     return ChangeNotifierProvider.value(
         value: this.metadata,
         builder: (bcontext, child) {
@@ -36,10 +36,10 @@ class InviteMessage extends Message {
               inviteTarget = jsonObj['GroupID'];
               inviteNick = jsonObj['GroupName'];
             } else {
-              return MessageRow(MalformedBubble());
+              return MessageRow(MalformedBubble(), index);
             }
           }
-          return MessageRow(InvitationBubble(overlay, inviteTarget, inviteNick, invite), key: key);
+          return MessageRow(InvitationBubble(overlay, inviteTarget, inviteNick, invite), index, key: key);
         });
   }
 

lib/models/messages/malformedmessage.dart

@@ -9,11 +9,11 @@ class MalformedMessage extends Message {
   MalformedMessage(this.metadata);
 
   @override
-  Widget getWidget(BuildContext context, Key key) {
+  Widget getWidget(BuildContext context, Key key, int index) {
     return ChangeNotifierProvider.value(
         value: this.metadata,
         builder: (context, child) {
-          return MessageRow(MalformedBubble(), key: key);
+          return MessageRow(MalformedBubble(), index, key: key);
         });
   }
 

lib/models/messages/quotedmessage.dart

@@ -48,7 +48,7 @@ class QuotedMessage extends Message {
   }
 
   @override
-  Widget getWidget(BuildContext context, Key key) {
+  Widget getWidget(BuildContext context, Key key, int index) {
     try {
       dynamic message = jsonDecode(this.content);
 
@@ -59,7 +59,7 @@ class QuotedMessage extends Message {
       return ChangeNotifierProvider.value(
           value: this.metadata,
           builder: (bcontext, child) {
-            return MessageRow(QuotedMessageBubble(message["body"], messageHandler(bcontext, metadata.profileOnion, metadata.conversationIdentifier, ByContentHash(message["quotedHash"]))), key: key);
+            return MessageRow(QuotedMessageBubble(message["body"], messageHandler(bcontext, metadata.profileOnion, metadata.conversationIdentifier, ByContentHash(message["quotedHash"]))), index, key: key);
           });
     } catch (e) {
       return MalformedBubble();

lib/models/messages/textmessage.dart

@@ -29,12 +29,12 @@ class TextMessage extends Message {
   }
 
   @override
-  Widget getWidget(BuildContext context, Key key) {
+  Widget getWidget(BuildContext context, Key key, int index) {
     return ChangeNotifierProvider.value(
         value: this.metadata,
         builder: (bcontext, child) {
           return MessageRow(
-            MessageBubble(this.content),
+            MessageBubble(this.content), index,
             key: key,
           );
         });

lib/models/profile.dart

@@ -176,14 +176,13 @@ class ProfileInfoState extends ChangeNotifier {
         this._unreadMessages += contact["numUnread"] as int;
         if (profileContact != null) {
           profileContact.status = contact["status"];
-          profileContact.totalMessages = contact["numMessages"];
-          profileContact.unreadMessages = contact["numUnread"];
 
-          if (contact["numUnread"] > MaxUnreadBeforeCacheReset || (contact["numUnread"] > 0 && contact["lastSeenMessageId"] == -1)) {
-            profileContact.messageCache.resetIndexCache();
-          } else if (contact["numUnread"] > 0) {
-            profileContact.messageCache.addFrontIndexGap(contact["numUnread"], contact["lastSeenMessageId"]);
+          var newCount = contact["numMessages"];
+          if (newCount != profileContact.totalMessages) {
+            profileContact.messageCache.addFrontIndexGap(newCount - profileContact.totalMessages);
           }
+          profileContact.totalMessages = newCount;
+          profileContact.unreadMessages = contact["numUnread"];
           profileContact.lastMessageTime = DateTime.fromMillisecondsSinceEpoch(1000 * int.parse(contact["lastMsgTime"]));
         } else {
           this._contacts.add(ContactInfoState(

lib/views/contactsview.dart

@@ -30,7 +30,15 @@ class ContactsView extends StatefulWidget {
 void selectConversation(BuildContext context, int handle) {
   // requery instead of using contactinfostate directly because sometimes listview gets confused about data that resorts
   var initialIndex = Provider.of<ProfileInfoState>(context, listen: false).contactList.getContact(handle)!.unreadMessages;
-  Provider.of<ProfileInfoState>(context, listen: false).contactList.getContact(handle)!.unreadMessages = 0;
+  var previouslySelected = Provider.of<AppState>(context, listen: false).selectedConversation;
+  if (previouslySelected != null) {
+    Provider
+        .of<ProfileInfoState>(context, listen: false)
+        .contactList
+        .getContact(previouslySelected)!
+        .unselected();
+  }
+  Provider.of<ProfileInfoState>(context, listen: false).contactList.getContact(handle)!.selected();
   // triggers update in Double/TripleColumnView
   Provider.of<AppState>(context, listen: false).initialScrollIndex = initialIndex;
   Provider.of<AppState>(context, listen: false).selectedConversation = handle;

lib/views/messageview.dart

@@ -168,6 +168,16 @@ class _MessageViewState extends State<MessageView> {
 
   Future<bool> _onWillPop() async {
     Provider.of<ContactInfoState>(context, listen: false).unreadMessages = 0;
+
+    var previouslySelected = Provider.of<AppState>(context, listen: false).selectedConversation;
+    if (previouslySelected != null) {
+      Provider
+          .of<ProfileInfoState>(context, listen: false)
+          .contactList
+          .getContact(previouslySelected)!
+          .unselected();
+    }
+
     Provider.of<AppState>(context, listen: false).selectedConversation = null;
     return true;
   }

lib/widgets/messagelist.dart

@@ -1,6 +1,7 @@
 import 'package:cwtch/models/appstate.dart';
 import 'package:cwtch/models/contact.dart';
 import 'package:cwtch/models/message.dart';
+import 'package:cwtch/models/messagecache.dart';
 import 'package:cwtch/models/profile.dart';
 import 'package:cwtch/widgets/messageloadingbubble.dart';
 import 'package:flutter/material.dart';
@@ -8,6 +9,7 @@ import 'package:flutter/rendering.dart';
 import 'package:provider/provider.dart';
 import 'package:scrollable_positioned_list/scrollable_positioned_list.dart';
 import 'package:flutter_gen/gen_l10n/app_localizations.dart';
+import '../main.dart';
 import '../settings.dart';
 
 class MessageList extends StatefulWidget {
@@ -22,6 +24,13 @@ class MessageList extends StatefulWidget {
 class _MessageListState extends State<MessageList> {
   @override
   Widget build(BuildContext outerContext) {
+    // On Android we can have unsynced messages at the front of the index from when the UI was asleep, if there are some, kick off sync of those first
+    if (Provider.of<ContactInfoState>(context).messageCache.indexUnsynced != 0) {
+      var conversationId = Provider.of<AppState>(outerContext, listen: false).selectedConversation!;
+      MessageCache? cache = Provider.of<ProfileInfoState>(outerContext, listen: false).contactList.getContact(conversationId)?.messageCache;
+      ByIndex(0).loadUnsynced(Provider.of<FlwtchState>(context, listen: false).cwtch, Provider.of<AppState>(outerContext, listen: false).selectedProfile!, conversationId, cache!);
+    }
+
     var initi = Provider.of<AppState>(outerContext, listen: false).initialScrollIndex;
     bool isP2P = !Provider.of<ContactInfoState>(context).isGroup;
     bool isGroupAndSyncing = Provider.of<ContactInfoState>(context).isGroup == true && Provider.of<ContactInfoState>(context).status == "Authenticated";
@@ -91,7 +100,7 @@ class _MessageListState extends State<MessageList> {
                                       // reliably use this without running into duplicate keys...it isn't ideal as it means keys need to be re-built
                                       // when new messages are added...however it is better than the alternative of not having widget keys at all.
                                       var key = Provider.of<ContactInfoState>(outerContext, listen: false).getMessageKey(contactHandle, messageIndex);
-                                      return message.getWidget(context, key);
+                                      return message.getWidget(context, key, messageIndex);
                                     } else {
                                       return MessageLoadingBubble();
                                     }

lib/widgets/messagerow.dart

@@ -18,8 +18,9 @@ import '../settings.dart';
 
 class MessageRow extends StatefulWidget {
   final Widget child;
+  final int index;
 
-  MessageRow(this.child, {Key? key}) : super(key: key);
+  MessageRow(this.child, this.index, {Key? key}) : super(key: key);
 
   @override
   MessageRowState createState() => MessageRowState();
@@ -32,12 +33,9 @@ class MessageRowState extends State<MessageRow> with SingleTickerProviderStateMi
   late Alignment _dragAlignment = Alignment.center;
   Alignment _dragAffinity = Alignment.center;
 
-  late int index;
-
   @override
   void initState() {
     super.initState();
-    index = Provider.of<MessageMetadata>(context, listen: false).messageID;
     _controller = AnimationController(vsync: this);
     _controller.addListener(() {
       setState(() {
@@ -224,8 +222,7 @@ class MessageRowState extends State<MessageRow> with SingleTickerProviderStateMi
                       children: widgetRow,
                     )))));
 
-    var markMsgId = Provider.of<ContactInfoState>(context).newMarkerMsgId;
-    if (markMsgId == Provider.of<MessageMetadata>(context).messageID) {
+    if (Provider.of<ContactInfoState>(context).newMarkerMsgIndex == widget.index) {
       return Column(crossAxisAlignment: fromMe ? CrossAxisAlignment.end : CrossAxisAlignment.start, children: [Align(alignment: Alignment.center, child: _bubbleNew()), mr]);
     } else {
       return mr;