do away with newmaxuid

now that expiration order is determined by a single loop ordered by
far-side UIDs, it is no longer necessary to accurately track the highest
seen UID.

as a side effect, this fixes a problem reported (way too long ago) by
Yuri D'Elia: we failed to up newmaxuid for messages we produced
ourselves, so we would keep enumerating the same messages until we also
propagated externally generated messages from that mailbox - which might
have been never for the server side of archive/trash mailboxes.

src/run-tests.pl

@@ -61,7 +61,7 @@ my @X01 = (
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", E, 5, "T", F, 6, "FT", G, 7, "FT", I, 9, "", J, 10, "" ],
  [ 10,
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", E, 5, "T", G, 7, "FT", H, 8, "T", J, 9, "", I, 10, "" ],
- [ 9, 0, 9,
+ [ 10, 0, 10,
    1, 1, "F", 2, 2, "F", 3, 3, "FS", 4, 4, "", 5, 5, "T", 6, 0, "", 7, 7, "FT", 0, 8, "", 10, 9, "", 9, 10, "" ],
 );
 test("full", \@x01, \@X01, @O01);
@@ -73,7 +73,7 @@ my @X02 = (
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", I, 9, "", J, 10, "" ],
  [ 10,
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", J, 9, "", I, 10, "" ],
- [ 9, 0, 9,
+ [ 10, 0, 10,
    1, 1, "F", 2, 2, "F", 3, 3, "FS", 4, 4, "", 10, 9, "", 9, 10, "" ],
 );
 test("full + expunge both", \@x01, \@X02, @O02);
@@ -85,7 +85,7 @@ my @X03 = (
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", E, 5, "T", F, 6, "FT", G, 7, "FT", I, 9, "", J, 10, "" ],
  [ 10,
    A, 1, "F", B, 2, "F", C, 3, "FS", D, 4, "", J, 9, "", I, 10, "" ],
- [ 9, 0, 9,
+ [ 10, 0, 10,
    1, 1, "F", 2, 2, "F", 3, 3, "FS", 4, 4, "", 5, 0, "T", 6, 0, "", 7, 0, "T", 10, 9, "", 9, 10, "" ],
 );
 test("full + expunge near side", \@x01, \@X03, @O03);
@@ -133,7 +133,7 @@ my @X07 = (
    A, 1, "F", B, 2, "", C, 3, "FS", D, 4, "", E, 5, "T", F, 6, "F", G, 7, "FT", I, 9, "", J, 10, "" ],
  [ 10,
    A, 1, "", B, 2, "F", C, 3, "F", D, 4, "", E, 5, "", G, 7, "", H, 8, "", J, 9, "", I, 10, "" ],
- [ 9, 0, 9,
+ [ 10, 0, 10,
    1, 1, "", 2, 2, "", 3, 3, "", 4, 4, "", 5, 5, "", 6, 6, "", 7, 7, "", 8, 8, "", 10, 9, "", 9, 10, "" ],
 );
 test("new", \@x01, \@X07, @O07);
@@ -168,7 +168,7 @@ my @X11 = (
    A, 1, "", B, 2, "*" ],
  [ 2,
    C, 1, "*", A, 2, "" ],
- [ 2, 0, 1,
+ [ 2, 0, 2,
    0, 1, "^", 1, 2, "", 2, 0, "^" ],
 );
 test("max size", \@x10, \@X11, @O11);
@@ -180,7 +180,7 @@ my @X22 = (
    A, 1, "", B, 2, "*", C, 3, "*" ],
  [ 2,
    C, 1, "*", A, 2, "" ],
- [ 2, 0, 1,
+ [ 3, 0, 2,
    3, 1, "", 1, 2, "", 2, 0, "^" ],
 );
 test("near side max size", \@X11, \@X22, @O22);
@@ -203,7 +203,7 @@ my @X31 = (
    A, 1, "F", B, 2, "", C, 3, "S", D, 4, "", E, 5, "S", F, 6, "" ],
  [ 5,
    A, 1, "F", B, 2, "", D, 3, "", E, 4, "S", F, 5, "" ],
- [ 6, 3, 0,
+ [ 6, 3, 5,
    1, 1, "F", 2, 2, "", 4, 3, "", 5, 4, "S", 6, 5, "" ],
 );
 test("max messages", \@x30, \@X31, @O31);
@@ -215,7 +215,7 @@ my @X32 = (
    A, 1, "F", B, 2, "", C, 3, "S", D, 4, "", E, 5, "S", F, 6, "" ],
  [ 4,
    A, 1, "F", D, 2, "", E, 3, "S", F, 4, "" ],
- [ 6, 3, 0,
+ [ 6, 3, 4,
    1, 1, "F", 4, 2, "", 5, 3, "S", 6, 4, "" ],
 );
 test("max messages vs. unread", \@x30, \@X32, @O32);
@@ -236,7 +236,7 @@ my @X51 = (
    A, 1, "S", B, 2, "FS", C, 3, "S", D, 4, "", E, 5, "", F, 6, "" ],
  [ 6,
    B, 2, "FS", D, 4, "", E, 5, "", F, 6, "" ],
- [ 6, 3, 0,
+ [ 6, 3, 6,
    2, 2, "FS", 4, 4, "", 5, 5, "", 6, 6, "" ],
 );
 test("max messages + expunge", \@x50, \@X51, @O51);

src/sync.c

@@ -167,7 +167,6 @@ typedef struct {
 	uint ref_count, nsrecs, opts[2];
 	uint new_pending[2], flags_pending[2], trash_pending[2];
 	uint maxuid[2];     // highest UID that was already propagated
-	uint newmaxuid[2];  // highest UID that is currently being propagated
 	uint uidval[2];     // UID validity value
 	uint newuidval[2];  // UID validity obtained from driver
 	uint finduid[2];    // TUID lookup makes sense only for UIDs >= this
@@ -285,6 +284,8 @@ match_tuids( sync_vars_t *svars, int t, message_t *msgs )
 			srec->msg[t] = tmsg;
 			ntmsg = tmsg->next;
 			srec->uid[t] = tmsg->uid;
+			if (tmsg->uid == svars->maxuid[t] + 1)
+				svars->maxuid[t] = tmsg->uid;
 			srec->status = 0;
 			srec->tuid[0] = 0;
 		}
@@ -613,7 +614,7 @@ clean_strdup( const char *s )
 }
 
 
-#define JOURNAL_VERSION "3"
+#define JOURNAL_VERSION "4"
 
 static int
 prepare_state( sync_vars_t *svars )
@@ -853,8 +854,6 @@ load_state( sync_vars_t *svars )
 		svars->maxxfuid = minwuid - 1;
 	}
 
-	svars->newmaxuid[F] = svars->maxuid[F];
-	svars->newmaxuid[N] = svars->maxuid[N];
 	int line = 0;
 	if ((jfp = fopen( svars->jname, "r" ))) {
 		if (!lock_state( svars ))
@@ -884,17 +883,17 @@ load_state( sync_vars_t *svars )
 				uint t1, t2, t3;
 				if ((c = buf[0]) == '#' ?
 				      (tn = 0, (sscanf( buf + 2, "%u %u %n", &t1, &t2, &tn ) < 2) || !tn || (ll - (uint)tn != TUIDL + 2)) :
-				      c == 'S' || c == '!' ?
+				      c == '!' ?
 				        (sscanf( buf + 2, "%u", &t1 ) != 1) :
-				        c == 'F' || c == 'T' || c == '+' || c == '&' || c == '-' || c == '=' || c == '|' ?
+				        c == 'N' || c == 'F' || c == 'T' || c == '+' || c == '&' || c == '-' || c == '=' || c == '|' ?
 				          (sscanf( buf + 2, "%u %u", &t1, &t2 ) != 2) :
 				          (sscanf( buf + 2, "%u %u %u", &t1, &t2, &t3 ) != 3))
 				{
 					error( "Error: malformed journal entry at %s:%d\n", svars->jname, line );
 					goto jbail;
 				}
-				if (c == 'S')
+				if (c == 'N')
-					svars->maxuid[t1] = svars->newmaxuid[t1];
+					svars->maxuid[t1] = t2;
 				else if (c == 'F')
 					svars->finduid[t1] = t2;
 				else if (c == 'T')
@@ -908,10 +907,6 @@ load_state( sync_vars_t *svars )
 					srec = nfcalloc( sizeof(*srec) );
 					srec->uid[F] = t1;
 					srec->uid[N] = t2;
-					if (svars->newmaxuid[F] < t1)
-						svars->newmaxuid[F] = t1;
-					if (svars->newmaxuid[N] < t2)
-						svars->newmaxuid[N] = t2;
 					debug( "  new entry(%u,%u)\n", t1, t2 );
 					srec->status = S_PENDING;
 					*svars->srecadd = srec;
@@ -935,6 +930,7 @@ load_state( sync_vars_t *svars )
 						break;
 					case '=':
 						debug( "aborted\n" );
+						if (svars->maxxfuid < srec->uid[F])
 							svars->maxxfuid = srec->uid[F];
 						srec->status = S_DEAD;
 						break;
@@ -949,12 +945,16 @@ load_state( sync_vars_t *svars )
 					case '<':
 						debug( "far side now %u\n", t3 );
 						srec->uid[F] = t3;
+						if (t3 == svars->maxuid[F] + 1)
+							svars->maxuid[F] = t3;
 						srec->status &= ~S_PENDING;
 						srec->tuid[0] = 0;
 						break;
 					case '>':
 						debug( "near side now %u\n", t3 );
 						srec->uid[N] = t3;
+						if (t3 == svars->maxuid[N] + 1)
+							svars->maxuid[N] = t3;
 						srec->status &= ~S_PENDING;
 						srec->tuid[0] = 0;
 						break;
@@ -1575,22 +1575,26 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 	for (t = 0; t < 2; t++) {
 		debug( "synchronizing new messages on %s\n", str_fn[1-t] );
 		for (tmsg = svars->msgs[1-t]; tmsg; tmsg = tmsg->next) {
-			// If messages were previously ignored due to being excessive, they would now
-			// appear to be newer than the messages that got actually synced, so increment
-			// newmaxuid immediately to make sure we always look only at the newest ones.
-			// However, committing it to maxuid must be delayed until all messages were
-			// propagated, to ensure that all pending messages are still loaded next time
-			// in case of interruption - in particular skipping big messages would otherwise
-			// up the limit too early.
 			srec = tmsg->srec;
 			if (srec) {
 				if (srec->status & S_SKIPPED) {
 					// The message was skipped due to being too big.
+					// We must have already seen the UID, but we might have been interrupted.
+					if (svars->maxuid[1-t] < tmsg->uid)
+						svars->maxuid[1-t] = tmsg->uid;
 					if (!(svars->chan->ops[t] & OP_RENEW))
 						continue;
 				} else {
 					if (!(svars->chan->ops[t] & OP_NEW))
 						continue;
+					// This catches messages:
+					// - that are actually new
+					// - whose propagation got interrupted
+					// - whose propagation was completed, but not logged yet
+					// - that aren't actually new, but a result of syncing, and the instant
+					//   maxuid upping was prevented by the presence of actually new messages
+					if (svars->maxuid[1-t] < tmsg->uid)
+						svars->maxuid[1-t] = tmsg->uid;
 					if (!(srec->status & S_PENDING))
 						continue;  // Nothing to do - the message is paired or expired
 					// Propagation was scheduled, but we got interrupted
@@ -1606,13 +1610,14 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 			} else {
 				if (!(svars->chan->ops[t] & OP_NEW))
 					continue;
-				if (tmsg->uid <= svars->newmaxuid[1-t]) {
+				if (tmsg->uid <= svars->maxuid[1-t]) {
 					// The message should be already paired. It's not, so it was:
 					// - previously paired, but the entry was expired and pruned => ignore
 					// - attempted, but failed => ignore (the wisdom of this is debatable)
 					// - ignored, as it would have been expunged anyway => ignore (even if undeleted)
 					continue;
 				}
+				svars->maxuid[1-t] = tmsg->uid;
 				debug( "new message %u\n", tmsg->uid );
 
 				if ((svars->chan->ops[t] & OP_EXPUNGE) && (tmsg->flags & F_DELETED)) {
@@ -1628,8 +1633,6 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 				srec->uid[1-t] = tmsg->uid;
 				srec->msg[1-t] = tmsg;
 				tmsg->srec = srec;
-				if (svars->newmaxuid[1-t] < tmsg->uid)
-					svars->newmaxuid[1-t] = tmsg->uid;
 				JLOG( "+ %u %u", (srec->uid[F], srec->uid[N]), "fresh" );
 			}
 			if ((tmsg->flags & F_FLAGGED) || tmsg->size <= svars->chan->stores[t]->max_size) {
@@ -1748,6 +1751,7 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 					// If we have so many new messages that some of them are instantly expired,
 					// but some are still propagated because they are important, we need to
 					// ensure explicitly that the bulk fetch limit is upped.
+					if (svars->maxxfuid < srec->uid[F])
 						svars->maxxfuid = srec->uid[F];
 					srec->msg[F]->srec = NULL;
 					srec->status = S_DEAD;
@@ -1849,6 +1853,8 @@ msg_copied( int sts, uint uid, copy_vars_t *vars )
 		} else {
 			JLOG( "%c %u %u %u", ("<>"[t], srec->uid[F], srec->uid[N], uid), "%sed message", str_hl[t] );
 			vars->srec->uid[t] = uid;
+			if (uid == svars->maxuid[t] + 1)
+				svars->maxuid[t] = uid;
 			vars->srec->status &= ~S_PENDING;
 			vars->srec->tuid[0] = 0;
 		}
@@ -1918,10 +1924,6 @@ msgs_copied( sync_vars_t *svars, int t )
 	if (svars->new_pending[t])
 		goto out;
 
-	if (svars->maxuid[1-t] != svars->newmaxuid[1-t]) {
-		svars->maxuid[1-t] = svars->newmaxuid[1-t];
-		JLOG( "S %d", 1-t, "commit maxuid of %s", str_fn[1-t] );
-	}
 	sync_close( svars, 1-t );
 	if (check_cancel( svars ))
 		goto out;
@@ -2159,6 +2161,14 @@ box_closed_p2( sync_vars_t *svars, int t )
 	// the operations are idempotent, and we're about to commit the new state
 	// right afterwards anyway.
 
+	for (t = 0; t < 2; t++) {
+		// Committing maxuid is delayed until all messages were propagated, to
+		// ensure that all pending messages are still loaded next time in case
+		// of interruption - in particular skipping big messages would otherwise
+		// up the limit too early.
+		JLOG( "N %d %u", (t, svars->maxuid[t]), "up maxuid of %s", str_fn[t] );
+	}
+
 	if (((svars->state[F] | svars->state[N]) & ST_DID_EXPUNGE) || svars->chan->max_messages) {
 		debug( "purging obsolete entries\n" );
 		for (srec = svars->srecs; srec; srec = srec->next) {