lobo/isync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rewrite tracking of highest expired UID

Browse Source 
so far, we tracked the slave side, and calculated the master side on the
fly. that complicated things unnecessarily.
This commit is contained in:
Oswald Buddenhagen 2017-04-02 14:04:43 +02:00
parent 677accfd84
commit 36666f7e52
2 changed files with 66 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/run-tests.pl
View File

@ -195,7 +195,7 @@ my @X31 = (
1, 1, "F", 2, 2, "", 3, 3, "S", 4, 4, "", 5, 5, "S", 6, 6, "" ], 1, 1, "F", 2, 2, "", 3, 3, "S", 4, 4, "", 5, 5, "S", 6, 6, "" ],
[ 5, [ 5,
1, 1, "F", 2, 2, "", 4, 3, "", 5, 4, "S", 6, 5, "" ], 1, 1, "F", 2, 2, "", 4, 3, "", 5, 4, "S", 6, 5, "" ],
[ 6, 2, 0, [ 6, 3, 0,
1, 1, "F", 2, 2, "", 4, 3, "", 5, 4, "S", 6, 5, "" ], 1, 1, "F", 2, 2, "", 4, 3, "", 5, 4, "S", 6, 5, "" ],
); );
test("max messages", \@x30, \@X31, @O31); test("max messages", \@x30, \@X31, @O31);
@ -207,7 +207,7 @@ my @X32 = (
1, 1, "F", 2, 2, "", 3, 3, "S", 4, 4, "", 5, 5, "S", 6, 6, "" ], 1, 1, "F", 2, 2, "", 3, 3, "S", 4, 4, "", 5, 5, "S", 6, 6, "" ],
[ 4, [ 4,
1, 1, "F", 4, 2, "", 5, 3, "S", 6, 4, "" ], 1, 1, "F", 4, 2, "", 5, 3, "S", 6, 4, "" ],
[ 6, 1, 0, [ 6, 3, 0,
1, 1, "F", 4, 2, "", 5, 3, "S", 6, 4, "" ], 1, 1, "F", 4, 2, "", 5, 3, "S", 6, 4, "" ],
); );
test("max messages vs. unread", \@x30, \@X32, @O32); test("max messages vs. unread", \@x30, \@X32, @O32);
@ -367,7 +367,7 @@ sub showbox($)
# $filename # $filename
# Output: # Output:
# [ maxuid[M], smaxxuid, maxuid[S], # [ maxuid[M], mmaxxuid, maxuid[S],
# uid[M], uid[S], "flags", ... ], # uid[M], uid[S], "flags", ... ],
sub showstate($) sub showstate($)
{ {
@ -396,7 +396,7 @@ sub showstate($)
return; return;
} }
my @T = ($hdr{'MaxPulledUid'} // "missing", my @T = ($hdr{'MaxPulledUid'} // "missing",
$hdr{'MaxExpiredSlaveUid'} // "0", $hdr{'MaxExpiredMasterUid'} // "0",
$hdr{'MaxPushedUid'} // "missing"); $hdr{'MaxPushedUid'} // "missing");
for (@ls) { for (@ls) {
/^(-?\d+) (-?\d+) (.*)$/; /^(-?\d+) (-?\d+) (.*)$/;
@ -472,7 +472,7 @@ sub mkchan($$@)
open(FILE, ">", "slave/.mbsyncstate") or open(FILE, ">", "slave/.mbsyncstate") or
die "Cannot create sync state.\n"; die "Cannot create sync state.\n";
print FILE "MasterUidValidity 1\nMaxPulledUid ".shift(@t)."\n". print FILE "MasterUidValidity 1\nMaxPulledUid ".shift(@t)."\n".
"SlaveUidValidity 1\nMaxExpiredSlaveUid ".shift(@t)."\nMaxPushedUid ".shift(@t)."\n\n"; "SlaveUidValidity 1\nMaxExpiredMasterUid ".shift(@t)."\nMaxPushedUid ".shift(@t)."\n\n";
while (@t) { while (@t) {
print FILE shift(@t)." ".shift(@t)." ".shift(@t)."\n"; print FILE shift(@t)." ".shift(@t)." ".shift(@t)."\n";
} }
@ -515,13 +515,13 @@ sub ckbox($$$@)
# $filename, @syncstate # $filename, @syncstate
sub ckstate($@) sub ckstate($@)
{ {
my ($fn, $mmaxuid, $smaxxuid, $smaxuid, @T) = @_; my ($fn, $mmaxuid, $mmaxxuid, $smaxuid, @T) = @_;
my %hdr; my %hdr;
$hdr{'MasterUidValidity'} = "1"; $hdr{'MasterUidValidity'} = "1";
$hdr{'SlaveUidValidity'} = "1"; $hdr{'SlaveUidValidity'} = "1";
$hdr{'MaxPulledUid'} = $mmaxuid; $hdr{'MaxPulledUid'} = $mmaxuid;
$hdr{'MaxPushedUid'} = $smaxuid; $hdr{'MaxPushedUid'} = $smaxuid;
$hdr{'MaxExpiredSlaveUid'} = $smaxxuid if ($smaxxuid ne 0); $hdr{'MaxExpiredMasterUid'} = $mmaxxuid if ($mmaxxuid ne 0);
open(FILE, "<", $fn) or die "Cannot read sync state $fn.\n"; open(FILE, "<", $fn) or die "Cannot read sync state $fn.\n";
chomp(my @ls = <FILE>); chomp(my @ls = <FILE>);
close FILE; close FILE;

125
src/sync.c
View File

@ -159,8 +159,7 @@ typedef struct {
int newmaxuid[2]; /* highest UID that is currently being propagated */ int newmaxuid[2]; /* highest UID that is currently being propagated */
int uidval[2]; /* UID validity value */ int uidval[2]; /* UID validity value */
int newuid[2]; /* TUID lookup makes sense only for UIDs >= this */ int newuid[2]; /* TUID lookup makes sense only for UIDs >= this */
int mmaxxuid; /* highest expired UID on master during new message propagation */ int mmaxxuid; /* highest expired UID on master */
int smaxxuid; /* highest expired UID on slave */
} sync_vars_t; } sync_vars_t;
static void sync_ref( sync_vars_t *svars ) { ++svars->ref_count; } static void sync_ref( sync_vars_t *svars ) { ++svars->ref_count; }
@ -649,8 +648,8 @@ save_state( sync_vars_t *svars )
Fprintf( svars->nfp, Fprintf( svars->nfp,
"MasterUidValidity %d\nSlaveUidValidity %d\nMaxPulledUid %d\nMaxPushedUid %d\n", "MasterUidValidity %d\nSlaveUidValidity %d\nMaxPulledUid %d\nMaxPushedUid %d\n",
svars->uidval[M], svars->uidval[S], svars->maxuid[M], svars->maxuid[S] ); svars->uidval[M], svars->uidval[S], svars->maxuid[M], svars->maxuid[S] );
if (svars->smaxxuid) if (svars->mmaxxuid)
Fprintf( svars->nfp, "MaxExpiredSlaveUid %d\n", svars->smaxxuid ); Fprintf( svars->nfp, "MaxExpiredMasterUid %d\n", svars->mmaxxuid );
Fprintf( svars->nfp, "\n" ); Fprintf( svars->nfp, "\n" );
for (srec = svars->srecs; srec; srec = srec->next) { for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD) if (srec->status & S_DEAD)
@ -678,6 +677,7 @@ load_state( sync_vars_t *svars )
char *s; char *s;
FILE *jfp; FILE *jfp;
int line, t, t1, t2, t3; int line, t, t1, t2, t3;
int smaxxuid = 0;
char c; char c;
struct stat st; struct stat st;
char fbuf[16]; /* enlarge when support for keywords is added */ char fbuf[16]; /* enlarge when support for keywords is added */
@ -701,7 +701,7 @@ load_state( sync_vars_t *svars )
if (line == 1 && isdigit( buf[0] )) { if (line == 1 && isdigit( buf[0] )) {
if (sscanf( buf, "%63s %63s", buf1, buf2 ) != 2 || if (sscanf( buf, "%63s %63s", buf1, buf2 ) != 2 ||
sscanf( buf1, "%d:%d", &svars->uidval[M], &svars->maxuid[M] ) < 2 || sscanf( buf1, "%d:%d", &svars->uidval[M], &svars->maxuid[M] ) < 2 ||
sscanf( buf2, "%d:%d:%d", &svars->uidval[S], &svars->smaxxuid, &svars->maxuid[S] ) < 3) { sscanf( buf2, "%d:%d:%d", &svars->uidval[S], &smaxxuid, &svars->maxuid[S] ) < 3) {
error( "Error: invalid sync state header in %s\n", svars->dname ); error( "Error: invalid sync state header in %s\n", svars->dname );
goto jbail; goto jbail;
} }
@ -719,8 +719,10 @@ load_state( sync_vars_t *svars )
svars->maxuid[M] = t1; svars->maxuid[M] = t1;
else if (!strcmp( buf1, "MaxPushedUid" )) else if (!strcmp( buf1, "MaxPushedUid" ))
svars->maxuid[S] = t1; svars->maxuid[S] = t1;
else if (!strcmp( buf1, "MaxExpiredSlaveUid" )) else if (!strcmp( buf1, "MaxExpiredMasterUid" ))
svars->smaxxuid = t1; svars->mmaxxuid = t1;
else if (!strcmp( buf1, "MaxExpiredSlaveUid" )) // Legacy
smaxxuid = t1;
else { else {
error( "Error: unrecognized sync state header entry at %s:%d\n", svars->dname, line ); error( "Error: unrecognized sync state header entry at %s:%d\n", svars->dname, line );
goto jbail; goto jbail;
@ -768,9 +770,36 @@ load_state( sync_vars_t *svars )
} }
svars->existing = 0; svars->existing = 0;
} }
// This is legacy support for pre-1.3 sync states.
if (smaxxuid) {
int minwuid = INT_MAX;
for (srec = svars->srecs; srec; srec = srec->next) {
if ((srec->status & S_DEAD) || srec->uid[M] <= 0)
continue;
if (srec->status & S_EXPIRED) {
if (!srec->uid[S]) {
// The expired message was already gone.
continue;
}
// The expired message was not expunged yet, so re-examine it.
// This will happen en masse, so just extend the bulk fetch.
} else {
if (srec->uid[S] > 0 && smaxxuid >= srec->uid[S]) {
// The non-expired message is in the generally expired range,
// so don't make it contribute to the bulk fetch.
continue;
}
// Usual non-expired message.
}
if (minwuid > srec->uid[M])
minwuid = srec->uid[M];
}
svars->mmaxxuid = minwuid - 1;
}
svars->newmaxuid[M] = svars->maxuid[M]; svars->newmaxuid[M] = svars->maxuid[M];
svars->newmaxuid[S] = svars->maxuid[S]; svars->newmaxuid[S] = svars->maxuid[S];
svars->mmaxxuid = INT_MAX;
line = 0; line = 0;
if ((jfp = fopen( svars->jname, "r" ))) { if ((jfp = fopen( svars->jname, "r" ))) {
if (!lock_state( svars )) if (!lock_state( svars ))
@ -814,7 +843,7 @@ load_state( sync_vars_t *svars )
else if (c == 'T') else if (c == 'T')
*int_array_append( &svars->trashed_msgs[t1] ) = t2; *int_array_append( &svars->trashed_msgs[t1] ) = t2;
else if (c == '!') else if (c == '!')
svars->smaxxuid = t1; svars->mmaxxuid = t1;
else if (c == '|') { else if (c == '|') {
svars->uidval[M] = t1; svars->uidval[M] = t1;
svars->uidval[S] = t2; svars->uidval[S] = t2;
@ -897,11 +926,9 @@ load_state( sync_vars_t *svars )
case '/': case '/':
t3 = (srec->status & S_EXPIRE); t3 = (srec->status & S_EXPIRE);
debug( "expired now %d\n", t3 / S_EXPIRE ); debug( "expired now %d\n", t3 / S_EXPIRE );
if (t3) { if (t3)
if (svars->smaxxuid < srec->uid[S])
svars->smaxxuid = srec->uid[S];
srec->status |= S_EXPIRED; srec->status |= S_EXPIRED;
} else else
srec->status &= ~S_EXPIRED; srec->status &= ~S_EXPIRED;
break; break;
default: default:
@ -919,6 +946,7 @@ load_state( sync_vars_t *svars )
} }
} }
svars->replayed = line; svars->replayed = line;
return 1; return 1;
} }
@ -1238,31 +1266,9 @@ box_opened2( sync_vars_t *svars, int t )
/* When messages have been expired on the slave, the master fetch is split into /* When messages have been expired on the slave, the master fetch is split into
* two ranges: The bulk fetch which corresponds with the most recent messages, and an * two ranges: The bulk fetch which corresponds with the most recent messages, and an
* exception list of messages which would have been expired if they weren't important. */ * exception list of messages which would have been expired if they weren't important. */
debug( "preparing master selection - max expired slave uid is %d\n", svars->smaxxuid ); debug( "preparing master selection - max expired master uid is %d\n", svars->mmaxxuid );
/* First, find out the lower bound for the bulk fetch. */ /* First, find out the lower bound for the bulk fetch. */
minwuid = INT_MAX; minwuid = svars->mmaxxuid + 1;
for (srec = svars->srecs; srec; srec = srec->next) {
if ((srec->status & S_DEAD) || srec->uid[M] <= 0)
continue;
if (srec->status & S_EXPIRED) {
if (!srec->uid[S]) {
/* The expired message was already gone. */
continue;
}
/* The expired message was not expunged yet, so re-examine it.
* This will happen en masse, so just extend the bulk fetch. */
} else {
if (svars->smaxxuid >= srec->uid[S]) {
/* The non-expired message is in the generally expired range, so don't
* make it contribute to the bulk fetch. */
continue;
}
/* Usual non-expired message. */
}
if (minwuid > srec->uid[M])
minwuid = srec->uid[M];
}
debug( " min non-orphaned master uid is %d\n", minwuid );
/* Next, calculate the exception fetch. */ /* Next, calculate the exception fetch. */
for (srec = svars->srecs; srec; srec = srec->next) { for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD) if (srec->status & S_DEAD)
@ -1665,6 +1671,8 @@ box_loaded( int sts, void *aux )
/* The message is excess or was already (being) expired. */ /* The message is excess or was already (being) expired. */
srec->status |= S_NEXPIRE; srec->status |= S_NEXPIRE;
debug( " old pair(%d,%d) expired\n", srec->uid[M], srec->uid[S] ); debug( " old pair(%d,%d) expired\n", srec->uid[M], srec->uid[S] );
if (svars->mmaxxuid < srec->uid[M])
svars->mmaxxuid = srec->uid[M];
todel--; todel--;
} }
} }
@ -1721,6 +1729,9 @@ box_loaded( int sts, void *aux )
if (srec->status & S_NEXPIRE) { if (srec->status & S_NEXPIRE) {
jFprintf( svars, "= %d %d\n", srec->uid[M], srec->uid[S] ); jFprintf( svars, "= %d %d\n", srec->uid[M], srec->uid[S] );
debug( " pair(%d,%d): 1 (abort)\n", srec->uid[M], srec->uid[S] ); debug( " pair(%d,%d): 1 (abort)\n", srec->uid[M], srec->uid[S] );
// If we have so many new messages that some of them are instantly expired,
// but some are still propagated because they are important, we need to
// ensure explicitly that the bulk fetch limit is upped.
svars->mmaxxuid = srec->uid[M]; svars->mmaxxuid = srec->uid[M];
srec->msg[M]->srec = 0; srec->msg[M]->srec = 0;
srec->status = S_DEAD; srec->status = S_DEAD;
@ -1849,16 +1860,6 @@ msg_copied_p2( sync_vars_t *svars, sync_rec_t *srec, int t, int uid )
srec->uid[t] = uid; srec->uid[t] = uid;
srec->tuid[0] = 0; srec->tuid[0] = 0;
} }
if (t == S && svars->mmaxxuid < srec->uid[M]) {
/* If we have so many new messages that some of them are instantly expired,
* but some are still propagated because they are important, we need to
* ensure explicitly that the bulk fetch limit is upped. */
svars->mmaxxuid = INT_MAX;
if (svars->smaxxuid < srec->uid[S] - 1) {
svars->smaxxuid = srec->uid[S] - 1;
jFprintf( svars, "! %d\n", svars->smaxxuid );
}
}
} }
static void msgs_found_new( int sts, void *aux ); static void msgs_found_new( int sts, void *aux );
@ -1984,8 +1985,6 @@ flags_set_p2( sync_vars_t *svars, sync_rec_t *srec, int t )
if (t == S) { if (t == S) {
uint nex = (srec->status / S_NEXPIRE) & 1; uint nex = (srec->status / S_NEXPIRE) & 1;
if (nex != ((srec->status / S_EXPIRED) & 1)) { if (nex != ((srec->status / S_EXPIRED) & 1)) {
if (nex && (svars->smaxxuid < srec->uid[S]))
svars->smaxxuid = srec->uid[S];
jFprintf( svars, "/ %d %d\n", srec->uid[M], srec->uid[S] ); jFprintf( svars, "/ %d %d\n", srec->uid[M], srec->uid[S] );
debug( " pair(%d,%d): expired %d (commit)\n", srec->uid[M], srec->uid[S], nex ); debug( " pair(%d,%d): expired %d (commit)\n", srec->uid[M], srec->uid[S], nex );
srec->status = (srec->status & ~S_EXPIRED) | (nex * S_EXPIRED); srec->status = (srec->status & ~S_EXPIRED) | (nex * S_EXPIRED);
@ -2144,35 +2143,23 @@ static void
box_closed_p2( sync_vars_t *svars, int t ) box_closed_p2( sync_vars_t *svars, int t )
{ {
sync_rec_t *srec; sync_rec_t *srec;
int minwuid;
svars->state[t] |= ST_CLOSED; svars->state[t] |= ST_CLOSED;
if (!(svars->state[1-t] & ST_CLOSED)) if (!(svars->state[1-t] & ST_CLOSED))
return; return;
// All the journalling done in this function is merely for the autotest -
// the operations are idempotent, and we're about to commit the new state
// right afterwards anyway.
if (((svars->state[M] | svars->state[S]) & ST_DID_EXPUNGE) || svars->chan->max_messages) { if (((svars->state[M] | svars->state[S]) & ST_DID_EXPUNGE) || svars->chan->max_messages) {
debug( "purging obsolete entries\n" ); debug( "purging obsolete entries\n" );
minwuid = INT_MAX;
if (svars->chan->max_messages) {
debug( " max expired slave uid is %d\n", svars->smaxxuid );
for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD)
continue;
if (!((srec->uid[S] <= 0 || ((srec->status & S_DEL(S)) && (svars->state[S] & ST_DID_EXPUNGE))) &&
(srec->uid[M] <= 0 || ((srec->status & S_DEL(M)) && (svars->state[M] & ST_DID_EXPUNGE)) || (srec->status & S_EXPIRED))) &&
svars->smaxxuid < srec->uid[S] && minwuid > srec->uid[M])
minwuid = srec->uid[M];
}
debug( " min non-orphaned master uid is %d\n", minwuid );
}
for (srec = svars->srecs; srec; srec = srec->next) { for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD) if (srec->status & S_DEAD)
continue; continue;
if (srec->uid[S] <= 0 || ((srec->status & S_DEL(S)) && (svars->state[S] & ST_DID_EXPUNGE))) { if (srec->uid[S] <= 0 || ((srec->status & S_DEL(S)) && (svars->state[S] & ST_DID_EXPUNGE))) {
if (srec->uid[M] <= 0 || ((srec->status & S_DEL(M)) && (svars->state[M] & ST_DID_EXPUNGE)) || if (srec->uid[M] <= 0 || ((srec->status & S_DEL(M)) && (svars->state[M] & ST_DID_EXPUNGE)) ||
((srec->status & S_EXPIRED) && svars->maxuid[M] >= srec->uid[M] && minwuid > srec->uid[M])) { ((srec->status & S_EXPIRED) && svars->maxuid[M] >= srec->uid[M] && svars->mmaxxuid >= srec->uid[M])) {
debug( " -> killing (%d,%d)\n", srec->uid[M], srec->uid[S] ); debug( " -> killing (%d,%d)\n", srec->uid[M], srec->uid[S] );
srec->status = S_DEAD; srec->status = S_DEAD;
jFprintf( svars, "- %d %d\n", srec->uid[M], srec->uid[S] ); jFprintf( svars, "- %d %d\n", srec->uid[M], srec->uid[S] );
@ -2189,6 +2176,12 @@ box_closed_p2( sync_vars_t *svars, int t )
} }
} }
// This is just an optimization, so it needs no journaling of intermediate states.
// However, doing it before the entry purge would require ensuring that the
// exception list includes all relevant messages.
debug( "max expired uid on master is now %d\n", svars->mmaxxuid );
jFprintf( svars, "! %d\n", svars->mmaxxuid );
save_state( svars ); save_state( svars );
sync_bail( svars ); sync_bail( svars );