lobo/isync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement Message-Id based UIDVALIDITY recovery

Browse Source
This commit is contained in:
Oswald Buddenhagen 2016-12-18 20:50:20 +01:00
parent f9fe75602e
commit 77acc26812
8 changed files with 150 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@ -6,6 +6,8 @@ A Maildir sub-folder naming style without extra dots has been added.
Support for TLS client certificates was added.
Support for recovering from baseless UID validity changes was added.
[1.2.0]
The 'isync' compatibility wrapper is now deprecated.

10
README
View File

@ -45,16 +45,6 @@ isync executable still exists; it is a compatibility wrapper around mbsync.
Courier 1.4.3 is known to be buggy, version 1.7.3 works fine.
c-client (UW-IMAP, Pine) is mostly fine, but versions less than 2004a.352
tend to change UIDVALIDITY pretty often when used with unix/mbox mailboxes,
making isync refuse synchronization.
M$ Exchange (up to 2010 at least) occasionally exposes the same problem.
The "cure" is to simply copy the new UIDVALIDITY from the affected
mailbox to mbsync's state file. This is a Bad Hack (TM), but it works -
use at your own risk (if the UIDVALIDITY change was genuine, this will
delete all messages in the affected mailbox - not that this ever
happened to me).
M$ Exchange (2013 at least) needs DisableExtension MOVE to be compatible
with the Trash functionality.

1
src/driver.c
View File

@ -34,6 +34,7 @@ free_generic_messages( message_t *msgs )
for (; msgs; msgs = tmsg) {
tmsg = msgs->next;
free( msgs->msgid );
free( msgs );
}
}

3
src/driver.h
View File

@ -63,6 +63,7 @@ typedef struct store_conf {
typedef struct message {
struct message *next;
struct sync_rec *srec;
char *msgid; /* owned */
/* string_list_t *keywords; */
size_t size; /* zero implies "not fetched" */
int uid;
@ -80,6 +81,7 @@ typedef struct message {
#define OPEN_SETFLAGS (1<<6)
#define OPEN_APPEND (1<<7)
#define OPEN_FIND (1<<8)
#define OPEN_OLD_IDS (1<<9)
typedef struct store {
struct store *next;
@ -208,6 +210,7 @@ struct driver {
* and those named in the excs array (smaller than minuid).
* The driver takes ownership of the excs array.
* Messages starting with newuid need to have the TUID populated when OPEN_FIND is set.
* Messages up to seenuid need to have the Message-Id populated when OPEN_OLD_IDS is set.
* Messages up to seenuid need to have the size populated when OPEN_OLD_SIZE is set;
* likewise messages above seenuid when OPEN_NEW_SIZE is set. */
void (*load_box)( store_t *ctx, int minuid, int maxuid, int newuid, int seenuid, int_array_t excs,

55
src/drv_imap.c
View File

@ -905,7 +905,7 @@ static int
parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
{
list_t *tmp, *flags;
char *body = 0, *tuid = 0;
char *body = 0, *tuid = 0, *msgid = 0;
imap_message_t *cur;
msg_data_t *msgdata;
struct imap_cmd *cmdp;
@ -983,8 +983,42 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
tmp = tmp->next;
if (!is_atom( tmp ))
goto bfail;
if (starts_with_upper( tmp->val, tmp->len, "X-TUID: ", 8 ))
tuid = tmp->val + 8;
int off, in_msgid = 0;
for (char *val = tmp->val, *end; (end = strchr( val, '\n' )); val = end + 1) {
int len = (int)(end - val);
if (len && end[-1] == '\r')
len--;
if (!len)
break;
if (starts_with_upper( val, len, "X-TUID: ", 8 )) {
if (len < 8 + TUIDL) {
error( "IMAP error: malformed X-TUID header (UID %d)\n", uid );
continue;
}
tuid = val + 8;
in_msgid = 0;
continue;
}
if (starts_with_upper( val, len, "MESSAGE-ID:", 11 )) {
off = 11;
} else if (in_msgid) {
if (!isspace( val[0] )) {
in_msgid = 0;
continue;
}
off = 1;
} else {
continue;
}
while (off < len && isspace( val[off] ))
off++;
if (off == len) {
in_msgid = 1;
continue;
}
msgid = nfstrndup( val + off, len - off );
in_msgid = 0;
}
} else {
bfail:
error( "IMAP error: unable to parse BODY[HEADER.FIELDS ...]\n" );
@ -1018,6 +1052,7 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
cur->gen.status = status;
cur->gen.size = size;
cur->gen.srec = 0;
cur->gen.msgid = msgid;
if (tuid)
strncpy( cur->gen.tuid, tuid, TUIDL );
else
@ -2322,7 +2357,7 @@ imap_prepare_load_box( store_t *gctx, int opts )
gctx->opts = opts;
}
enum { WantSize = 1, WantTuids = 2 };
enum { WantSize = 1, WantTuids = 2, WantMsgids = 4 };
typedef struct imap_range {
int first, last, flags;
} imap_range_t;
@ -2375,7 +2410,7 @@ imap_load_box( store_t *gctx, int minuid, int maxuid, int newuid, int seenuid, i
if (i != j)
bl += sprintf( buf + bl, ":%d", excs.data[i] );
}
imap_submit_load( ctx, buf, 0, sts );
imap_submit_load( ctx, buf, shifted_bit( ctx->gen.opts, OPEN_OLD_IDS, WantMsgids ), sts );
}
if (maxuid == INT_MAX)
maxuid = ctx->gen.uidnext ? ctx->gen.uidnext - 1 : 0x7fffffff;
@ -2390,6 +2425,8 @@ imap_load_box( store_t *gctx, int minuid, int maxuid, int newuid, int seenuid, i
shifted_bit( ctx->gen.opts, OPEN_NEW_SIZE, WantSize), seenuid );
if (ctx->gen.opts & OPEN_FIND)
imap_set_range( ranges, &nranges, 0, WantTuids, newuid - 1 );
if (ctx->gen.opts & OPEN_OLD_IDS)
imap_set_range( ranges, &nranges, WantMsgids, 0, seenuid );
for (int r = 0; r < nranges; r++) {
sprintf( buf, "%d:%d", ranges[r].first, ranges[r].last );
imap_submit_load( ctx, buf, ranges[r].flags, sts );
@ -2404,10 +2441,14 @@ static void
imap_submit_load( imap_store_t *ctx, const char *buf, int flags, struct imap_cmd_refcounted_state *sts )
{
imap_exec( ctx, imap_refcounted_new_cmd( sts ), imap_refcounted_done_box,
"UID FETCH %s (UID%s%s%s)", buf,
"UID FETCH %s (UID%s%s%s%s%s%s%s)", buf,
(ctx->gen.opts & OPEN_FLAGS) ? " FLAGS" : "",
(flags & WantSize) ? " RFC822.SIZE" : "",
(flags & WantTuids) ? " BODY.PEEK[HEADER.FIELDS (X-TUID)]" : "");
(flags & (WantTuids | WantMsgids)) ? " BODY.PEEK[HEADER.FIELDS (" : "",
(flags & WantTuids) ? "X-TUID" : "",
!(~flags & (WantTuids | WantMsgids)) ? " " : "",
(flags & WantMsgids) ? "MESSAGE-ID" : "",
(flags & (WantTuids | WantMsgids)) ? ")]" : "");
}
/******************* imap_fetch_msg *******************/

52
src/drv_maildir.c
View File

@ -442,6 +442,7 @@ static const char *subdirs[] = { "cur", "new", "tmp" };
typedef struct {
char *base;
char *msgid;
int size;
uint uid:31, recent:1;
char tuid[TUIDL];
@ -926,6 +927,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
continue;
entry = msg_t_array_append( msglist );
entry->base = nfstrdup( e->d_name );
entry->msgid = 0;
entry->uid = uid;
entry->recent = i;
entry->size = 0;
@ -1050,7 +1052,8 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
}
int want_size = (uid > ctx->seenuid) ? (ctx->gen.opts & OPEN_NEW_SIZE) : (ctx->gen.opts & OPEN_OLD_SIZE);
int want_tuid = ((ctx->gen.opts & OPEN_FIND) && uid >= ctx->newuid);
if (!want_size && !want_tuid)
int want_msgid = ((ctx->gen.opts & OPEN_OLD_IDS) && uid <= ctx->seenuid);
if (!want_size && !want_tuid && !want_msgid)
continue;
if (!fnl)
nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base );
@ -1064,7 +1067,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
}
entry->size = st.st_size;
}
if (want_tuid) {
if (want_tuid || want_msgid) {
if (!(f = fopen( buf, "r" ))) {
if (errno != ENOENT) {
sys_error( "Maildir error: cannot open %s", buf );
@ -1072,13 +1075,45 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
}
goto retry;
}
while (fgets( nbuf, sizeof(nbuf), f )) {
if (!nbuf[0] || nbuf[0] == '\n')
break;
if (starts_with( nbuf, -1, "X-TUID: ", 8 ) && nbuf[8 + TUIDL] == '\n') {
memcpy( entry->tuid, nbuf + 8, TUIDL );
int off, in_msgid = 0;
while ((want_tuid || want_msgid) && fgets( nbuf, sizeof(nbuf), f )) {
int bufl = strlen( nbuf );
if (bufl && nbuf[bufl - 1] == '\n')
--bufl;
if (bufl && nbuf[bufl - 1] == '\r')
--bufl;
if (!bufl)
break;
if (want_tuid && starts_with( nbuf, bufl, "X-TUID: ", 8 )) {
if (bufl < 8 + TUIDL) {
error( "Maildir error: malformed X-TUID header (UID %d)\n", uid );
continue;
}
memcpy( entry->tuid, nbuf + 8, TUIDL );
want_tuid = 0;
in_msgid = 0;
continue;
}
if (want_msgid && starts_with_upper( nbuf, bufl, "MESSAGE-ID:", 11 )) {
off = 11;
} else if (in_msgid) {
if (!isspace( nbuf[0] )) {
in_msgid = 0;
continue;
}
off = 1;
} else {
continue;
}
while (off < bufl && isspace( nbuf[off] ))
off++;
if (off == bufl) {
in_msgid = 1;
continue;
}
entry->msgid = nfstrndup( nbuf + off, bufl - off );
want_msgid = 0;
in_msgid = 0;
}
fclose( f );
}
@ -1093,6 +1128,8 @@ maildir_init_msg( maildir_store_t *ctx, maildir_message_t *msg, msg_t *entry )
{
msg->base = entry->base;
entry->base = 0; /* prevent deletion */
msg->gen.msgid = entry->msgid;
entry->msgid = 0; /* prevent deletion */
msg->gen.size = entry->size;
msg->gen.srec = 0;
strncpy( msg->gen.tuid, entry->tuid, TUIDL );
@ -1350,6 +1387,7 @@ maildir_rescan( maildir_store_t *ctx )
debug( "updating message %d\n", msg->gen.uid );
msg->gen.status &= ~(M_FLAGS|M_RECENT);
free( msg->base );
free( msg->gen.msgid );
maildir_init_msg( ctx, msg, msglist.array.data + i );
i++, msgapp = &msg->gen.next;
}

4
src/mbsync.1
View File

@ -36,8 +36,8 @@ the operation set can be selected in a fine-grained manner.
.br
Synchronization is based on unique message identifiers (UIDs), so no
identification conflicts can occur (unlike with some other mail synchronizers).
OTOH, \fBmbsync\fR is susceptible to UID validity changes (that \fIshould\fR
never happen, but see "Compatibility" in the README).
OTOH, \fBmbsync\fR is susceptible to UID validity changes (but will recover
just fine if the change is unfounded).
Synchronization state is kept in one local text file per mailbox pair;
these files are protected against concurrent \fBmbsync\fR processes.
Mailboxes can be safely modified while \fBmbsync\fR operates

56
src/sync.c
View File

@ -1165,12 +1165,13 @@ box_opened2( sync_vars_t *svars, int t )
fails = 0;
for (t = 0; t < 2; t++)
if (svars->uidval[t] >= 0 && svars->uidval[t] != ctx[t]->uidvalidity) {
error( "Error: UIDVALIDITY of %s changed (got %d, expected %d)\n",
str_ms[t], ctx[t]->uidvalidity, svars->uidval[t] );
if (svars->uidval[t] >= 0 && svars->uidval[t] != ctx[t]->uidvalidity)
fails++;
}
if (fails) {
if (fails == 2) {
error( "Error: channel %s: UIDVALIDITY of both master and slave changed\n"
"(master got %d, expected %d; slave got %d, expected %d).\n",
svars->chan->name,
ctx[M]->uidvalidity, svars->uidval[M], ctx[S]->uidvalidity, svars->uidval[S] );
bail:
svars->ret = SYNC_FAIL;
sync_bail( svars );
@ -1193,6 +1194,8 @@ box_opened2( sync_vars_t *svars, int t )
Fprintf( svars->jfp, JOURNAL_VERSION "\n" );
opts[M] = opts[S] = 0;
if (fails)
opts[M] = opts[S] = OPEN_OLD|OPEN_OLD_IDS;
for (t = 0; t < 2; t++) {
if (chan->ops[t] & (OP_DELETE|OP_FLAGS)) {
opts[t] |= OPEN_SETFLAGS;
@ -1323,7 +1326,7 @@ load_box( sync_vars_t *svars, int t, int minwuid, int_array_t mexcs )
if (minwuid > svars->maxuid[t] + 1)
minwuid = svars->maxuid[t] + 1;
maxwuid = INT_MAX;
if (svars->ctx[t]->opts & OPEN_OLD_SIZE)
if (svars->ctx[t]->opts & (OPEN_OLD_IDS|OPEN_OLD_SIZE))
seenuid = get_seenuid( svars, t );
else
seenuid = 0;
@ -1429,6 +1432,47 @@ box_loaded( int sts, void *aux )
if (!(svars->state[1-t] & ST_LOADED))
return;
for (t = 0; t < 2; t++) {
if (svars->uidval[t] >= 0 && svars->uidval[t] != svars->ctx[t]->uidvalidity) {
unsigned need = 0, got = 0;
debug( "trying to re-approve uid validity of %s\n", str_ms[t] );
for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD)
continue;
if (!srec->msg[t])
continue; // Message disappeared.
need++; // Present paired messages require re-validation.
if (!srec->msg[t]->msgid)
continue; // Messages without ID are useless for re-validation.
if (!srec->msg[1-t])
continue; // Partner disappeared.
if (!srec->msg[1-t]->msgid || strcmp( srec->msg[M]->msgid, srec->msg[S]->msgid )) {
error( "Error: channel %s, %s %s: UIDVALIDITY genuinely changed (at UID %d).\n",
svars->chan->name, str_ms[t], svars->orig_name[t], srec->uid[t] );
uvchg:
svars->ret |= SYNC_FAIL;
cancel_sync( svars );
return;
}
got++;
}
if (got < 20 && got * 5 < need * 4) {
// Too few confirmed messages. This is very likely in the drafts folder.
// A proper fallback would be fetching more headers (which potentially need
// normalization) or the message body (which should be truncated for sanity)
// and comparing.
error( "Error: channel %s, %s %s: Unable to recover from UIDVALIDITY change\n"
"(got %d, expected %d).\n",
svars->chan->name, str_ms[t], svars->orig_name[t],
svars->ctx[t]->uidvalidity, svars->uidval[t] );
goto uvchg;
}
notice( "Notice: channel %s, %s %s: Recovered from change of UIDVALIDITY.\n",
svars->chan->name, str_ms[t], svars->orig_name[t] );
svars->uidval[t] = -1;
}
}
if (svars->uidval[M] < 0 || svars->uidval[S] < 0) {
svars->uidval[M] = svars->ctx[M]->uidvalidity;
svars->uidval[S] = svars->ctx[S]->uidvalidity;