implement Message-Id based UIDVALIDITY recovery
This commit is contained in:
parent
f9fe75602e
commit
77acc26812
2
NEWS
2
NEWS
|
@ -6,6 +6,8 @@ A Maildir sub-folder naming style without extra dots has been added.
|
|||
|
||||
Support for TLS client certificates was added.
|
||||
|
||||
Support for recovering from baseless UID validity changes was added.
|
||||
|
||||
[1.2.0]
|
||||
|
||||
The 'isync' compatibility wrapper is now deprecated.
|
||||
|
|
10
README
10
README
|
@ -45,16 +45,6 @@ isync executable still exists; it is a compatibility wrapper around mbsync.
|
|||
|
||||
Courier 1.4.3 is known to be buggy, version 1.7.3 works fine.
|
||||
|
||||
c-client (UW-IMAP, Pine) is mostly fine, but versions less than 2004a.352
|
||||
tend to change UIDVALIDITY pretty often when used with unix/mbox mailboxes,
|
||||
making isync refuse synchronization.
|
||||
M$ Exchange (up to 2010 at least) occasionally exposes the same problem.
|
||||
The "cure" is to simply copy the new UIDVALIDITY from the affected
|
||||
mailbox to mbsync's state file. This is a Bad Hack (TM), but it works -
|
||||
use at your own risk (if the UIDVALIDITY change was genuine, this will
|
||||
delete all messages in the affected mailbox - not that this ever
|
||||
happened to me).
|
||||
|
||||
M$ Exchange (2013 at least) needs DisableExtension MOVE to be compatible
|
||||
with the Trash functionality.
|
||||
|
||||
|
|
|
@ -34,6 +34,7 @@ free_generic_messages( message_t *msgs )
|
|||
|
||||
for (; msgs; msgs = tmsg) {
|
||||
tmsg = msgs->next;
|
||||
free( msgs->msgid );
|
||||
free( msgs );
|
||||
}
|
||||
}
|
||||
|
|
|
@ -63,6 +63,7 @@ typedef struct store_conf {
|
|||
typedef struct message {
|
||||
struct message *next;
|
||||
struct sync_rec *srec;
|
||||
char *msgid; /* owned */
|
||||
/* string_list_t *keywords; */
|
||||
size_t size; /* zero implies "not fetched" */
|
||||
int uid;
|
||||
|
@ -80,6 +81,7 @@ typedef struct message {
|
|||
#define OPEN_SETFLAGS (1<<6)
|
||||
#define OPEN_APPEND (1<<7)
|
||||
#define OPEN_FIND (1<<8)
|
||||
#define OPEN_OLD_IDS (1<<9)
|
||||
|
||||
typedef struct store {
|
||||
struct store *next;
|
||||
|
@ -208,6 +210,7 @@ struct driver {
|
|||
* and those named in the excs array (smaller than minuid).
|
||||
* The driver takes ownership of the excs array.
|
||||
* Messages starting with newuid need to have the TUID populated when OPEN_FIND is set.
|
||||
* Messages up to seenuid need to have the Message-Id populated when OPEN_OLD_IDS is set.
|
||||
* Messages up to seenuid need to have the size populated when OPEN_OLD_SIZE is set;
|
||||
* likewise messages above seenuid when OPEN_NEW_SIZE is set. */
|
||||
void (*load_box)( store_t *ctx, int minuid, int maxuid, int newuid, int seenuid, int_array_t excs,
|
||||
|
|
|
@ -905,7 +905,7 @@ static int
|
|||
parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
|
||||
{
|
||||
list_t *tmp, *flags;
|
||||
char *body = 0, *tuid = 0;
|
||||
char *body = 0, *tuid = 0, *msgid = 0;
|
||||
imap_message_t *cur;
|
||||
msg_data_t *msgdata;
|
||||
struct imap_cmd *cmdp;
|
||||
|
@ -983,8 +983,42 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
|
|||
tmp = tmp->next;
|
||||
if (!is_atom( tmp ))
|
||||
goto bfail;
|
||||
if (starts_with_upper( tmp->val, tmp->len, "X-TUID: ", 8 ))
|
||||
tuid = tmp->val + 8;
|
||||
int off, in_msgid = 0;
|
||||
for (char *val = tmp->val, *end; (end = strchr( val, '\n' )); val = end + 1) {
|
||||
int len = (int)(end - val);
|
||||
if (len && end[-1] == '\r')
|
||||
len--;
|
||||
if (!len)
|
||||
break;
|
||||
if (starts_with_upper( val, len, "X-TUID: ", 8 )) {
|
||||
if (len < 8 + TUIDL) {
|
||||
error( "IMAP error: malformed X-TUID header (UID %d)\n", uid );
|
||||
continue;
|
||||
}
|
||||
tuid = val + 8;
|
||||
in_msgid = 0;
|
||||
continue;
|
||||
}
|
||||
if (starts_with_upper( val, len, "MESSAGE-ID:", 11 )) {
|
||||
off = 11;
|
||||
} else if (in_msgid) {
|
||||
if (!isspace( val[0] )) {
|
||||
in_msgid = 0;
|
||||
continue;
|
||||
}
|
||||
off = 1;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
while (off < len && isspace( val[off] ))
|
||||
off++;
|
||||
if (off == len) {
|
||||
in_msgid = 1;
|
||||
continue;
|
||||
}
|
||||
msgid = nfstrndup( val + off, len - off );
|
||||
in_msgid = 0;
|
||||
}
|
||||
} else {
|
||||
bfail:
|
||||
error( "IMAP error: unable to parse BODY[HEADER.FIELDS ...]\n" );
|
||||
|
@ -1018,6 +1052,7 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
|
|||
cur->gen.status = status;
|
||||
cur->gen.size = size;
|
||||
cur->gen.srec = 0;
|
||||
cur->gen.msgid = msgid;
|
||||
if (tuid)
|
||||
strncpy( cur->gen.tuid, tuid, TUIDL );
|
||||
else
|
||||
|
@ -2322,7 +2357,7 @@ imap_prepare_load_box( store_t *gctx, int opts )
|
|||
gctx->opts = opts;
|
||||
}
|
||||
|
||||
enum { WantSize = 1, WantTuids = 2 };
|
||||
enum { WantSize = 1, WantTuids = 2, WantMsgids = 4 };
|
||||
typedef struct imap_range {
|
||||
int first, last, flags;
|
||||
} imap_range_t;
|
||||
|
@ -2375,7 +2410,7 @@ imap_load_box( store_t *gctx, int minuid, int maxuid, int newuid, int seenuid, i
|
|||
if (i != j)
|
||||
bl += sprintf( buf + bl, ":%d", excs.data[i] );
|
||||
}
|
||||
imap_submit_load( ctx, buf, 0, sts );
|
||||
imap_submit_load( ctx, buf, shifted_bit( ctx->gen.opts, OPEN_OLD_IDS, WantMsgids ), sts );
|
||||
}
|
||||
if (maxuid == INT_MAX)
|
||||
maxuid = ctx->gen.uidnext ? ctx->gen.uidnext - 1 : 0x7fffffff;
|
||||
|
@ -2390,6 +2425,8 @@ imap_load_box( store_t *gctx, int minuid, int maxuid, int newuid, int seenuid, i
|
|||
shifted_bit( ctx->gen.opts, OPEN_NEW_SIZE, WantSize), seenuid );
|
||||
if (ctx->gen.opts & OPEN_FIND)
|
||||
imap_set_range( ranges, &nranges, 0, WantTuids, newuid - 1 );
|
||||
if (ctx->gen.opts & OPEN_OLD_IDS)
|
||||
imap_set_range( ranges, &nranges, WantMsgids, 0, seenuid );
|
||||
for (int r = 0; r < nranges; r++) {
|
||||
sprintf( buf, "%d:%d", ranges[r].first, ranges[r].last );
|
||||
imap_submit_load( ctx, buf, ranges[r].flags, sts );
|
||||
|
@ -2404,10 +2441,14 @@ static void
|
|||
imap_submit_load( imap_store_t *ctx, const char *buf, int flags, struct imap_cmd_refcounted_state *sts )
|
||||
{
|
||||
imap_exec( ctx, imap_refcounted_new_cmd( sts ), imap_refcounted_done_box,
|
||||
"UID FETCH %s (UID%s%s%s)", buf,
|
||||
"UID FETCH %s (UID%s%s%s%s%s%s%s)", buf,
|
||||
(ctx->gen.opts & OPEN_FLAGS) ? " FLAGS" : "",
|
||||
(flags & WantSize) ? " RFC822.SIZE" : "",
|
||||
(flags & WantTuids) ? " BODY.PEEK[HEADER.FIELDS (X-TUID)]" : "");
|
||||
(flags & (WantTuids | WantMsgids)) ? " BODY.PEEK[HEADER.FIELDS (" : "",
|
||||
(flags & WantTuids) ? "X-TUID" : "",
|
||||
!(~flags & (WantTuids | WantMsgids)) ? " " : "",
|
||||
(flags & WantMsgids) ? "MESSAGE-ID" : "",
|
||||
(flags & (WantTuids | WantMsgids)) ? ")]" : "");
|
||||
}
|
||||
|
||||
/******************* imap_fetch_msg *******************/
|
||||
|
|
|
@ -442,6 +442,7 @@ static const char *subdirs[] = { "cur", "new", "tmp" };
|
|||
|
||||
typedef struct {
|
||||
char *base;
|
||||
char *msgid;
|
||||
int size;
|
||||
uint uid:31, recent:1;
|
||||
char tuid[TUIDL];
|
||||
|
@ -926,6 +927,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
|
|||
continue;
|
||||
entry = msg_t_array_append( msglist );
|
||||
entry->base = nfstrdup( e->d_name );
|
||||
entry->msgid = 0;
|
||||
entry->uid = uid;
|
||||
entry->recent = i;
|
||||
entry->size = 0;
|
||||
|
@ -1050,7 +1052,8 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
|
|||
}
|
||||
int want_size = (uid > ctx->seenuid) ? (ctx->gen.opts & OPEN_NEW_SIZE) : (ctx->gen.opts & OPEN_OLD_SIZE);
|
||||
int want_tuid = ((ctx->gen.opts & OPEN_FIND) && uid >= ctx->newuid);
|
||||
if (!want_size && !want_tuid)
|
||||
int want_msgid = ((ctx->gen.opts & OPEN_OLD_IDS) && uid <= ctx->seenuid);
|
||||
if (!want_size && !want_tuid && !want_msgid)
|
||||
continue;
|
||||
if (!fnl)
|
||||
nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base );
|
||||
|
@ -1064,7 +1067,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
|
|||
}
|
||||
entry->size = st.st_size;
|
||||
}
|
||||
if (want_tuid) {
|
||||
if (want_tuid || want_msgid) {
|
||||
if (!(f = fopen( buf, "r" ))) {
|
||||
if (errno != ENOENT) {
|
||||
sys_error( "Maildir error: cannot open %s", buf );
|
||||
|
@ -1072,13 +1075,45 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
|
|||
}
|
||||
goto retry;
|
||||
}
|
||||
while (fgets( nbuf, sizeof(nbuf), f )) {
|
||||
if (!nbuf[0] || nbuf[0] == '\n')
|
||||
int off, in_msgid = 0;
|
||||
while ((want_tuid || want_msgid) && fgets( nbuf, sizeof(nbuf), f )) {
|
||||
int bufl = strlen( nbuf );
|
||||
if (bufl && nbuf[bufl - 1] == '\n')
|
||||
--bufl;
|
||||
if (bufl && nbuf[bufl - 1] == '\r')
|
||||
--bufl;
|
||||
if (!bufl)
|
||||
break;
|
||||
if (starts_with( nbuf, -1, "X-TUID: ", 8 ) && nbuf[8 + TUIDL] == '\n') {
|
||||
if (want_tuid && starts_with( nbuf, bufl, "X-TUID: ", 8 )) {
|
||||
if (bufl < 8 + TUIDL) {
|
||||
error( "Maildir error: malformed X-TUID header (UID %d)\n", uid );
|
||||
continue;
|
||||
}
|
||||
memcpy( entry->tuid, nbuf + 8, TUIDL );
|
||||
break;
|
||||
want_tuid = 0;
|
||||
in_msgid = 0;
|
||||
continue;
|
||||
}
|
||||
if (want_msgid && starts_with_upper( nbuf, bufl, "MESSAGE-ID:", 11 )) {
|
||||
off = 11;
|
||||
} else if (in_msgid) {
|
||||
if (!isspace( nbuf[0] )) {
|
||||
in_msgid = 0;
|
||||
continue;
|
||||
}
|
||||
off = 1;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
while (off < bufl && isspace( nbuf[off] ))
|
||||
off++;
|
||||
if (off == bufl) {
|
||||
in_msgid = 1;
|
||||
continue;
|
||||
}
|
||||
entry->msgid = nfstrndup( nbuf + off, bufl - off );
|
||||
want_msgid = 0;
|
||||
in_msgid = 0;
|
||||
}
|
||||
fclose( f );
|
||||
}
|
||||
|
@ -1093,6 +1128,8 @@ maildir_init_msg( maildir_store_t *ctx, maildir_message_t *msg, msg_t *entry )
|
|||
{
|
||||
msg->base = entry->base;
|
||||
entry->base = 0; /* prevent deletion */
|
||||
msg->gen.msgid = entry->msgid;
|
||||
entry->msgid = 0; /* prevent deletion */
|
||||
msg->gen.size = entry->size;
|
||||
msg->gen.srec = 0;
|
||||
strncpy( msg->gen.tuid, entry->tuid, TUIDL );
|
||||
|
@ -1350,6 +1387,7 @@ maildir_rescan( maildir_store_t *ctx )
|
|||
debug( "updating message %d\n", msg->gen.uid );
|
||||
msg->gen.status &= ~(M_FLAGS|M_RECENT);
|
||||
free( msg->base );
|
||||
free( msg->gen.msgid );
|
||||
maildir_init_msg( ctx, msg, msglist.array.data + i );
|
||||
i++, msgapp = &msg->gen.next;
|
||||
}
|
||||
|
|
|
@ -36,8 +36,8 @@ the operation set can be selected in a fine-grained manner.
|
|||
.br
|
||||
Synchronization is based on unique message identifiers (UIDs), so no
|
||||
identification conflicts can occur (unlike with some other mail synchronizers).
|
||||
OTOH, \fBmbsync\fR is susceptible to UID validity changes (that \fIshould\fR
|
||||
never happen, but see "Compatibility" in the README).
|
||||
OTOH, \fBmbsync\fR is susceptible to UID validity changes (but will recover
|
||||
just fine if the change is unfounded).
|
||||
Synchronization state is kept in one local text file per mailbox pair;
|
||||
these files are protected against concurrent \fBmbsync\fR processes.
|
||||
Mailboxes can be safely modified while \fBmbsync\fR operates
|
||||
|
|
56
src/sync.c
56
src/sync.c
|
@ -1165,12 +1165,13 @@ box_opened2( sync_vars_t *svars, int t )
|
|||
|
||||
fails = 0;
|
||||
for (t = 0; t < 2; t++)
|
||||
if (svars->uidval[t] >= 0 && svars->uidval[t] != ctx[t]->uidvalidity) {
|
||||
error( "Error: UIDVALIDITY of %s changed (got %d, expected %d)\n",
|
||||
str_ms[t], ctx[t]->uidvalidity, svars->uidval[t] );
|
||||
if (svars->uidval[t] >= 0 && svars->uidval[t] != ctx[t]->uidvalidity)
|
||||
fails++;
|
||||
}
|
||||
if (fails) {
|
||||
if (fails == 2) {
|
||||
error( "Error: channel %s: UIDVALIDITY of both master and slave changed\n"
|
||||
"(master got %d, expected %d; slave got %d, expected %d).\n",
|
||||
svars->chan->name,
|
||||
ctx[M]->uidvalidity, svars->uidval[M], ctx[S]->uidvalidity, svars->uidval[S] );
|
||||
bail:
|
||||
svars->ret = SYNC_FAIL;
|
||||
sync_bail( svars );
|
||||
|
@ -1193,6 +1194,8 @@ box_opened2( sync_vars_t *svars, int t )
|
|||
Fprintf( svars->jfp, JOURNAL_VERSION "\n" );
|
||||
|
||||
opts[M] = opts[S] = 0;
|
||||
if (fails)
|
||||
opts[M] = opts[S] = OPEN_OLD|OPEN_OLD_IDS;
|
||||
for (t = 0; t < 2; t++) {
|
||||
if (chan->ops[t] & (OP_DELETE|OP_FLAGS)) {
|
||||
opts[t] |= OPEN_SETFLAGS;
|
||||
|
@ -1323,7 +1326,7 @@ load_box( sync_vars_t *svars, int t, int minwuid, int_array_t mexcs )
|
|||
if (minwuid > svars->maxuid[t] + 1)
|
||||
minwuid = svars->maxuid[t] + 1;
|
||||
maxwuid = INT_MAX;
|
||||
if (svars->ctx[t]->opts & OPEN_OLD_SIZE)
|
||||
if (svars->ctx[t]->opts & (OPEN_OLD_IDS|OPEN_OLD_SIZE))
|
||||
seenuid = get_seenuid( svars, t );
|
||||
else
|
||||
seenuid = 0;
|
||||
|
@ -1429,6 +1432,47 @@ box_loaded( int sts, void *aux )
|
|||
if (!(svars->state[1-t] & ST_LOADED))
|
||||
return;
|
||||
|
||||
for (t = 0; t < 2; t++) {
|
||||
if (svars->uidval[t] >= 0 && svars->uidval[t] != svars->ctx[t]->uidvalidity) {
|
||||
unsigned need = 0, got = 0;
|
||||
debug( "trying to re-approve uid validity of %s\n", str_ms[t] );
|
||||
for (srec = svars->srecs; srec; srec = srec->next) {
|
||||
if (srec->status & S_DEAD)
|
||||
continue;
|
||||
if (!srec->msg[t])
|
||||
continue; // Message disappeared.
|
||||
need++; // Present paired messages require re-validation.
|
||||
if (!srec->msg[t]->msgid)
|
||||
continue; // Messages without ID are useless for re-validation.
|
||||
if (!srec->msg[1-t])
|
||||
continue; // Partner disappeared.
|
||||
if (!srec->msg[1-t]->msgid || strcmp( srec->msg[M]->msgid, srec->msg[S]->msgid )) {
|
||||
error( "Error: channel %s, %s %s: UIDVALIDITY genuinely changed (at UID %d).\n",
|
||||
svars->chan->name, str_ms[t], svars->orig_name[t], srec->uid[t] );
|
||||
uvchg:
|
||||
svars->ret |= SYNC_FAIL;
|
||||
cancel_sync( svars );
|
||||
return;
|
||||
}
|
||||
got++;
|
||||
}
|
||||
if (got < 20 && got * 5 < need * 4) {
|
||||
// Too few confirmed messages. This is very likely in the drafts folder.
|
||||
// A proper fallback would be fetching more headers (which potentially need
|
||||
// normalization) or the message body (which should be truncated for sanity)
|
||||
// and comparing.
|
||||
error( "Error: channel %s, %s %s: Unable to recover from UIDVALIDITY change\n"
|
||||
"(got %d, expected %d).\n",
|
||||
svars->chan->name, str_ms[t], svars->orig_name[t],
|
||||
svars->ctx[t]->uidvalidity, svars->uidval[t] );
|
||||
goto uvchg;
|
||||
}
|
||||
notice( "Notice: channel %s, %s %s: Recovered from change of UIDVALIDITY.\n",
|
||||
svars->chan->name, str_ms[t], svars->orig_name[t] );
|
||||
svars->uidval[t] = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (svars->uidval[M] < 0 || svars->uidval[S] < 0) {
|
||||
svars->uidval[M] = svars->ctx[M]->uidvalidity;
|
||||
svars->uidval[S] = svars->ctx[S]->uidvalidity;
|
||||
|
|
Loading…
Reference in New Issue
Block a user