lobo/isync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

employ alternative scheme to finding messages by TUID

Browse Source 
instead of SEARCHing every single message (which is slow and happens to
be unreliabe with M$ Exchange 2010), just FETCH the new messages from
the mailbox - the ones we just appended will be amongst them.
This commit is contained in:
Oswald Buddenhagen 2011-04-10 13:06:07 +02:00
parent b4cef554fc
commit 9c86ec3442
5 changed files with 183 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
src/drv_imap.c
View File

@ -79,7 +79,6 @@ typedef struct imap_store {
store_t gen; store_t gen;
const char *prefix; const char *prefix;
int ref_count; int ref_count;
int uidnext; /* from SELECT responses */
/* trash folder's existence is not confirmed yet */ /* trash folder's existence is not confirmed yet */
enum { TrashUnknown, TrashChecking, TrashKnown } trashnc; enum { TrashUnknown, TrashChecking, TrashKnown } trashnc;
unsigned got_namespace:1; unsigned got_namespace:1;
@ -602,7 +601,7 @@ static int
parse_fetch( imap_store_t *ctx, list_t *list ) parse_fetch( imap_store_t *ctx, list_t *list )
{ {
list_t *tmp, *flags; list_t *tmp, *flags;
char *body = 0; char *body = 0, *tuid = 0;
imap_message_t *cur; imap_message_t *cur;
msg_data_t *msgdata; msg_data_t *msgdata;
struct imap_cmd *cmdp; struct imap_cmd *cmdp;
@ -663,6 +662,20 @@ parse_fetch( imap_store_t *ctx, list_t *list )
size = tmp->len; size = tmp->len;
} else } else
error( "IMAP error: unable to parse BODY[]\n" ); error( "IMAP error: unable to parse BODY[]\n" );
} else if (!strcmp( "BODY[HEADER.FIELDS", tmp->val )) {
tmp = tmp->next;
if (is_list( tmp )) {
tmp = tmp->next;
if (!is_atom( tmp ) || strcmp( tmp->val, "]" ))
goto bfail;
tmp = tmp->next;
if (!is_atom( tmp ) || memcmp( tmp->val, "X-TUID: ", 8 ))
goto bfail;
tuid = tmp->val + 8;
} else {
bfail:
error( "IMAP error: unable to parse BODY[HEADER.FIELDS ...]\n" );
}
} }
} }
} }
@ -690,6 +703,13 @@ parse_fetch( imap_store_t *ctx, list_t *list )
cur->gen.flags = mask; cur->gen.flags = mask;
cur->gen.status = status; cur->gen.status = status;
cur->gen.size = size; cur->gen.size = size;
cur->gen.srec = 0;
if (tuid)
strncpy( cur->gen.tuid, tuid, TUIDL );
else
cur->gen.tuid[0] = 0;
if (ctx->gen.uidnext <= uid) /* in case the server sends no UIDNEXT */
ctx->gen.uidnext = uid + 1;
} }
free_list( list ); free_list( list );
@ -731,7 +751,7 @@ parse_response_code( imap_store_t *ctx, struct imap_cmd *cmd, char *s )
return RESP_CANCEL; return RESP_CANCEL;
} }
} else if (!strcmp( "UIDNEXT", arg )) { } else if (!strcmp( "UIDNEXT", arg )) {
if (!(arg = next_arg( &s )) || (ctx->uidnext = strtol( arg, &p, 10 ), *p)) { if (!(arg = next_arg( &s )) || (ctx->gen.uidnext = strtol( arg, &p, 10 ), *p)) {
error( "IMAP error: malformed NEXTUID status\n" ); error( "IMAP error: malformed NEXTUID status\n" );
return RESP_CANCEL; return RESP_CANCEL;
} }
@ -756,35 +776,6 @@ parse_response_code( imap_store_t *ctx, struct imap_cmd *cmd, char *s )
return RESP_OK; return RESP_OK;
} }
static void
parse_search( imap_store_t *ctx, char *cmd )
{
char *arg;
struct imap_cmd *cmdp;
int uid;
if (!(arg = next_arg( &cmd )))
uid = -1;
else if (!(uid = atoi( arg ))) {
error( "IMAP error: malformed SEARCH response\n" );
return;
} else if (next_arg( &cmd )) {
warn( "IMAP warning: SEARCH returns multiple matches\n" );
uid = -1; /* to avoid havoc */
}
/* Find the first command that expects a UID - this is guaranteed
* to come in-order, as there are no other means to identify which
* SEARCH response belongs to which request.
*/
for (cmdp = ctx->in_progress; cmdp; cmdp = cmdp->next)
if (cmdp->param.uid == -1) {
((struct imap_cmd_out_uid *)cmdp)->out_uid = uid;
return;
}
error( "IMAP error: unexpected SEARCH response (UID %u)\n", uid );
}
static void static void
parse_list_rsp( imap_store_t *ctx, char *cmd ) parse_list_rsp( imap_store_t *ctx, char *cmd )
{ {
@ -861,8 +852,6 @@ imap_socket_read( void *aux )
parse_capability( ctx, cmd ); parse_capability( ctx, cmd );
else if (!strcmp( "LIST", arg )) else if (!strcmp( "LIST", arg ))
parse_list_rsp( ctx, cmd ); parse_list_rsp( ctx, cmd );
else if (!strcmp( "SEARCH", arg ))
parse_search( ctx, cmd );
else if ((arg1 = next_arg( &cmd ))) { else if ((arg1 = next_arg( &cmd ))) {
if (!strcmp( "EXISTS", arg1 )) if (!strcmp( "EXISTS", arg1 ))
ctx->gen.count = atoi( arg ); ctx->gen.count = atoi( arg );
@ -980,7 +969,7 @@ get_cmd_result_p2( imap_store_t *ctx, struct imap_cmd *cmd, int response )
if (response != RESP_OK) { if (response != RESP_OK) {
done_imap_cmd( ctx, ocmd, response ); done_imap_cmd( ctx, ocmd, response );
} else { } else {
ctx->uidnext = 0; ctx->gen.uidnext = 0;
if (ocmd->param.to_trash) if (ocmd->param.to_trash)
ctx->trashnc = TrashKnown; ctx->trashnc = TrashKnown;
ocmd->param.create = 0; ocmd->param.create = 0;
@ -1447,7 +1436,7 @@ imap_select( store_t *gctx, int create,
prefix = ctx->prefix; prefix = ctx->prefix;
} }
ctx->uidnext = -1; ctx->gen.uidnext = -1;
INIT_IMAP_CMD(imap_cmd_simple, cmd, cb, aux) INIT_IMAP_CMD(imap_cmd_simple, cmd, cb, aux)
cmd->gen.param.create = create; cmd->gen.param.create = create;
@ -1458,11 +1447,11 @@ imap_select( store_t *gctx, int create,
/******************* imap_load *******************/ /******************* imap_load *******************/
static int imap_submit_load( imap_store_t *, const char *, struct imap_cmd_refcounted_state * ); static int imap_submit_load( imap_store_t *, const char *, int, struct imap_cmd_refcounted_state * );
static void imap_load_p2( imap_store_t *, struct imap_cmd *, int ); static void imap_load_p2( imap_store_t *, struct imap_cmd *, int );
static void static void
imap_load( store_t *gctx, int minuid, int maxuid, int *excs, int nexcs, imap_load( store_t *gctx, int minuid, int maxuid, int newuid, int *excs, int nexcs,
void (*cb)( int sts, void *aux ), void *aux ) void (*cb)( int sts, void *aux ), void *aux )
{ {
imap_store_t *ctx = (imap_store_t *)gctx; imap_store_t *ctx = (imap_store_t *)gctx;
@ -1487,14 +1476,21 @@ imap_load( store_t *gctx, int minuid, int maxuid, int *excs, int nexcs,
if (i != j) if (i != j)
bl += sprintf( buf + bl, ":%d", excs[i] ); bl += sprintf( buf + bl, ":%d", excs[i] );
} }
if (imap_submit_load( ctx, buf, sts ) < 0) if (imap_submit_load( ctx, buf, 0, sts ) < 0)
goto done; goto done;
} }
if (maxuid == INT_MAX) if (maxuid == INT_MAX)
maxuid = ctx->uidnext >= 0 ? ctx->uidnext - 1 : 1000000000; maxuid = ctx->gen.uidnext >= 0 ? ctx->gen.uidnext - 1 : 1000000000;
if (maxuid >= minuid) { if (maxuid >= minuid) {
sprintf( buf, "%d:%d", minuid, maxuid ); if ((ctx->gen.opts & OPEN_FIND) && minuid < newuid) {
imap_submit_load( ctx, buf, sts ); sprintf( buf, "%d:%d", minuid, newuid - 1 );
if (imap_submit_load( ctx, buf, 0, sts ) < 0)
goto done;
sprintf( buf, "%d:%d", newuid, maxuid );
} else {
sprintf( buf, "%d:%d", minuid, maxuid );
}
imap_submit_load( ctx, buf, (ctx->gen.opts & OPEN_FIND), sts );
} }
done: done:
free( excs ); free( excs );
@ -1503,12 +1499,13 @@ imap_load( store_t *gctx, int minuid, int maxuid, int *excs, int nexcs,
} }
static int static int
imap_submit_load( imap_store_t *ctx, const char *buf, struct imap_cmd_refcounted_state *sts ) imap_submit_load( imap_store_t *ctx, const char *buf, int tuids, struct imap_cmd_refcounted_state *sts )
{ {
return imap_exec( ctx, imap_refcounted_new_cmd( sts ), imap_load_p2, return imap_exec( ctx, imap_refcounted_new_cmd( sts ), imap_load_p2,
"UID FETCH %s (UID%s%s)", buf, "UID FETCH %s (UID%s%s%s)", buf,
(ctx->gen.opts & OPEN_FLAGS) ? " FLAGS" : "", (ctx->gen.opts & OPEN_FLAGS) ? " FLAGS" : "",
(ctx->gen.opts & OPEN_SIZE) ? " RFC822.SIZE" : "" ); (ctx->gen.opts & OPEN_SIZE) ? " RFC822.SIZE" : "",
tuids ? " BODY.PEEK[HEADER.FIELDS (X-TUID)]" : "");
} }
static void static void
@ -1693,35 +1690,18 @@ imap_store_msg_p2( imap_store_t *ctx ATTR_UNUSED, struct imap_cmd *cmd, int resp
cmdp->callback( response, cmdp->out_uid, cmdp->callback_aux ); cmdp->callback( response, cmdp->out_uid, cmdp->callback_aux );
} }
/******************* imap_find_msg *******************/ /******************* imap_find_new_msgs *******************/
static void imap_find_msg_p2( imap_store_t *, struct imap_cmd *, int );
static void static void
imap_find_msg( store_t *gctx, const char *tuid, imap_find_new_msgs( store_t *gctx,
void (*cb)( int sts, int uid, void *aux ), void *aux ) void (*cb)( int sts, void *aux ), void *aux )
{ {
imap_store_t *ctx = (imap_store_t *)gctx; imap_store_t *ctx = (imap_store_t *)gctx;
struct imap_cmd_out_uid *cmd; struct imap_cmd_simple *cmd;
INIT_IMAP_CMD(imap_cmd_out_uid, cmd, cb, aux) INIT_IMAP_CMD(imap_cmd_simple, cmd, cb, aux)
cmd->gen.param.uid = -1; /* we're looking for a UID */ imap_exec( (imap_store_t *)ctx, &cmd->gen, imap_done_simple_box,
cmd->out_uid = -1; /* in case we get no SEARCH response at all */ "UID FETCH %d:1000000000 (UID BODY.PEEK[HEADER.FIELDS (X-TUID)])", ctx->gen.uidnext );
imap_exec( ctx, &cmd->gen, imap_find_msg_p2,
"UID SEARCH HEADER X-TUID %." stringify(TUIDL) "s", tuid );
}
static void
imap_find_msg_p2( imap_store_t *ctx ATTR_UNUSED, struct imap_cmd *cmd, int response )
{
struct imap_cmd_out_uid *cmdp = (struct imap_cmd_out_uid *)cmd;
transform_msg_response( &response );
if (response != DRV_OK)
cmdp->callback( response, -1, cmdp->callback_aux );
else
cmdp->callback( cmdp->out_uid <= 0 ? DRV_MSG_BAD : DRV_OK,
cmdp->out_uid, cmdp->callback_aux );
} }
/******************* imap_list *******************/ /******************* imap_list *******************/
@ -1917,7 +1897,7 @@ struct driver imap_driver = {
imap_load, imap_load,
imap_fetch_msg, imap_fetch_msg,
imap_store_msg, imap_store_msg,
imap_find_msg, imap_find_new_msgs,
imap_set_flags, imap_set_flags,
imap_trash_msg, imap_trash_msg,
imap_close, imap_close,

32
src/drv_maildir.c
View File

@ -24,6 +24,7 @@
#include "isync.h" #include "isync.h"
#include <assert.h>
#include <limits.h> #include <limits.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@ -57,13 +58,12 @@ typedef struct maildir_store_conf {
typedef struct maildir_message { typedef struct maildir_message {
message_t gen; message_t gen;
char *base; char *base;
char tuid[TUIDL];
} maildir_message_t; } maildir_message_t;
typedef struct maildir_store { typedef struct maildir_store {
store_t gen; store_t gen;
int uvfd, uvok, nuid; int uvfd, uvok, nuid;
int minuid, maxuid, nexcs, *excs; int minuid, maxuid, newuid, nexcs, *excs;
#ifdef USE_DB #ifdef USE_DB
DB *db; DB *db;
#endif /* USE_DB */ #endif /* USE_DB */
@ -642,7 +642,7 @@ maildir_scan( maildir_store_t *ctx, msglist_t *msglist )
goto again; goto again;
} }
uid = entry->uid; uid = entry->uid;
if (ctx->gen.opts & (OPEN_SIZE|OPEN_FIND)) if ((ctx->gen.opts & OPEN_SIZE) || ((ctx->gen.opts & OPEN_FIND) && uid >= ctx->newuid))
nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base ); nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base );
#ifdef USE_DB #ifdef USE_DB
} else if (ctx->db) { } else if (ctx->db) {
@ -651,7 +651,7 @@ maildir_scan( maildir_store_t *ctx, msglist_t *msglist )
return ret; return ret;
} }
entry->uid = uid; entry->uid = uid;
if (ctx->gen.opts & (OPEN_SIZE|OPEN_FIND)) if ((ctx->gen.opts & OPEN_SIZE) || ((ctx->gen.opts & OPEN_FIND) && uid >= ctx->newuid))
nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base ); nfsnprintf( buf + bl, sizeof(buf) - bl, "%s/%s", subdirs[entry->recent], entry->base );
#endif /* USE_DB */ #endif /* USE_DB */
} else { } else {
@ -696,7 +696,7 @@ maildir_scan( maildir_store_t *ctx, msglist_t *msglist )
} }
entry->size = st.st_size; entry->size = st.st_size;
} }
if (ctx->gen.opts & OPEN_FIND) { if ((ctx->gen.opts & OPEN_FIND) && uid >= ctx->newuid) {
if (!(f = fopen( buf, "r" ))) { if (!(f = fopen( buf, "r" ))) {
if (errno != ENOENT) { if (errno != ENOENT) {
sys_error( "Maildir error: cannot open %s", buf ); sys_error( "Maildir error: cannot open %s", buf );
@ -730,7 +730,8 @@ maildir_init_msg( maildir_store_t *ctx, maildir_message_t *msg, msg_t *entry )
msg->base = entry->base; msg->base = entry->base;
entry->base = 0; /* prevent deletion */ entry->base = 0; /* prevent deletion */
msg->gen.size = entry->size; msg->gen.size = entry->size;
strncpy( msg->tuid, entry->tuid, TUIDL ); msg->gen.srec = 0;
strncpy( msg->gen.tuid, entry->tuid, TUIDL );
if (entry->recent) if (entry->recent)
msg->gen.status |= M_RECENT; msg->gen.status |= M_RECENT;
if (ctx->gen.opts & OPEN_FLAGS) { if (ctx->gen.opts & OPEN_FLAGS) {
@ -861,7 +862,7 @@ maildir_prepare_opts( store_t *gctx, int opts )
} }
static void static void
maildir_load( store_t *gctx, int minuid, int maxuid, int *excs, int nexcs, maildir_load( store_t *gctx, int minuid, int maxuid, int newuid, int *excs, int nexcs,
void (*cb)( int sts, void *aux ), void *aux ) void (*cb)( int sts, void *aux ), void *aux )
{ {
maildir_store_t *ctx = (maildir_store_t *)gctx; maildir_store_t *ctx = (maildir_store_t *)gctx;
@ -871,6 +872,7 @@ maildir_load( store_t *gctx, int minuid, int maxuid, int *excs, int nexcs,
ctx->minuid = minuid; ctx->minuid = minuid;
ctx->maxuid = maxuid; ctx->maxuid = maxuid;
ctx->newuid = newuid;
ctx->excs = nfrealloc( excs, nexcs * sizeof(int) ); ctx->excs = nfrealloc( excs, nexcs * sizeof(int) );
ctx->nexcs = nexcs; ctx->nexcs = nexcs;
@ -1086,18 +1088,10 @@ maildir_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
} }
static void static void
maildir_find_msg( store_t *gctx, const char *tuid, maildir_find_new_msgs( store_t *gctx ATTR_UNUSED,
void (*cb)( int sts, int uid, void *aux ), void *aux ) void (*cb)( int sts, void *aux ) ATTR_UNUSED, void *aux ATTR_UNUSED )
{ {
message_t *msg; assert( !"maildir_find_new_msgs is not supposed to be called" );
/* using a hash table might turn out to be more appropriate ... */
for (msg = gctx->msgs; msg; msg = msg->next)
if (!(msg->status & M_DEAD) && !memcmp( ((maildir_message_t *)msg)->tuid, tuid, TUIDL )) {
cb( DRV_OK, msg->uid, aux );
return;
}
cb( DRV_MSG_BAD, -1, aux );
} }
static void static void
@ -1324,7 +1318,7 @@ struct driver maildir_driver = {
maildir_load, maildir_load,
maildir_fetch_msg, maildir_fetch_msg,
maildir_store_msg, maildir_store_msg,
maildir_find_msg, maildir_find_new_msgs,
maildir_set_flags, maildir_set_flags,
maildir_trash_msg, maildir_trash_msg,
maildir_close, maildir_close,

19
src/isync.h
View File

@ -184,6 +184,8 @@ typedef struct group_conf {
#define M_DEAD (1<<1) /* expunged */ #define M_DEAD (1<<1) /* expunged */
#define M_FLAGS (1<<2) /* flags fetched */ #define M_FLAGS (1<<2) /* flags fetched */
#define TUIDL 12
typedef struct message { typedef struct message {
struct message *next; struct message *next;
struct sync_rec *srec; struct sync_rec *srec;
@ -191,6 +193,7 @@ typedef struct message {
size_t size; /* zero implies "not fetched" */ size_t size; /* zero implies "not fetched" */
int uid; int uid;
unsigned char flags, status; unsigned char flags, status;
char tuid[TUIDL];
} message_t; } message_t;
/* For opts, both in store and driver_t->select() */ /* For opts, both in store and driver_t->select() */
@ -217,6 +220,7 @@ typedef struct store {
char *path; /* own */ char *path; /* own */
message_t *msgs; /* own */ message_t *msgs; /* own */
int uidvalidity; int uidvalidity;
int uidnext; /* from SELECT responses */
unsigned opts; /* maybe preset? */ unsigned opts; /* maybe preset? */
/* note that the following do _not_ reflect stats from msgs, but mailbox totals */ /* note that the following do _not_ reflect stats from msgs, but mailbox totals */
int count; /* # of messages */ int count; /* # of messages */
@ -255,8 +259,6 @@ typedef struct {
*/ */
#define DRV_CRLF 1 #define DRV_CRLF 1
#define TUIDL 12
struct driver { struct driver {
int flags; int flags;
@ -298,8 +300,9 @@ struct driver {
/* Load the message attributes needed to perform the requested operations. /* Load the message attributes needed to perform the requested operations.
* Consider only messages with UIDs between minuid and maxuid (inclusive) * Consider only messages with UIDs between minuid and maxuid (inclusive)
* and those named in the excs array (smaller than minuid). * and those named in the excs array (smaller than minuid).
* The driver takes ownership of the excs array. */ * The driver takes ownership of the excs array. Messages below newuid do not need
void (*load)( store_t *ctx, int minuid, int maxuid, int *excs, int nexcs, * to have the TUID populated even if OPEN_FIND is set. */
void (*load)( store_t *ctx, int minuid, int maxuid, int newuid, int *excs, int nexcs,
void (*cb)( int sts, void *aux ), void *aux ); void (*cb)( int sts, void *aux ), void *aux );
/* Fetch the contents and flags of the given message from the current mailbox. */ /* Fetch the contents and flags of the given message from the current mailbox. */
@ -311,9 +314,11 @@ struct driver {
void (*store_msg)( store_t *ctx, msg_data_t *data, int to_trash, void (*store_msg)( store_t *ctx, msg_data_t *data, int to_trash,
void (*cb)( int sts, int uid, void *aux ), void *aux ); void (*cb)( int sts, int uid, void *aux ), void *aux );
/* Find a message by its temporary UID header to determine its real UID. */ /* Index the messages which have newly appeared in the mailbox, including their
void (*find_msg)( store_t *ctx, const char *tuid, * temporary UID headers. This is needed if store_msg() does not guarantee returning
void (*cb)( int sts, int uid, void *aux ), void *aux ); * a UID; otherwise the driver needs to implement only the OPEN_FIND flag. */
void (*find_new_msgs)( store_t *ctx,
void (*cb)( int sts, void *aux ), void *aux );
/* Add/remove the named flags to/from the given message. The message may be either /* Add/remove the named flags to/from the given message. The message may be either
* a pre-fetched one (in which case the in-memory representation is updated), * a pre-fetched one (in which case the in-memory representation is updated),

4
src/run-tests.pl
View File

@ -366,7 +366,7 @@ sub showstate($)
close FILE; close FILE;
return; return;
} }
if (!/^1:(\d+) 1:(\d+):(\d+)\n$/) { if (!/^1:(\d+):0 1:(\d+):(\d+):0\n$/) {
print STDERR " Malformed sync state header '$_'.\n"; print STDERR " Malformed sync state header '$_'.\n";
close FILE; close FILE;
return; return;
@ -507,7 +507,7 @@ sub ckstate($@)
return 1; return 1;
} }
chomp($l); chomp($l);
my $xl = "1:".shift(@T)." 1:".shift(@T).":".shift(@T); my $xl = "1:".shift(@T).":0 1:".shift(@T).":".shift(@T).":0";
if ($l ne $xl) { if ($l ne $xl) {
print STDERR "Sync state header mismatch: '$l' instead of '$xl'.\n"; print STDERR "Sync state header mismatch: '$l' instead of '$xl'.\n";
return 1; return 1;

229
src/sync.c
View File

@ -96,6 +96,7 @@ make_flags( int flags, char *buf )
#define S_EXPIRE (1<<5) #define S_EXPIRE (1<<5)
#define S_NEXPIRE (1<<6) #define S_NEXPIRE (1<<6)
#define S_EXP_S (1<<7) #define S_EXP_S (1<<7)
#define S_FIND (1<<8)
#define mvBit(in,ib,ob) ((unsigned char)(((unsigned)in) * (ob) / (ib))) #define mvBit(in,ib,ob) ((unsigned char)(((unsigned)in) * (ob) / (ib)))
@ -146,15 +147,13 @@ typedef struct {
store_t *ctx[2]; store_t *ctx[2];
driver_t *drv[2]; driver_t *drv[2];
int state[2], ref_count, ret, lfd; int state[2], ref_count, ret, lfd;
int find_old_total[2], find_old_done[2];
int new_total[2], new_done[2]; int new_total[2], new_done[2];
int find_new_total[2], find_new_done[2];
int flags_total[2], flags_done[2]; int flags_total[2], flags_done[2];
int trash_total[2], trash_done[2]; int trash_total[2], trash_done[2];
int maxuid[2]; /* highest UID that was already propagated */ int maxuid[2]; /* highest UID that was already propagated */
int uidval[2]; /* UID validity value */ int uidval[2]; /* UID validity value */
int uidnext[2]; /* next expected UID; TUID lookup makes sense only for lower UIDs */
int smaxxuid; /* highest expired UID on slave */ int smaxxuid; /* highest expired UID on slave */
unsigned find:1;
} sync_vars_t; } sync_vars_t;
static void sync_ref( sync_vars_t *svars ) { ++svars->ref_count; } static void sync_ref( sync_vars_t *svars ) { ++svars->ref_count; }
@ -190,10 +189,8 @@ static int check_cancel( sync_vars_t *svars );
/* operation dependencies: /* operation dependencies:
select(S): - select(S): -
find_old(S): select(S) select(M): select(S) | -
select(M): find_old(S) | - new(M), new(S), flags(M): select(M) & select(S)
find_old(M): select(M)
new(M), new(S), flags(M): find_old(M) & find_old(S)
flags(S): count(new(S)) flags(S): count(new(S))
find_new(x): new(x) find_new(x): new(x)
trash(x): flags(x) trash(x): flags(x)
@ -201,9 +198,9 @@ static int check_cancel( sync_vars_t *svars );
cleanup: close(M) & close(S) cleanup: close(M) & close(S)
*/ */
#define ST_SENT_FIND_OLD (1<<0) #define ST_LOADED (1<<0)
#define ST_SENT_NEW (1<<1) #define ST_SENT_NEW (1<<1)
#define ST_SENT_FIND_NEW (1<<2) #define ST_FOUND_NEW (1<<2)
#define ST_SENT_FLAGS (1<<3) #define ST_SENT_FLAGS (1<<3)
#define ST_SENT_TRASH (1<<4) #define ST_SENT_TRASH (1<<4)
#define ST_CLOSED (1<<5) #define ST_CLOSED (1<<5)
@ -214,6 +211,51 @@ static int check_cancel( sync_vars_t *svars );
#define ST_DID_EXPUNGE (1<<16) #define ST_DID_EXPUNGE (1<<16)
static void
match_tuids( sync_vars_t *svars, int t )
{
sync_rec_t *srec;
message_t *tmsg, *ntmsg = 0;
const char *diag;
for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD)
continue;
if (srec->uid[t] == -2 && srec->tuid[0]) {
debug( " pair(%d,%d): lookup %s, TUID %." stringify(TUIDL) "s\n", srec->uid[M], srec->uid[S], str_ms[t], srec->tuid );
for (tmsg = ntmsg; tmsg; tmsg = tmsg->next) {
if (tmsg->status & M_DEAD)
continue;
if (tmsg->tuid[0] && !memcmp( tmsg->tuid, srec->tuid, TUIDL )) {
diag = (tmsg == ntmsg) ? "adjacently" : "after gap";
goto mfound;
}
}
for (tmsg = svars->ctx[t]->msgs; tmsg != ntmsg; tmsg = tmsg->next) {
if (tmsg->status & M_DEAD)
continue;
if (tmsg->tuid[0] && !memcmp( tmsg->tuid, srec->tuid, TUIDL )) {
diag = "after reset";
goto mfound;
}
}
debug( " -> TUID lost\n" );
Fprintf( svars->jfp, "& %d %d\n", srec->uid[M], srec->uid[S] );
srec->flags = 0;
srec->tuid[0] = 0;
continue;
mfound:
debug( " -> new UID %d %s\n", tmsg->uid, diag );
Fprintf( svars->jfp, "%c %d %d %d\n", "<>"[t], srec->uid[M], srec->uid[S], tmsg->uid );
tmsg->srec = srec;
ntmsg = tmsg->next;
srec->uid[t] = tmsg->uid;
srec->tuid[0] = 0;
}
}
}
typedef struct copy_vars { typedef struct copy_vars {
void (*cb)( int sts, int uid, struct copy_vars *vars ); void (*cb)( int sts, int uid, struct copy_vars *vars );
void *aux; void *aux;
@ -410,9 +452,7 @@ stats( sync_vars_t *svars )
cols = 36; cols = 36;
if (!(DFlags & QUIET)) { if (!(DFlags & QUIET)) {
for (t = 0; t < 2; t++) { for (t = 0; t < 2; t++) {
l = sprintf( buf[t], "?%d/%d +%d/%d *%d/%d #%d/%d", l = sprintf( buf[t], "+%d/%d *%d/%d #%d/%d",
svars->find_old_done[t] + svars->find_new_done[t],
svars->find_old_total[t] + svars->find_new_total[t],
svars->new_done[t], svars->new_total[t], svars->new_done[t], svars->new_total[t],
svars->flags_done[t], svars->flags_total[t], svars->flags_done[t], svars->flags_total[t],
svars->trash_done[t], svars->trash_total[t] ); svars->trash_done[t], svars->trash_total[t] );
@ -591,7 +631,7 @@ box_selected( int sts, void *aux )
struct stat st; struct stat st;
struct flock lck; struct flock lck;
char fbuf[16]; /* enlarge when support for keywords is added */ char fbuf[16]; /* enlarge when support for keywords is added */
char buf[64]; char buf[128], buf1[64], buf2[64];
if (check_ret( sts, aux )) if (check_ret( sts, aux ))
return; return;
@ -668,7 +708,9 @@ box_selected( int sts, void *aux )
sync_bail( svars ); sync_bail( svars );
return; return;
} }
if (sscanf( buf, "%d:%d %d:%d:%d", &svars->uidval[M], &svars->maxuid[M], &svars->uidval[S], &svars->smaxxuid, &svars->maxuid[S]) != 5) { if (sscanf( buf, "%63s %63s", buf1, buf2 ) != 2 ||
sscanf( buf1, "%d:%d:%d", &svars->uidval[M], &svars->maxuid[M], &svars->uidnext[M] ) < 2 ||
sscanf( buf2, "%d:%d:%d:%d", &svars->uidval[S], &svars->smaxxuid, &svars->maxuid[S], &svars->uidnext[S] ) < 3) {
error( "Error: invalid sync state header in %s\n", svars->dname ); error( "Error: invalid sync state header in %s\n", svars->dname );
goto jbail; goto jbail;
} }
@ -731,7 +773,7 @@ box_selected( int sts, void *aux )
} }
if (buf[0] == '#' ? if (buf[0] == '#' ?
(t3 = 0, (sscanf( buf + 2, "%d %d %n", &t1, &t2, &t3 ) < 2) || !t3 || (t - t3 != TUIDL + 3)) : (t3 = 0, (sscanf( buf + 2, "%d %d %n", &t1, &t2, &t3 ) < 2) || !t3 || (t - t3 != TUIDL + 3)) :
buf[0] == '(' || buf[0] == ')' ? buf[0] == '(' || buf[0] == ')' || buf[0] == '{' || buf[0] == '}' ?
(sscanf( buf + 2, "%d", &t1 ) != 1) : (sscanf( buf + 2, "%d", &t1 ) != 1) :
buf[0] == '+' || buf[0] == '&' || buf[0] == '-' || buf[0] == '|' || buf[0] == '/' || buf[0] == '\\' ? buf[0] == '+' || buf[0] == '&' || buf[0] == '-' || buf[0] == '|' || buf[0] == '/' || buf[0] == '\\' ?
(sscanf( buf + 2, "%d %d", &t1, &t2 ) != 2) : (sscanf( buf + 2, "%d %d", &t1, &t2 ) != 2) :
@ -744,6 +786,10 @@ box_selected( int sts, void *aux )
svars->maxuid[M] = t1; svars->maxuid[M] = t1;
else if (buf[0] == ')') else if (buf[0] == ')')
svars->maxuid[S] = t1; svars->maxuid[S] = t1;
else if (buf[0] == '{')
svars->uidnext[M] = t1;
else if (buf[0] == '}')
svars->uidnext[S] = t1;
else if (buf[0] == '|') { else if (buf[0] == '|') {
svars->uidval[M] = t1; svars->uidval[M] = t1;
svars->uidval[S] = t2; svars->uidval[S] = t2;
@ -900,15 +946,14 @@ box_selected( int sts, void *aux )
opts[S] |= OPEN_OLD|OPEN_FLAGS; opts[S] |= OPEN_OLD|OPEN_FLAGS;
if (srec->tuid[0]) { if (srec->tuid[0]) {
if (srec->uid[M] == -2) if (srec->uid[M] == -2)
opts[M] |= OPEN_OLD|OPEN_FIND; opts[M] |= OPEN_NEW|OPEN_FIND, svars->state[M] |= S_FIND;
else if (srec->uid[S] == -2) else if (srec->uid[S] == -2)
opts[S] |= OPEN_OLD|OPEN_FIND; opts[S] |= OPEN_NEW|OPEN_FIND, svars->state[S] |= S_FIND;
} }
} }
svars->drv[M]->prepare_opts( ctx[M], opts[M] ); svars->drv[M]->prepare_opts( ctx[M], opts[M] );
svars->drv[S]->prepare_opts( ctx[S], opts[S] ); svars->drv[S]->prepare_opts( ctx[S], opts[S] );
svars->find = line != 0;
if (!svars->smaxxuid && load_box( svars, M, (ctx[M]->opts & OPEN_OLD) ? 1 : INT_MAX, 0, 0 )) if (!svars->smaxxuid && load_box( svars, M, (ctx[M]->opts & OPEN_OLD) ? 1 : INT_MAX, 0, 0 ))
return; return;
load_box( svars, S, (ctx[S]->opts & OPEN_OLD) ? 1 : INT_MAX, 0, 0 ); load_box( svars, S, (ctx[S]->opts & OPEN_OLD) ? 1 : INT_MAX, 0, 0 );
@ -935,75 +980,7 @@ load_box( sync_vars_t *svars, int t, int minwuid, int *mexcs, int nmexcs )
maxwuid = 0; maxwuid = 0;
info( "Loading %s...\n", str_ms[t] ); info( "Loading %s...\n", str_ms[t] );
debug( maxwuid == INT_MAX ? "loading %s [%d,inf]\n" : "loading %s [%d,%d]\n", str_ms[t], minwuid, maxwuid ); debug( maxwuid == INT_MAX ? "loading %s [%d,inf]\n" : "loading %s [%d,%d]\n", str_ms[t], minwuid, maxwuid );
DRIVER_CALL_RET(load( svars->ctx[t], minwuid, maxwuid, mexcs, nmexcs, box_loaded, AUX )); DRIVER_CALL_RET(load( svars->ctx[t], minwuid, maxwuid, svars->uidnext[t], mexcs, nmexcs, box_loaded, AUX ));
}
typedef struct {
void *aux;
sync_rec_t *srec;
} find_vars_t;
static void msg_found_sel( int sts, int uid, void *aux );
static void msgs_found_sel( sync_vars_t *svars, int t );
static void
box_loaded( int sts, void *aux )
{
find_vars_t *fv;
sync_rec_t *srec;
SVARS_CHECK_RET;
info( "%s: %d messages, %d recent\n", str_ms[t], svars->ctx[t]->count, svars->ctx[t]->recent );
if (svars->find) {
/*
* Alternatively, the TUIDs could be fetched into the messages and
* looked up here. This would make the search faster (probably) and
* save roundtrips. On the downside, quite some additional data would
* have to be fetched for every message and the IMAP driver would be
* more complicated. This is a corner case anyway, so why bother.
*/
debug( "finding previously copied messages\n" );
for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD)
continue;
if (srec->uid[t] == -2 && srec->tuid[0]) {
debug( " pair(%d,%d): lookup %s, TUID %." stringify(TUIDL) "s\n", srec->uid[M], srec->uid[S], str_ms[t], srec->tuid );
svars->find_old_total[t]++;
stats( svars );
fv = nfmalloc( sizeof(*fv) );
fv->aux = AUX;
fv->srec = srec;
DRIVER_CALL(find_msg( svars->ctx[t], srec->tuid, msg_found_sel, fv ));
}
}
}
svars->state[t] |= ST_SENT_FIND_OLD;
msgs_found_sel( svars, t );
}
static void
msg_found_sel( int sts, int uid, void *aux )
{
SVARS_CHECK_RET_VARS(find_vars_t);
switch (sts) {
case DRV_OK:
debug( " -> new UID %d\n", uid );
Fprintf( svars->jfp, "%c %d %d %d\n", "<>"[t], vars->srec->uid[M], vars->srec->uid[S], uid );
vars->srec->uid[t] = uid;
vars->srec->tuid[0] = 0;
break;
default:
debug( " -> TUID lost\n" );
Fprintf( svars->jfp, "& %d %d\n", vars->srec->uid[M], vars->srec->uid[S] );
vars->srec->flags = 0;
vars->srec->tuid[0] = 0;
break;
}
free( vars );
svars->find_old_done[t]++;
stats( svars );
msgs_found_sel( svars, t );
} }
typedef struct { typedef struct {
@ -1021,8 +998,9 @@ static void msg_copied_p2( sync_vars_t *svars, sync_rec_t *srec, int t, message_
static void msgs_copied( sync_vars_t *svars, int t ); static void msgs_copied( sync_vars_t *svars, int t );
static void static void
msgs_found_sel( sync_vars_t *svars, int t ) box_loaded( int sts, void *aux )
{ {
DECL_SVARS;
sync_rec_t *srec, *nsrec = 0; sync_rec_t *srec, *nsrec = 0;
message_t *tmsg; message_t *tmsg;
copy_vars_t *cv; copy_vars_t *cv;
@ -1032,8 +1010,18 @@ msgs_found_sel( sync_vars_t *svars, int t )
int sflags, nflags, aflags, dflags, nex; int sflags, nflags, aflags, dflags, nex;
char fbuf[16]; /* enlarge when support for keywords is added */ char fbuf[16]; /* enlarge when support for keywords is added */
if (!(svars->state[t] & ST_SENT_FIND_OLD) || svars->find_old_done[t] < svars->find_old_total[t]) if (check_ret( sts, aux ))
return; return;
INIT_SVARS(aux);
svars->state[t] |= ST_LOADED;
info( "%s: %d messages, %d recent\n", str_ms[t], svars->ctx[t]->count, svars->ctx[t]->recent );
if (svars->state[t] & S_FIND) {
svars->state[t] &= ~S_FIND;
debug( "matching previously copied messages on %s\n", str_ms[t] );
match_tuids( svars, t );
}
Fprintf( svars->jfp, "%c %d\n", "{}"[t], svars->ctx[t]->uidnext );
/* /*
* Mapping tmsg -> srec (this variant) is dog slow for new messages. * Mapping tmsg -> srec (this variant) is dog slow for new messages.
@ -1043,6 +1031,8 @@ msgs_found_sel( sync_vars_t *svars, int t )
*/ */
debug( "matching messages on %s against sync records\n", str_ms[t] ); debug( "matching messages on %s against sync records\n", str_ms[t] );
for (tmsg = svars->ctx[t]->msgs; tmsg; tmsg = tmsg->next) { for (tmsg = svars->ctx[t]->msgs; tmsg; tmsg = tmsg->next) {
if (tmsg->srec) /* found by TUID */
continue;
uid = tmsg->uid; uid = tmsg->uid;
if (DFlags & DEBUG) { if (DFlags & DEBUG) {
make_flags( tmsg->flags, fbuf ); make_flags( tmsg->flags, fbuf );
@ -1133,7 +1123,7 @@ msgs_found_sel( sync_vars_t *svars, int t )
return; return;
} }
if (!(svars->state[1-t] & ST_SENT_FIND_OLD) || svars->find_old_done[1-t] < svars->find_old_total[1-t]) if (!(svars->state[1-t] & ST_LOADED))
return; return;
if (svars->uidval[M] < 0 || svars->uidval[S] < 0) { if (svars->uidval[M] < 0 || svars->uidval[S] < 0) {
@ -1369,6 +1359,8 @@ msg_copied( int sts, int uid, copy_vars_t *vars )
SVARS_CHECK_CANCEL_RET; SVARS_CHECK_CANCEL_RET;
switch (sts) { switch (sts) {
case SYNC_OK: case SYNC_OK:
if (uid < 0)
svars->state[t] |= S_FIND;
msg_copied_p2( svars, vars->srec, t, vars->msg, uid ); msg_copied_p2( svars, vars->srec, t, vars->msg, uid );
break; break;
case SYNC_NOGOOD: case SYNC_NOGOOD:
@ -1405,55 +1397,45 @@ msg_copied_p2( sync_vars_t *svars, sync_rec_t *srec, int t, message_t *tmsg, int
} }
} }
static void msg_found_new( int sts, int uid, void *aux ); static void msgs_found_new( int sts, void *aux );
static void msgs_new_done( sync_vars_t *svars, int t );
static void sync_close( sync_vars_t *svars, int t ); static void sync_close( sync_vars_t *svars, int t );
static void static void
msgs_copied( sync_vars_t *svars, int t ) msgs_copied( sync_vars_t *svars, int t )
{ {
sync_rec_t *srec;
find_vars_t *fv;
if (!(svars->state[t] & ST_SENT_NEW) || svars->new_done[t] < svars->new_total[t]) if (!(svars->state[t] & ST_SENT_NEW) || svars->new_done[t] < svars->new_total[t])
return; return;
debug( "finding just copied messages on %s\n", str_ms[t] ); if (svars->state[t] & S_FIND) {
for (srec = svars->srecs; srec; srec = srec->next) { debug( "finding just copied messages on %s\n", str_ms[t] );
if (srec->status & S_DEAD) svars->drv[t]->find_new_msgs( svars->ctx[t], msgs_found_new, AUX );
continue; } else {
if (srec->tuid[0] && srec->uid[t] == -2) { msgs_new_done( svars, t );
debug( " pair(%d,%d): lookup %s, TUID %." stringify(TUIDL) "s\n", srec->uid[M], srec->uid[S], str_ms[t], srec->tuid );
svars->find_new_total[t]++;
stats( svars );
fv = nfmalloc( sizeof(*fv) );
fv->aux = AUX;
fv->srec = srec;
DRIVER_CALL(find_msg( svars->ctx[t], srec->tuid, msg_found_new, fv ));
}
} }
svars->state[t] |= ST_SENT_FIND_NEW;
sync_close( svars, t );
} }
static void static void
msg_found_new( int sts, int uid, void *aux ) msgs_found_new( int sts, void *aux )
{ {
SVARS_CHECK_RET_VARS(find_vars_t); SVARS_CHECK_RET;
switch (sts) { switch (sts) {
case DRV_OK: case DRV_OK:
debug( " -> new UID %d\n", uid ); debug( "matching just copied messages on %s\n", str_ms[t] );
break; break;
default: default:
warn( "Warning: cannot find newly stored message %." stringify(TUIDL) "s on %s.\n", vars->srec->tuid, str_ms[t] ); warn( "Warning: cannot find newly stored messages on %s.\n", str_ms[t] );
uid = 0;
break; break;
} }
Fprintf( svars->jfp, "%c %d %d %d\n", "<>"[t], vars->srec->uid[M], vars->srec->uid[S], uid ); match_tuids( svars, t );
vars->srec->uid[t] = uid; msgs_new_done( svars, t );
vars->srec->tuid[0] = 0; }
free( vars );
svars->find_new_done[t]++; static void
stats( svars ); msgs_new_done( sync_vars_t *svars, int t )
{
Fprintf( svars->jfp, "%c %d\n", "{}"[t], svars->ctx[t]->uidnext );
svars->state[t] |= ST_FOUND_NEW;
sync_close( svars, t ); sync_close( svars, t );
} }
@ -1614,8 +1596,7 @@ static void box_closed_p2( sync_vars_t *svars, int t );
static void static void
sync_close( sync_vars_t *svars, int t ) sync_close( sync_vars_t *svars, int t )
{ {
if ((~svars->state[t] & (ST_SENT_FIND_NEW|ST_SENT_TRASH)) || if ((~svars->state[t] & (ST_FOUND_NEW|ST_SENT_TRASH)) ||
svars->find_new_done[t] < svars->find_new_total[t] ||
svars->trash_done[t] < svars->trash_total[t]) svars->trash_done[t] < svars->trash_total[t])
return; return;
@ -1686,7 +1667,9 @@ box_closed_p2( sync_vars_t *svars, int t )
} }
} }
Fprintf( svars->nfp, "%d:%d %d:%d:%d\n", svars->uidval[M], svars->maxuid[M], svars->uidval[S], svars->smaxxuid, svars->maxuid[S] ); Fprintf( svars->nfp, "%d:%d:%d %d:%d:%d:%d\n",
svars->uidval[M], svars->maxuid[M], svars->ctx[M]->uidnext,
svars->uidval[S], svars->smaxxuid, svars->maxuid[S], svars->ctx[S]->uidnext );
for (srec = svars->srecs; srec; srec = srec->next) { for (srec = svars->srecs; srec; srec = srec->next) {
if (srec->status & S_DEAD) if (srec->status & S_DEAD)
continue; continue;