add a very limited INCLUDE_ONLY option

the result of propagating a deletion is flagging as deleted, so shut up
if the only remote change is exactly that.

Dockerfile

@@ -0,0 +1,41 @@
+#FROM debian:bookworm-20231030-slim
+FROM debian:bullseye-20220801
+
+# need to add
+      # removed
+      # libsasl2-modules \
+      # ca-certificates \
+
+# version pinning is being handled in our from line
+# hadolint ignore=DL3008
+RUN true && \
+    apt-get update && \
+    groupadd --gid 1000 user && \
+    useradd -m --home-dir /home/user --shell /bin/sh --uid 1000 --gid 1000 user && \
+    apt-get install -y --no-install-recommends \
+      libsasl2-2 \
+      libsasl2-dev \
+      perl \
+      libdatetime-format-dateparse-perl \
+      autoconf \
+      automake \
+      zlib1g-dev \
+      libdb-dev \
+      libsasl2-dev \
+      libssl-dev \
+      gcc \
+      make \
+      git \
+      && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    true
+
+# Built with
+# apt install build-essential dh-autoreconf git
+# apt install libssl-dev
+# apt install zlib1g-dev
+# apt install libsasl2-dev
+
+WORKDIR /home/user
+USER user

build

@@ -0,0 +1,3 @@
+#!/bin/sh
+# Note we assume this is run with podman. Take -u 0:0 out if using docker
+docker run -u 0:0 -it --rm -v $(pwd):/make -w /make isync-build make 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([isync], [1.4.0])
+AC_INIT([isync], [1.4.4])
 AC_CONFIG_HEADERS([autodefs.h])
 
 AC_CANONICAL_TARGET
@@ -62,8 +62,8 @@ if test "x$ob_cv_perl_ver" = "xno"; then
 fi
 
 AC_CACHE_CHECK([whether strftime supports %z], ob_cv_strftime_z,
-    [AC_TRY_RUN(
-[#include <time.h>
+    [AC_RUN_IFELSE([AC_LANG_SOURCE([[
+#include <time.h>
 #include <string.h>
 
 int main(void)
@@ -73,12 +73,12 @@ int main(void)
     strftime(buf, sizeof(buf), "%z", localtime(&t));
     return !(buf[0] == '+' || buf[0] == '-');
 }
-], [ob_cv_strftime_z=yes], [ob_cv_strftime_z=no], [ob_cv_strftime_z="yes (assumed)"])])
+]])], [ob_cv_strftime_z=yes], [ob_cv_strftime_z=no], [ob_cv_strftime_z="yes (assumed)"])])
 if test "x$ob_cv_strftime_z" = x"no"; then
     AC_MSG_ERROR([libc lacks necessary feature])
 fi
 
-AC_CHECK_HEADERS(sys/poll.h sys/select.h)
+AC_CHECK_HEADERS(poll.h sys/select.h)
 AC_CHECK_FUNCS(vasprintf strnlen memrchr timegm)
 
 AC_CHECK_LIB(socket, socket, [SOCK_LIBS="-lsocket"])
@@ -96,7 +96,7 @@ fi
 
 have_ssl_paths=
 AC_ARG_WITH(ssl,
-  AC_HELP_STRING([--with-ssl[=PATH]], [where to look for SSL [detect]]),
+  AS_HELP_STRING([--with-ssl[=PATH]], [where to look for SSL [detect]]),
   [ob_cv_with_ssl=$withval])
 if test "x$ob_cv_with_ssl" != xno; then
   case $ob_cv_with_ssl in
@@ -193,12 +193,13 @@ AC_CACHE_CHECK([for Berkeley DB >= 4.1], ac_cv_berkdb4,
   [ac_cv_berkdb4=no
     sav_LIBS=$LIBS
     LIBS="$LIBS -ldb"
-   AC_TRY_LINK([#include <db.h>],
-               [DB *db;
-	        db_create(&db, 0, 0);
-	        db->truncate(db, 0, 0, 0);
-	        db->open(db, 0, "foo", "foo", DB_HASH, DB_CREATE, 0)],
-	       [ac_cv_berkdb4=yes])
+    AC_LINK_IFELSE([AC_LANG_PROGRAM(
+        [#include <db.h>],
+        [DB *db;
+         db_create(&db, 0, 0);
+         db->truncate(db, 0, 0, 0);
+         db->open(db, 0, "foo", "foo", DB_HASH, DB_CREATE, 0);
+        ])], [ac_cv_berkdb4=yes], [])
     LIBS=$sav_LIBS
   ])
 if test "x$ac_cv_berkdb4" = xyes; then

src/common.h

@@ -226,7 +226,7 @@ typedef struct notifier {
 	struct notifier *next;
 	void (*cb)( int what, void *aux );
 	void *aux;
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	uint index;
 #else
 	int fd;
@@ -234,8 +234,8 @@ typedef struct notifier {
 #endif
 } notifier_t;
 
-#ifdef HAVE_SYS_POLL_H
-# include <sys/poll.h>
+#ifdef HAVE_POLL_H
+# include <poll.h>
 #else
 # define POLLIN 1
 # define POLLOUT 4

src/driver.h

@@ -35,7 +35,6 @@ typedef struct driver driver_t;
 	struct store_conf *next; \
 	char *name; \
 	driver_t *driver; \
-	const char *path;  /* should this be here? its interpretation is driver-specific */ \
 	const char *flat_delim; \
 	const char *map_inbox; \
 	const char *trash; \

src/drv_imap.c

@@ -74,6 +74,7 @@ typedef union imap_store_conf {
 	struct {
 		STORE_CONF
 		imap_server_conf_t *server;
+		char *path;  // Note: this may be modified after the delimiter is determined.
 		char delimiter;
 		char use_namespace;
 		char use_lsub;
@@ -114,8 +115,8 @@ union imap_store {
 	struct {
 		STORE(union imap_store)
 		const char *label;  // foreign
-		const char *prefix;
 		const char *name;
+		char *prefix;
 		uint ref_count;
 		uint opts;
 		enum { SST_BAD, SST_HALF, SST_GOOD } state;
@@ -216,7 +217,6 @@ typedef union {
 		IMAP_CMD
 		void (*callback)( int sts, uint uid, void *aux );
 		void *callback_aux;
-		uint out_uid;
 	};
 } imap_cmd_out_uid_t;
 
@@ -335,42 +335,45 @@ done_imap_cmd( imap_store_t *ctx, imap_cmd_t *cmd, int response )
 static void
 send_imap_cmd( imap_store_t *ctx, imap_cmd_t *cmd )
 {
-	int litplus, iovcnt = 1;
-	int bufl;
-	const char *buffmt;
-	conn_iovec_t iov[3];
-	char buf[4096];
+	int litplus, iovcnt = 3;
+	uint tbufl, lbufl;
+	conn_iovec_t iov[5];
+	char tagbuf[16];
+	char lenbuf[16];
 
 	cmd->tag = ++ctx->nexttag;
+	tbufl = nfsnprintf( tagbuf, sizeof(tagbuf), "%d ", cmd->tag );
 	if (!cmd->param.data) {
-		buffmt = "%d %s\r\n";
+		memcpy( lenbuf, "\r\n", 3 );
+		lbufl = 2;
 		litplus = 0;
 	} else if ((cmd->param.to_trash && ctx->trashnc == TrashUnknown) || !CAP(LITERALPLUS) || cmd->param.data_len >= 100*1024) {
-		buffmt = "%d %s{%d}\r\n";
+		lbufl = nfsnprintf( lenbuf, sizeof(lenbuf), "{%u}\r\n", cmd->param.data_len );
 		litplus = 0;
 	} else {
-		buffmt = "%d %s{%d+}\r\n";
+		lbufl = nfsnprintf( lenbuf, sizeof(lenbuf), "{%u+}\r\n", cmd->param.data_len );
 		litplus = 1;
 	}
-DIAG_PUSH
-DIAG_DISABLE("-Wformat-nonliteral")
-	bufl = nfsnprintf( buf, sizeof(buf), buffmt,
-	                   cmd->tag, cmd->cmd, cmd->param.data_len );
-DIAG_POP
 	if (DFlags & DEBUG_NET) {
 		if (ctx->num_in_progress)
 			printf( "(%d in progress) ", ctx->num_in_progress );
 		if (starts_with( cmd->cmd, -1, "LOGIN", 5 ))
-			printf( "%s>>> %d LOGIN <user> <pass>\n", ctx->label, cmd->tag );
+			printf( "%s>>> %sLOGIN <user> <pass>\r\n", ctx->label, tagbuf );
 		else if (starts_with( cmd->cmd, -1, "AUTHENTICATE PLAIN", 18 ))
-			printf( "%s>>> %d AUTHENTICATE PLAIN <authdata>\n", ctx->label, cmd->tag );
+			printf( "%s>>> %sAUTHENTICATE PLAIN <authdata>\r\n", ctx->label, tagbuf );
 		else
-			printf( "%s>>> %s", ctx->label, buf );
+			printf( "%s>>> %s%s%s", ctx->label, tagbuf, cmd->cmd, lenbuf );
 		fflush( stdout );
 	}
-	iov[0].buf = buf;
-	iov[0].len = (uint)bufl;
+	iov[0].buf = tagbuf;
+	iov[0].len = tbufl;
 	iov[0].takeOwn = KeepOwn;
+	iov[1].buf = cmd->cmd;
+	iov[1].len = strlen( cmd->cmd );
+	iov[1].takeOwn = KeepOwn;
+	iov[2].buf = lenbuf;
+	iov[2].len = lbufl;
+	iov[2].takeOwn = KeepOwn;
 	if (litplus) {
 		if (DFlags & DEBUG_NET_ALL) {
 			printf( "%s>>>>>>>>>\n", ctx->label );
@@ -378,15 +381,15 @@ DIAG_POP
 			printf( "%s>>>>>>>>>\n", ctx->label );
 			fflush( stdout );
 		}
-		iov[1].buf = cmd->param.data;
-		iov[1].len = cmd->param.data_len;
-		iov[1].takeOwn = GiveOwn;
+		iov[3].buf = cmd->param.data;
+		iov[3].len = cmd->param.data_len;
+		iov[3].takeOwn = GiveOwn;
 		cmd->param.data = NULL;
 		ctx->buffer_mem -= cmd->param.data_len;
-		iov[2].buf = "\r\n";
-		iov[2].len = 2;
-		iov[2].takeOwn = KeepOwn;
-		iovcnt = 3;
+		iov[4].buf = "\r\n";
+		iov[4].len = 2;
+		iov[4].takeOwn = KeepOwn;
+		iovcnt = 5;
 	}
 	socket_write( &ctx->conn, iov, iovcnt );
 	if (cmd->param.to_trash && ctx->trashnc == TrashUnknown)
@@ -512,7 +515,20 @@ imap_vprintf( const char *fmt, va_list ap )
 	const char *s;
 	char *d, *ed;
 	char c;
-	char buf[4096];
+#define MAX_SEGS 16
+#define add_seg(s, l) \
+		do { \
+			if (nsegs == MAX_SEGS) \
+				oob(); \
+			segs[nsegs] = s; \
+			segls[nsegs++] = l; \
+			totlen += l; \
+		} while (0)
+	int nsegs = 0;
+	uint totlen = 0;
+	const char *segs[MAX_SEGS];
+	uint segls[MAX_SEGS];
+	char buf[1000];
 
 	d = buf;
 	ed = d + sizeof(buf);
@@ -521,12 +537,10 @@ imap_vprintf( const char *fmt, va_list ap )
 		c = *fmt;
 		if (!c || c == '%') {
 			uint l = fmt - s;
-			if (d + l > ed)
-				oob();
-			memcpy( d, s, l );
-			d += l;
+			if (l)
+				add_seg( s, l );
 			if (!c)
-				return nfstrndup( buf, (size_t)(d - buf) );
+				break;
 			uint maxlen = UINT_MAX;
 			c = *++fmt;
 			if (c == '\\') {
@@ -535,6 +549,7 @@ imap_vprintf( const char *fmt, va_list ap )
 					fputs( "Fatal: unsupported escaped format specifier. Please report a bug.\n", stderr );
 					abort();
 				}
+				char *bd = d;
 				s = va_arg( ap, const char * );
 				while ((c = *s++)) {
 					if (d + 2 > ed)
@@ -543,6 +558,9 @@ imap_vprintf( const char *fmt, va_list ap )
 						*d++ = '\\';
 					*d++ = c;
 				}
+				l = d - bd;
+				if (l)
+					add_seg( bd, l );
 			} else { /* \\ cannot be combined with anything else. */
 				if (c == '.') {
 					c = *++fmt;
@@ -556,18 +574,21 @@ imap_vprintf( const char *fmt, va_list ap )
 				if (c == 'c') {
 					if (d + 1 > ed)
 						oob();
+					add_seg( d, 1 );
 					*d++ = (char)va_arg( ap , int );
 				} else if (c == 's') {
 					s = va_arg( ap, const char * );
 					l = strnlen( s, maxlen );
-					if (d + l > ed)
-						oob();
-					memcpy( d, s, l );
-					d += l;
+					if (l)
+						add_seg( s, l );
 				} else if (c == 'd') {
-					d += nfsnprintf( d, ed - d, "%d", va_arg( ap , int ) );
+					l = nfsnprintf( d, ed - d, "%d", va_arg( ap, int ) );
+					add_seg( d, l );
+					d += l;
 				} else if (c == 'u') {
-					d += nfsnprintf( d, ed - d, "%u", va_arg( ap , uint ) );
+					l = nfsnprintf( d, ed - d, "%u", va_arg( ap, uint ) );
+					add_seg( d, l );
+					d += l;
 				} else {
 					fputs( "Fatal: unsupported format specifier. Please report a bug.\n", stderr );
 					abort();
@@ -578,6 +599,13 @@ imap_vprintf( const char *fmt, va_list ap )
 			fmt++;
 		}
 	}
+	char *out = d = nfmalloc( totlen + 1 );
+	for (int i = 0; i < nsegs; i++) {
+		memcpy( d, segs[i], segls[i] );
+		d += segls[i];
+	}
+	*d = 0;
+	return out;
 }
 
 static void
@@ -849,6 +877,11 @@ parse_imap_list( imap_store_t *ctx, char **sp, parse_list_state_t *sts )
 			bytes = (int)(cur->len = strtoul( s + 1, &s, 10 ));
 			if (*s != '}' || *++s)
 				goto bail;
+			if ((uint)bytes >= INT_MAX) {
+				error( "IMAP error: excessively large literal from %s "
+				       "- THIS MIGHT BE AN ATTEMPT TO HACK YOU!\n", ctx->conn.name );
+				goto bail;
+			}
 
 			s = cur->val = nfmalloc( cur->len + 1 );
 			s[cur->len] = 0;
@@ -924,6 +957,7 @@ parse_imap_list( imap_store_t *ctx, char **sp, parse_list_state_t *sts )
 	}
   bail:
 	free_list( sts->head );
+	sts->level = 0;
 	return LIST_BAD;
 }
 
@@ -1181,7 +1215,8 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
 		for (cmdp = ctx->in_progress; cmdp; cmdp = cmdp->next)
 			if (cmdp->param.uid == uid)
 				goto gotuid;
-		goto badrsp;
+		error( "IMAP error: unexpected FETCH response with BODY (UID %u)\n", uid );
+		return LIST_BAD;
 	  gotuid:
 		msgdata = ((imap_cmd_fetch_msg_t *)cmdp)->msg_data;
 		msgdata->data = body->val;
@@ -1208,9 +1243,8 @@ parse_fetch_rsp( imap_store_t *ctx, list_t *list, char *s ATTR_UNUSED )
 			memcpy( cur->tuid, tuid, TUIDL );
 		status &= ~(M_FLAGS | M_RECENT | M_SIZE | M_HEADER);
 	} else {
-	  badrsp:
-		error( "IMAP error: unexpected FETCH response (UID %u)\n", uid );
-		return LIST_BAD;
+		// These may come in as a result of STORE FLAGS despite .SILENT.
+		status &= ~(M_FLAGS | M_RECENT);
 	}
 
 	if (status) {
@@ -1252,48 +1286,64 @@ parse_response_code( imap_store_t *ctx, imap_cmd_t *cmd, char *s )
 	if (!s || *s != '[')
 		return RESP_OK;		/* no response code */
 	s++;
-	if (!(p = strchr( s, ']' ))) {
-	  bad_resp:
+	if (!(arg = next_arg( &s ))) {
 		error( "IMAP error: malformed response code\n" );
 		return RESP_CANCEL;
 	}
-	*p++ = 0;
-	if (!(arg = next_arg( &s )))
-		goto bad_resp;
 	if (!strcmp( "UIDVALIDITY", arg )) {
 		if (!(arg = next_arg( &s )) ||
-		    (ctx->uidvalidity = strtoul( arg, &earg, 10 ), *earg))
+		    (ctx->uidvalidity = strtoul( arg, &earg, 10 ), *earg != ']'))
 		{
 			error( "IMAP error: malformed UIDVALIDITY status\n" );
 			return RESP_CANCEL;
 		}
 	} else if (!strcmp( "UIDNEXT", arg )) {
 		if (!(arg = next_arg( &s )) ||
-		    (ctx->uidnext = strtoul( arg, &earg, 10 ), *earg))
+		    (ctx->uidnext = strtoul( arg, &earg, 10 ), *earg != ']'))
 		{
 			error( "IMAP error: malformed UIDNEXT status\n" );
 			return RESP_CANCEL;
 		}
 	} else if (!strcmp( "CAPABILITY", arg )) {
+		if (!s || !(p = strchr( s, ']' ))) {
+			error( "IMAP error: malformed CAPABILITY status\n" );
+			return RESP_CANCEL;
+		}
+		*p = 0;
 		parse_capability( ctx, s );
-	} else if (!strcmp( "ALERT", arg )) {
+	} else if (!strcmp( "ALERT]", arg )) {
 		/* RFC2060 says that these messages MUST be displayed
 		 * to the user
 		 */
-		for (; isspace( (uchar)*p ); p++);
-		error( "*** IMAP ALERT *** %s\n", p );
-	} else if (cmd && !strcmp( "APPENDUID", arg )) {
+		if (!s) {
+			error( "IMAP error: malformed ALERT status\n" );
+			return RESP_CANCEL;
+		}
+		for (; isspace( (uchar)*s ); s++);
+		error( "*** IMAP ALERT *** %s\n", s );
+	} else if (!strcmp( "APPENDUID", arg )) {
+		// The checks ensure that:
+		// - cmd => this is the final tagged response of a command, at which
+		//   point cmd was already removed from ctx->in_progress, so param.uid
+		//   is available for reuse.
+		// - !param.uid => the command isn't actually a FETCH. This doesn't
+		//   really matter, as the field is safe to overwrite given the
+		//   previous condition; it just has no effect for non-APPENDs.
+		if (!cmd || cmd->param.uid) {
+			error( "IMAP error: unexpected APPENDUID status\n" );
+			return RESP_CANCEL;
+		}
 		if (!(arg = next_arg( &s )) ||
 		    (ctx->uidvalidity = strtoul( arg, &earg, 10 ), *earg) ||
 		    !(arg = next_arg( &s )) ||
-		    (((imap_cmd_out_uid_t *)cmd)->out_uid = strtoul( arg, &earg, 10 ), *earg))
+		    (cmd->param.uid = strtoul( arg, &earg, 10 ), *earg != ']'))
 		{
 			error( "IMAP error: malformed APPENDUID status\n" );
 			return RESP_CANCEL;
 		}
 	} else if (!strcmp( "PERMANENTFLAGS", arg )) {
 		parse_list_init( &ctx->parse_list_sts );
-		if (parse_imap_list( NULL, &s, &ctx->parse_list_sts ) != LIST_OK) {
+		if (parse_imap_list( NULL, &s, &ctx->parse_list_sts ) != LIST_OK || *s != ']') {
 			error( "IMAP error: malformed PERMANENTFLAGS status\n" );
 			return RESP_CANCEL;
 		}
@@ -1368,11 +1418,18 @@ is_INBOX( imap_store_t *ctx, const char *arg, int argl )
 	return 1;
 }
 
+static void
+normalize_INBOX( imap_store_t *ctx, char *arg, int argl )
+{
+	if (is_inbox( ctx, arg, argl ))
+		memcpy( arg, "INBOX", 5 );
+}
+
 static int
 parse_list_rsp_p2( imap_store_t *ctx, list_t *list, char *cmd ATTR_UNUSED )
 {
 	string_list_t *narg;
-	char *arg;
+	char *arg, c;
 	int argl;
 	uint l;
 
@@ -1382,33 +1439,31 @@ parse_list_rsp_p2( imap_store_t *ctx, list_t *list, char *cmd ATTR_UNUSED )
 	}
 	arg = list->val;
 	argl = (int)list->len;
+	if (argl > 1000) {
+		warn( "IMAP warning: ignoring unreasonably long mailbox name '%.100s[...]'\n", arg );
+		return LIST_OK;
+	}
+	// The server might be weird and have a non-uppercase INBOX. It
+	// may legitimately do so, but we need the canonical spelling.
+	normalize_INBOX( ctx, arg, argl );
 	if ((l = strlen( ctx->prefix ))) {
 		if (!starts_with( arg, argl, ctx->prefix, l )) {
-			if (is_inbox( ctx, arg, argl )) {
-				// INBOX and its subfolders bypass the namespace.
-				goto inbox;
+			if (!is_INBOX( ctx, arg, argl ))
+				return LIST_OK;
+			// INBOX and its subfolders bypass the namespace.
+		} else {
+			arg += l;
+			argl -= l;
+			// A folder named "INBOX" would be indistinguishable from the
+			// actual INBOX after prefix stripping, so drop it. This applies
+			// only to the fully uppercased spelling, as our canonical box
+			// names are case-sensitive (unlike IMAP's INBOX).
+			if (is_INBOX( ctx, arg, argl )) {
+				if (!arg[5])  // No need to complain about subfolders as well.
+					warn( "IMAP warning: ignoring INBOX in %s\n", ctx->prefix );
+				return LIST_OK;
 			}
-			return LIST_OK;
 		}
-		arg += l;
-		argl -= l;
-		// A folder named "INBOX" would be indistinguishable from the
-		// actual INBOX after prefix stripping, so drop it. This applies
-		// only to the fully uppercased spelling, as our canonical box
-		// names are case-sensitive (unlike IMAP's INBOX).
-		if (is_INBOX( ctx, arg, argl )) {
-			if (!arg[5])  // No need to complain about subfolders as well.
-				warn( "IMAP warning: ignoring INBOX in %s\n", ctx->prefix );
-			return LIST_OK;
-		}
-	} else if (is_inbox( ctx, arg, argl )) {
-	  inbox:
-		// The server might be weird and have a non-uppercase INBOX. It
-		// may legitimately do so, but we need the canonical spelling.
-		// Note that we do that only after prefix matching, under the
-		// assumption that the NAMESPACE (or Path) matches the
-		// capitalization of LIST.
-		memcpy( arg, "INBOX", 5 );
 	}
 	if (argl >= 5 && !memcmp( arg + argl - 5, ".lock", 5 )) /* workaround broken servers */
 		return LIST_OK;
@@ -1416,6 +1471,34 @@ parse_list_rsp_p2( imap_store_t *ctx, list_t *list, char *cmd ATTR_UNUSED )
 		warn( "IMAP warning: ignoring mailbox %s (reserved character '/' in name)\n", arg );
 		return LIST_OK;
 	}
+	// Validate the normalized name. Technically speaking, we could tolerate
+	// '//' and '/./', and '/../' being forbidden is a limitation of the Maildir
+	// driver, but there isn't really a legitimate reason for these being present.
+	for (const char *p = narg->string, *sp = p;;) {
+		if (!(c = *p) || c == '/') {
+			uint pcl = (uint)(p - sp);
+			if (!pcl) {
+				error( "IMAP warning: ignoring mailbox '%s' due to empty name component\n", narg->string );
+				free( narg );
+				return LIST_OK;
+			}
+			if (pcl == 1 && sp[0] == '.') {
+				error( "IMAP warning: ignoring mailbox '%s' due to '.' component\n", narg->string );
+				free( narg );
+				return LIST_OK;
+			}
+			if (pcl == 2 && sp[0] == '.' && sp[1] == '.') {
+				error( "IMAP error: LIST'd mailbox name '%s' contains '..' component - THIS MIGHT BE AN ATTEMPT TO HACK YOU!\n", narg->string );
+				free( narg );
+				return LIST_BAD;
+			}
+			if (!c)
+				break;
+			sp = ++p;
+		} else {
+			++p;
+		}
+	}
 	narg->next = ctx->boxes;
 	ctx->boxes = narg;
 	return LIST_OK;
@@ -2505,6 +2588,8 @@ imap_open_store_finalize( imap_store_t *ctx )
 	ctx->state = SST_GOOD;
 	if (!ctx->prefix)
 		ctx->prefix = "";
+	else
+		normalize_INBOX( ctx, ctx->prefix, -1 );
 	ctx->trashnc = TrashUnknown;
 	ctx->callbacks.imap_open( DRV_OK, ctx->callback_aux );
 }
@@ -3129,7 +3214,6 @@ imap_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
 	ctx->buffer_mem += data->len;
 	cmd->param.data_len = data->len;
 	cmd->param.data = data->data;
-	cmd->out_uid = 0;
 
 	if (to_trash) {
 		cmd->param.create = 1;
@@ -3165,7 +3249,7 @@ imap_store_msg_p2( imap_store_t *ctx ATTR_UNUSED, imap_cmd_t *cmd, int response
 	imap_cmd_out_uid_t *cmdp = (imap_cmd_out_uid_t *)cmd;
 
 	transform_msg_response( &response );
-	cmdp->callback( response, cmdp->out_uid, cmdp->callback_aux );
+	cmdp->callback( response, cmdp->param.uid, cmdp->callback_aux );
 }
 
 /******************* imap_find_new_msgs *******************/

src/drv_maildir.c

@@ -54,6 +54,7 @@ typedef union maildir_store_conf {
 	store_conf_t gen;
 	struct {
 		STORE_CONF
+		char *path;
 		char *inbox;
 #ifdef USE_DB
 		int alt_map;
@@ -394,7 +395,7 @@ static int maildir_list_inbox( maildir_store_t *ctx, int flags, const char *base
 static int maildir_list_path( maildir_store_t *ctx, int flags, const char *inbox );
 
 static int
-maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags,
+maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
                       const char *inbox, uint inboxLen, const char *basePath, uint basePathLen,
                       char *path, int pathLen, char *name, int nameLen )
 {
@@ -416,6 +417,12 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags,
 		closedir( dir );
 		return -1;
 	}
+	if (++depth > 10) {
+		// We do the other checks first to avoid confusing error messages for files.
+		error( "Maildir error: path %s is too deeply nested. Symlink loop?\n", path );
+		closedir( dir );
+		return -1;
+	}
 	while ((de = readdir( dir ))) {
 		const char *ent = de->d_name;
 		if (ent[0] == '.' && (!ent[1] || (ent[1] == '.' && !ent[2])))
@@ -463,7 +470,7 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags,
 				add_string_list( &ctx->boxes, name );
 			path[pl] = 0;
 			name[nl++] = '/';
-			if (maildir_list_recurse( ctx, isBox + 1, flags, inbox, inboxLen, basePath, basePathLen, path, pl, name, nl ) < 0) {
+			if (maildir_list_recurse( ctx, isBox + 1, flags, depth, inbox, inboxLen, basePath, basePathLen, path, pl, name, nl ) < 0) {
 				closedir( dir );
 				return -1;
 			}
@@ -484,7 +491,7 @@ maildir_list_inbox( maildir_store_t *ctx, int flags, const char *basePath )
 
 	add_string_list( &ctx->boxes, "INBOX" );
 	return maildir_list_recurse(
-	        ctx, 1, flags, NULL, 0, basePath, basePath ? strlen( basePath ) - 1 : 0,
+	        ctx, 1, flags, 0, NULL, 0, basePath, basePath ? strlen( basePath ) - 1 : 0,
 	        path, nfsnprintf( path, _POSIX_PATH_MAX, "%s/", ctx->conf->inbox ),
 	        name, nfsnprintf( name, _POSIX_PATH_MAX, "INBOX/" ) );
 }
@@ -501,7 +508,7 @@ maildir_list_path( maildir_store_t *ctx, int flags, const char *inbox )
 	if (maildir_ensure_path( ctx->conf ) < 0)
 		return -1;
 	return maildir_list_recurse(
-	        ctx, 0, flags, inbox, inbox ? strlen( inbox ) : 0, NULL, 0,
+	        ctx, 0, flags, 0, inbox, inbox ? strlen( inbox ) : 0, NULL, 0,
 	        path, nfsnprintf( path, _POSIX_PATH_MAX, "%s", ctx->conf->path ),
 	        name, 0 );
 }
@@ -913,6 +920,28 @@ maildir_compare( const void *l, const void *r )
 	return strcmp( lm->base, rm->base );
 }
 
+int is_gmail(const char* name) {
+	// NOTE: This is not an exact science. For instance, there is a good
+	// shot that mail sent from containers will end up matching this pattern
+    char token[16];  // If host name is longer than 12, then we already have our answer
+
+	// we expect files in the form of: "1700172103.R12128272304961211247.hostname,U=27:2,S"
+	// we are looking for "hostname"
+	// All gmail mails have a hostname in the form of "ff6d55f971cc".
+	// All are 12 characters long
+	// All are hexadecimal
+    if (sscanf(name, "%*[^.].%*[^.].%15[^,]", token) == 1) {
+		token[14] = '\0';
+		size_t len = strlen(token);
+		if (len != 12) return 0;
+		if (strspn(token, "0123456789abcdef") == len) return 1;
+    }
+
+	// We are here because a) the input format is not as expected,
+	// or b) the hostname is not gmail
+    return 0;
+}
+
 static int
 maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 {
@@ -984,9 +1013,25 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 #endif /* USE_DB */
 				return DRV_BOX_BAD;
 			}
+			const char *filter = getenv("MBSYNC_MAILDIR_IGNORE");
+			const char *MAGIC_INCLUDE = "^[^\\.]+\\.[^\\.]+\\.[0-9a-f]{12}\\,.*";
+			char *include = getenv("MBSYNC_MAILDIR_INCLUDE_ONLY");
+			if (include && strncmp(MAGIC_INCLUDE, include, strlen(MAGIC_INCLUDE)) != 0) {
+				include = NULL;
+				error("MBSYNC_MAILDIR_INCLUDE_ONLY can only be '%s'\n", MAGIC_INCLUDE);
+			}
+			if (include && filter) {
+				include = NULL;
+				error("MBSYNC_MAILDIR_IGNORE cannot be used with MBSYNC_MAILDIR_INCLUDE_ONLY. INCLUDE_ONLY will be ignored\n");
+			}
+
 			while ((e = readdir( d ))) {
 				if (*e->d_name == '.')
 					continue;
+				if (filter && strstr(e->d_name, filter))
+					continue;
+				if (include && !is_gmail(e->d_name))
+					continue;
 				ctx->total_msgs++;
 				ctx->recent_msgs += i;
 #ifdef USE_DB
@@ -1161,7 +1206,8 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 					}
 					goto retry;
 				}
-				entry->size = (uint)st.st_size;
+				// The clipped value is good enough for MaxSize comparisons.
+				entry->size = st.st_size > UINT_MAX ? UINT_MAX : (uint)st.st_size;
 			}
 			if (want_tuid || want_msgid) {
 				if (!(f = fopen( buf, "r" ))) {
@@ -1556,12 +1602,17 @@ maildir_fetch_msg( store_t *gctx, message_t *gmsg, msg_data_t *data, int minimal
 		}
 	}
 	fstat( fd, &st );
+	if (st.st_size > INT_MAX) {
+		error( "Maildir error: %s is too big", buf );
+		goto mbad;
+	}
 	data->len = st.st_size;
 	if (data->date == -1)
 		data->date = st.st_mtime;
 	data->data = nfmalloc( data->len );
 	if (read( fd, data->data, data->len ) != data->len) {
 		sys_error( "Maildir error: cannot read %s", buf );
+	  mbad:
 		close( fd );
 		cb( DRV_MSG_BAD, aux );
 		return;

src/sync.c

@@ -406,11 +406,11 @@ copy_msg_bytes( char **out_ptr, const char *in_buf, uint *in_idx, uint in_len, i
 }
 
 static int
-copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
+copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars, int t )
 {
 	char *in_buf = vars->data.data;
 	uint in_len = vars->data.len;
-	uint idx = 0, sbreak = 0, ebreak = 0, break2 = 0;
+	uint idx = 0, sbreak = 0, ebreak = 0, break2 = UINT_MAX;
 	uint lines = 0, hdr_crs = 0, bdy_crs = 0, app_cr = 0, extra = 0;
 	uint add_subj = 0;
 
@@ -428,9 +428,10 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 					if (!vars->minimal)
 						goto oke;
 				} else {
-					if (!break2 && vars->minimal && !strncasecmp( in_buf + start, "Subject:", 8 )) {
+					if (break2 == UINT_MAX && vars->minimal &&
+					    starts_with_upper( in_buf + start, (int)(in_len - start), "SUBJECT:", 8 )) {
 						break2 = start + 8;
-						if (in_buf[break2] == ' ')
+						if (break2 < in_len && in_buf[break2] == ' ')
 							break2++;
 					}
 					lines++;
@@ -441,7 +442,7 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 						sbreak = ebreak = start;
 					if (vars->minimal) {
 						in_len = idx;
-						if (!break2) {
+						if (break2 == UINT_MAX) {
 							break2 = start;
 							add_subj = 1;
 						}
@@ -451,7 +452,8 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 				goto nloop;
 			}
 		}
-		/* invalid message */
+		warn( "Warning: message %u from %s has incomplete header; skipping.\n",
+		      vars->msg->uid, str_fn[1-t] );
 		free( in_buf );
 		return 0;
 	  oke:
@@ -493,10 +495,16 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 	}
 
 	vars->data.len = in_len + extra;
+	if (vars->data.len > INT_MAX) {
+		warn( "Warning: message %u from %s is too big after conversion; skipping.\n",
+		      vars->msg->uid, str_fn[1-t] );
+		free( in_buf );
+		return 0;
+	}
 	char *out_buf = vars->data.data = nfmalloc( vars->data.len );
 	idx = 0;
 	if (vars->srec) {
-		if (break2 && break2 < sbreak) {
+		if (break2 < sbreak) {
 			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
 			memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
 			out_buf += strlen(dummy_pfx);
@@ -512,7 +520,7 @@ copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
 		*out_buf++ = '\n';
 		idx = ebreak;
 
-		if (break2 >= sbreak) {
+		if (break2 != UINT_MAX && break2 >= sbreak) {
 			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
 			if (!add_subj) {
 				memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
@@ -556,9 +564,7 @@ msg_fetched( int sts, void *aux )
 		scr = (svars->drv[1-t]->get_caps( svars->ctx[1-t] ) / DRV_CRLF) & 1;
 		tcr = (svars->drv[t]->get_caps( svars->ctx[t] ) / DRV_CRLF) & 1;
 		if (vars->srec || scr != tcr) {
-			if (!copy_msg_convert( scr, tcr, vars )) {
-				warn( "Warning: message %u from %s has incomplete header.\n",
-				      vars->msg->uid, str_fn[1-t] );
+			if (!copy_msg_convert( scr, tcr, vars, t )) {
 				vars->cb( SYNC_NOGOOD, 0, vars );
 				return;
 			}
@@ -1365,11 +1371,10 @@ box_opened2( sync_vars_t *svars, int t )
 	for (t = 0; t < 2; t++)
 		if (svars->uidval[t] != UIDVAL_BAD && svars->uidval[t] != svars->newuidval[t])
 			fails++;
+	// If only one side changed UIDVALIDITY, we will try to re-approve it further down.
 	if (fails == 2) {
-		error( "Error: channel %s: UIDVALIDITY of both far and near side changed\n"
-		       "(far side got %u, expected %u; near side got %u, expected %u).\n",
-		       svars->chan->name,
-		       svars->newuidval[F], svars->uidval[F], svars->newuidval[N], svars->uidval[N] );
+		error( "Error: channel %s: UIDVALIDITY of both far side %s and near side %s changed.\n",
+		       svars->chan->name, svars->orig_name[F], svars->orig_name[N]);
 	  bail:
 		svars->ret = SYNC_FAIL;
 		sync_bail( svars );
@@ -1590,14 +1595,17 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 
 	for (t = 0; t < 2; t++) {
 		if (svars->uidval[t] != UIDVAL_BAD && svars->uidval[t] != svars->newuidval[t]) {
+			// This code checks whether the messages with known UIDs are actually the
+			// same messages, as recognized by their Message-IDs.
 			unsigned need = 0, got = 0;
 			debug( "trying to re-approve uid validity of %s\n", str_fn[t] );
 			for (srec = svars->srecs; srec; srec = srec->next) {
 				if (srec->status & S_DEAD)
 					continue;
+				need++;
 				if (!srec->msg[t])
 					continue;  // Message disappeared.
-				need++;  // Present paired messages require re-validation.
+				// Present paired messages require re-validation.
 				if (!srec->msg[t]->msgid)
 					continue;  // Messages without ID are useless for re-validation.
 				if (!srec->msg[1-t])
@@ -1612,15 +1620,19 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 				}
 				got++;
 			}
+			// We encountered no messages that contradict the hypothesis that the
+			// UIDVALIDITY change was spurious.
+			// If we got enough messages confirming the hypothesis, we just accept it.
+			// If there aren't quite enough messages, we check that at least 80% of
+			// those previously present are still there and confirm the hypothesis;
+			// this also covers the case of a box that was already empty.
 			if (got < 20 && got * 5 < need * 4) {
 				// Too few confirmed messages. This is very likely in the drafts folder.
 				// A proper fallback would be fetching more headers (which potentially need
 				// normalization) or the message body (which should be truncated for sanity)
 				// and comparing.
-				error( "Error: channel %s, %s box %s: Unable to recover from UIDVALIDITY change\n"
-				       "(got %u, expected %u).\n",
-				       svars->chan->name, str_fn[t], svars->orig_name[t],
-				       svars->newuidval[t], svars->uidval[t] );
+				error( "Error: channel %s, %s box %s: Unable to recover from UIDVALIDITY change.\n",
+				       svars->chan->name, str_fn[t], svars->orig_name[t] );
 				goto uvchg;
 			}
 			notice( "Notice: channel %s, %s box %s: Recovered from change of UIDVALIDITY.\n",
@@ -1684,7 +1696,11 @@ box_loaded( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux
 						JLOG( "> %u %u 0", (srec->uid[F], srec->uid[N]), "near side expired, orphaning far side" );
 						srec->uid[N] = 0;
 					} else {
-						if (srec->msg[t] && (srec->msg[t]->status & M_FLAGS) && srec->msg[t]->flags != srec->flags)
+						if (srec->msg[t] && (srec->msg[t]->status & M_FLAGS) &&
+						    // Ignore deleted flag, as that's what we'll change ourselves ...
+						    (((srec->msg[t]->flags & ~F_DELETED) != (srec->flags & ~F_DELETED)) ||
+						     // ... except for undeletion, as that's the opposite.
+						     (!(srec->msg[t]->flags & F_DELETED) && (srec->flags & F_DELETED))))
 							notice( "Notice: conflicting changes in (%u,%u)\n", srec->uid[F], srec->uid[N] );
 						if (svars->chan->ops[t] & OP_DELETE) {
 							debug( "  %sing delete\n", str_hl[t] );

src/util.c

@@ -665,7 +665,7 @@ list_unlink( list_head_t *head )
 
 static notifier_t *notifiers;
 static int changed;  /* Iterator may be invalid now. */
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 static struct pollfd *pollfds;
 static uint npolls, rpolls;
 #else
@@ -677,7 +677,7 @@ static uint npolls, rpolls;
 void
 init_notifier( notifier_t *sn, int fd, void (*cb)( int, void * ), void *aux )
 {
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	uint idx = npolls++;
 	if (rpolls < npolls) {
 		rpolls = npolls;
@@ -699,7 +699,7 @@ init_notifier( notifier_t *sn, int fd, void (*cb)( int, void * ), void *aux )
 void
 conf_notifier( notifier_t *sn, short and_events, short or_events )
 {
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	uint idx = sn->index;
 	pollfds[idx].events = (pollfds[idx].events & and_events) | or_events;
 #else
@@ -710,7 +710,7 @@ conf_notifier( notifier_t *sn, short and_events, short or_events )
 short
 notifier_config( notifier_t *sn )
 {
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	return pollfds[sn->index].events;
 #else
 	return sn->events;
@@ -721,7 +721,7 @@ void
 wipe_notifier( notifier_t *sn )
 {
 	notifier_t **snp;
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	uint idx;
 #endif
 
@@ -731,7 +731,7 @@ wipe_notifier( notifier_t *sn )
 	sn->next = NULL;
 	changed = 1;
 
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	idx = sn->index;
 	memmove( pollfds + idx, pollfds + idx + 1, (--npolls - idx) * sizeof(*pollfds) );
 	for (sn = notifiers; sn; sn = sn->next) {
@@ -803,7 +803,7 @@ event_wait( void )
 	notifier_t *sn;
 	int m;
 
-#ifdef HAVE_SYS_POLL_H
+#ifdef HAVE_POLL_H
 	int timeout = -1;
 	if ((head = timers.next) != &timers) {
 		wakeup_t *tmr = (wakeup_t *)head;