added support for IMAP DEFLATE

initial patch by Jesse Weaver <pianohacker@gmail.com>, but mostly
rewritten by me.
This commit is contained in:
Oswald Buddenhagen 2014-12-13 12:09:15 +01:00
parent f0b80e7d35
commit 139b90be29
6 changed files with 279 additions and 27 deletions

2
NEWS
View File

@ -10,6 +10,8 @@ Support for SASL (flexible authentication) has been added.
Support for Windows file systems has been added. Support for Windows file systems has been added.
Support for compressed data transfer has been added.
[1.1.0] [1.1.0]
Support for hierarchical mailboxes in Patterns. Support for hierarchical mailboxes in Patterns.

View File

@ -150,6 +150,15 @@ if test "x$ac_cv_berkdb4" = xno; then
AC_MSG_ERROR([Berkley DB >= 4.2 not found.]) AC_MSG_ERROR([Berkley DB >= 4.2 not found.])
fi fi
have_zlib=
AC_CHECK_LIB([z], [deflate],
[AC_CHECK_HEADER(zlib.h,
[have_zlib=1
AC_SUBST([Z_LIBS], ["-lz"])
AC_DEFINE([HAVE_LIBZ], 1, [if you have the zlib library])]
)]
)
AC_ARG_ENABLE(compat, AC_ARG_ENABLE(compat,
AC_HELP_STRING([--disable-compat], [don't include isync compatibility wrapper [no]]), AC_HELP_STRING([--disable-compat], [don't include isync compatibility wrapper [no]]),
[ob_cv_enable_compat=$enableval]) [ob_cv_enable_compat=$enableval])
@ -172,4 +181,9 @@ if test -n "$have_sasl_paths"; then
else else
AC_MSG_RESULT([Not using SASL]) AC_MSG_RESULT([Not using SASL])
fi fi
if test -n "$have_zlib"; then
AC_MSG_RESULT([Using zlib])
else
AC_MSG_RESULT([Not using zlib])
fi
AC_MSG_RESULT() AC_MSG_RESULT()

View File

@ -6,7 +6,7 @@ SUBDIRS = $(compat_dir)
bin_PROGRAMS = mbsync mdconvert bin_PROGRAMS = mbsync mdconvert
mbsync_SOURCES = main.c sync.c config.c util.c socket.c driver.c drv_imap.c drv_maildir.c mbsync_SOURCES = main.c sync.c config.c util.c socket.c driver.c drv_imap.c drv_maildir.c
mbsync_LDADD = -ldb $(SSL_LIBS) $(SOCK_LIBS) $(SASL_LIBS) mbsync_LDADD = -ldb $(SSL_LIBS) $(SOCK_LIBS) $(SASL_LIBS) $(Z_LIBS)
noinst_HEADERS = common.h config.h driver.h sync.h socket.h noinst_HEADERS = common.h config.h driver.h sync.h socket.h
mdconvert_SOURCES = mdconvert.c mdconvert_SOURCES = mdconvert.c

View File

@ -196,7 +196,8 @@ enum CAPABILITY {
UIDPLUS, UIDPLUS,
LITERALPLUS, LITERALPLUS,
MOVE, MOVE,
NAMESPACE NAMESPACE,
COMPRESS_DEFLATE
}; };
static const char *cap_list[] = { static const char *cap_list[] = {
@ -210,7 +211,8 @@ static const char *cap_list[] = {
"UIDPLUS", "UIDPLUS",
"LITERAL+", "LITERAL+",
"MOVE", "MOVE",
"NAMESPACE" "NAMESPACE",
"COMPRESS=DEFLATE"
}; };
#define RESP_OK 0 #define RESP_OK 0
@ -1486,6 +1488,9 @@ static void imap_open_store_authenticate2_p2( imap_store_t *, struct imap_cmd *,
static void imap_open_store_namespace( imap_store_t * ); static void imap_open_store_namespace( imap_store_t * );
static void imap_open_store_namespace_p2( imap_store_t *, struct imap_cmd *, int ); static void imap_open_store_namespace_p2( imap_store_t *, struct imap_cmd *, int );
static void imap_open_store_namespace2( imap_store_t * ); static void imap_open_store_namespace2( imap_store_t * );
#ifdef HAVE_LIBZ
static void imap_open_store_compress_p2( imap_store_t *, struct imap_cmd *, int );
#endif
static void imap_open_store_finalize( imap_store_t * ); static void imap_open_store_finalize( imap_store_t * );
#ifdef HAVE_LIBSSL #ifdef HAVE_LIBSSL
static void imap_open_store_ssl_bail( imap_store_t * ); static void imap_open_store_ssl_bail( imap_store_t * );
@ -2041,12 +2046,32 @@ imap_open_store_namespace2( imap_store_t *ctx )
ctx->prefix = nsp_1st_ns->val; ctx->prefix = nsp_1st_ns->val;
if (!ctx->delimiter) if (!ctx->delimiter)
ctx->delimiter = nfstrdup( nsp_1st_dl->val ); ctx->delimiter = nfstrdup( nsp_1st_dl->val );
#ifdef HAVE_LIBZ
if (CAP(COMPRESS_DEFLATE)) { /* XXX make that configurable */
imap_exec( ctx, 0, imap_open_store_compress_p2, "COMPRESS DEFLATE" );
return;
}
#endif
imap_open_store_finalize( ctx ); imap_open_store_finalize( ctx );
} else { } else {
imap_open_store_bail( ctx ); imap_open_store_bail( ctx );
} }
} }
#ifdef HAVE_LIBZ
static void
imap_open_store_compress_p2( imap_store_t *ctx, struct imap_cmd *cmd ATTR_UNUSED, int response )
{
if (response == RESP_NO) {
/* We already reported an error, but it's not fatal to us. */
imap_open_store_finalize( ctx );
} else if (response == RESP_OK) {
socket_start_deflate( &ctx->conn );
imap_open_store_finalize( ctx );
}
}
#endif
static void static void
imap_open_store_finalize( imap_store_t *ctx ) imap_open_store_finalize( imap_store_t *ctx )
{ {

View File

@ -280,6 +280,43 @@ static void start_tls_p3( conn_t *conn, int ok )
#endif /* HAVE_LIBSSL */ #endif /* HAVE_LIBSSL */
#ifdef HAVE_LIBZ
static void z_fake_cb( void * );
void
socket_start_deflate( conn_t *conn )
{
int result;
conn->in_z = nfcalloc( sizeof(*conn->in_z) );
result = inflateInit2(
conn->in_z,
-15 /* Use raw deflate */
);
if (result != Z_OK) {
error( "Fatal: Cannot initialize decompression: %s\n", conn->in_z->msg );
abort();
}
conn->out_z = nfcalloc( sizeof(*conn->out_z) );
result = deflateInit2(
conn->out_z,
Z_DEFAULT_COMPRESSION, /* Compression level */
Z_DEFLATED, /* Only valid value */
-15, /* Use raw deflate */
8, /* Default memory usage */
Z_DEFAULT_STRATEGY /* Don't try to do anything fancy */
);
if (result != Z_OK) {
error( "Fatal: Cannot initialize compression: %s\n", conn->out_z->msg );
abort();
}
init_wakeup( &conn->z_fake, z_fake_cb, conn );
}
#endif /* HAVE_LIBZ */
static void socket_fd_cb( int, void * ); static void socket_fd_cb( int, void * );
static void socket_fake_cb( void * ); static void socket_fake_cb( void * );
@ -500,6 +537,17 @@ socket_close( conn_t *sock )
sock->ssl = 0; sock->ssl = 0;
wipe_wakeup( &sock->ssl_fake ); wipe_wakeup( &sock->ssl_fake );
} }
#endif
#ifdef HAVE_LIBZ
if (sock->in_z) {
inflateEnd( sock->in_z );
free( sock->in_z );
sock->in_z = 0;
deflateEnd( sock->out_z );
free( sock->out_z );
sock->out_z = 0;
wipe_wakeup( &sock->z_fake );
}
#endif #endif
while (sock->write_buf) while (sock->write_buf)
dispose_chunk( sock ); dispose_chunk( sock );
@ -507,23 +555,30 @@ socket_close( conn_t *sock )
sock->append_buf = 0; sock->append_buf = 0;
} }
static void static int
socket_fill( conn_t *sock ) prepare_read( conn_t *sock, char **buf, int *len )
{ {
char *buf;
int n = sock->offset + sock->bytes; int n = sock->offset + sock->bytes;
int len = sizeof(sock->buf) - n; if (!(*len = sizeof(sock->buf) - n)) {
if (!len) {
error( "Socket error: receive buffer full. Probably protocol error.\n" ); error( "Socket error: receive buffer full. Probably protocol error.\n" );
socket_fail( sock ); socket_fail( sock );
return; return -1;
} }
*buf = sock->buf + n;
return 0;
}
static int
do_read( conn_t *sock, char *buf, int len )
{
int n;
assert( sock->fd >= 0 ); assert( sock->fd >= 0 );
buf = sock->buf + n;
#ifdef HAVE_LIBSSL #ifdef HAVE_LIBSSL
if (sock->ssl) { if (sock->ssl) {
if ((n = ssl_return( "read from", sock, SSL_read( sock->ssl, buf, len ) )) <= 0) if ((n = ssl_return( "read from", sock, SSL_read( sock->ssl, buf, len ) )) <= 0)
return; return n;
if (n == len && SSL_pending( sock->ssl )) if (n == len && SSL_pending( sock->ssl ))
conf_wakeup( &sock->ssl_fake, 0 ); conf_wakeup( &sock->ssl_fake, 0 );
} else } else
@ -532,15 +587,71 @@ socket_fill( conn_t *sock )
if ((n = read( sock->fd, buf, len )) < 0) { if ((n = read( sock->fd, buf, len )) < 0) {
sys_error( "Socket error: read from %s", sock->name ); sys_error( "Socket error: read from %s", sock->name );
socket_fail( sock ); socket_fail( sock );
return;
} else if (!n) { } else if (!n) {
error( "Socket error: read from %s: unexpected EOF\n", sock->name ); error( "Socket error: read from %s: unexpected EOF\n", sock->name );
socket_fail( sock ); socket_fail( sock );
return; return -1;
} }
} }
sock->bytes += n;
sock->read_callback( sock->callback_aux ); return n;
}
#ifdef HAVE_LIBZ
static void
socket_fill_z( conn_t *sock )
{
char *buf;
int len;
if (prepare_read( sock, &buf, &len ) < 0)
return;
sock->in_z->avail_out = len;
sock->in_z->next_out = (unsigned char *)buf;
if (inflate( sock->in_z, Z_SYNC_FLUSH ) != Z_OK) {
error( "Error decompressing data from %s: %s\n", sock->name, sock->in_z->msg );
socket_fail( sock );
return;
}
if (!sock->in_z->avail_out)
conf_wakeup( &sock->z_fake, 0 );
if ((len = (char *)sock->in_z->next_out - buf)) {
sock->bytes += len;
sock->read_callback( sock->callback_aux );
}
}
#endif
static void
socket_fill( conn_t *sock )
{
#ifdef HAVE_LIBZ
if (sock->in_z) {
/* The timer will preempt reads until the buffer is empty. */
assert( !sock->in_z->avail_in );
sock->in_z->next_in = (uchar *)sock->z_buf;
if ((sock->in_z->avail_in = do_read( sock, sock->z_buf, sizeof(sock->z_buf) )) <= 0)
return;
socket_fill_z( sock );
} else
#endif
{
char *buf;
int len;
if (prepare_read( sock, &buf, &len ) < 0)
return;
if ((len = do_read( sock, buf, len )) <= 0)
return;
sock->bytes += len;
sock->read_callback( sock->callback_aux );
}
} }
int int
@ -655,6 +766,49 @@ do_append( conn_t *conn, buff_chunk_t *bc )
* sufficiently small to keep SSL latency low with a slow uplink. */ * sufficiently small to keep SSL latency low with a slow uplink. */
#define WRITE_CHUNK_SIZE 1024 #define WRITE_CHUNK_SIZE 1024
static void
do_flush( conn_t *conn )
{
buff_chunk_t *bc = conn->append_buf;
#ifdef HAVE_LIBZ
if (conn->out_z) {
int buf_avail = conn->append_avail;
do {
if (!bc) {
buf_avail = WRITE_CHUNK_SIZE;
bc = nfmalloc( offsetof(buff_chunk_t, data) + buf_avail );
bc->len = 0;
}
conn->out_z->next_in = Z_NULL;
conn->out_z->avail_in = 0;
conn->out_z->next_out = (uchar *)bc->data + bc->len;
conn->out_z->avail_out = buf_avail;
if (deflate( conn->out_z, Z_PARTIAL_FLUSH ) != Z_OK) {
error( "Fatal: Compression error: %s\n", conn->out_z->msg );
abort();
}
bc->len = (char *)conn->out_z->next_out - bc->data;
if (bc->len) {
do_append( conn, bc );
bc = 0;
buf_avail = 0;
} else {
buf_avail = conn->out_z->avail_out;
}
} while (!conn->out_z->avail_out);
conn->append_buf = bc;
conn->append_avail = buf_avail;
} else
#endif
if (bc) {
do_append( conn, bc );
conn->append_buf = 0;
#ifdef HAVE_LIBZ
conn->append_avail = 0;
#endif
}
}
int int
socket_write( conn_t *conn, conn_iovec_t *iov, int iovcnt ) socket_write( conn_t *conn, conn_iovec_t *iov, int iovcnt )
{ {
@ -663,29 +817,54 @@ socket_write( conn_t *conn, conn_iovec_t *iov, int iovcnt )
for (i = 0; i < iovcnt; i++) for (i = 0; i < iovcnt; i++)
total += iov[i].len; total += iov[i].len;
bc = conn->append_buf; if (total >= WRITE_CHUNK_SIZE) {
if (bc && total >= WRITE_CHUNK_SIZE) {
/* If the new data is too big, queue the pending buffer to avoid latency. */ /* If the new data is too big, queue the pending buffer to avoid latency. */
do_append( conn, bc ); do_flush( conn );
bc = 0;
} }
bc = conn->append_buf;
#ifdef HAVE_LIBZ
buf_avail = conn->append_avail;
#endif
while (total) { while (total) {
if (!bc) { if (!bc) {
/* We don't do anything special when compressing, as there is no way to
* predict a reasonable output buffer size anyway - deflatePending() does
* not account for consumed but not yet compressed input, and adding up
* the deflateBound()s would be a tad *too* pessimistic. */
buf_avail = total > WRITE_CHUNK_SIZE ? total : WRITE_CHUNK_SIZE; buf_avail = total > WRITE_CHUNK_SIZE ? total : WRITE_CHUNK_SIZE;
bc = nfmalloc( offsetof(buff_chunk_t, data) + buf_avail ); bc = nfmalloc( offsetof(buff_chunk_t, data) + buf_avail );
bc->len = 0; bc->len = 0;
#ifndef HAVE_LIBZ
} else { } else {
/* A pending buffer will always be of standard size - over-sized /* A pending buffer will always be of standard size - over-sized
* buffers are immediately filled and queued. */ * buffers are immediately filled and queued. */
buf_avail = WRITE_CHUNK_SIZE - bc->len; buf_avail = WRITE_CHUNK_SIZE - bc->len;
#endif
} }
while (total) { while (total) {
len = iov->len - offset; len = iov->len - offset;
if (len > buf_avail) #ifdef HAVE_LIBZ
len = buf_avail; if (conn->out_z) {
memcpy( bc->data + bc->len, iov->buf + offset, len ); conn->out_z->next_in = (uchar *)iov->buf + offset;
bc->len += len; conn->out_z->avail_in = len;
buf_avail -= len; conn->out_z->next_out = (uchar *)bc->data + bc->len;
conn->out_z->avail_out = buf_avail;
if (deflate( conn->out_z, Z_NO_FLUSH ) != Z_OK) {
error( "Fatal: Compression error: %s\n", conn->out_z->msg );
abort();
}
bc->len = (char *)conn->out_z->next_out - bc->data;
buf_avail = conn->out_z->avail_out;
len -= conn->out_z->avail_in;
} else
#endif
{
if (len > buf_avail)
len = buf_avail;
memcpy( bc->data + bc->len, iov->buf + offset, len );
bc->len += len;
buf_avail -= len;
}
offset += len; offset += len;
total -= len; total -= len;
if (offset == iov->len) { if (offset == iov->len) {
@ -702,8 +881,16 @@ socket_write( conn_t *conn, conn_iovec_t *iov, int iovcnt )
} }
} }
conn->append_buf = bc; conn->append_buf = bc;
#ifdef HAVE_LIBZ
conn->append_avail = buf_avail;
#endif
/* Queue the pending write once the main loop goes idle. */ /* Queue the pending write once the main loop goes idle. */
conf_wakeup( &conn->fd_fake, bc ? 0 : -1 ); conf_wakeup( &conn->fd_fake,
#ifdef HAVE_LIBZ
/* Always give zlib a chance to flush its internal buffer. */
conn->out_z ||
#endif
bc ? 0 : -1 );
/* If no writes were queued before, ensure that flushing commences. */ /* If no writes were queued before, ensure that flushing commences. */
if (!exwb) if (!exwb)
return do_queued_write( conn ); return do_queued_write( conn );
@ -763,13 +950,22 @@ socket_fake_cb( void *aux )
conn_t *conn = (conn_t *)aux; conn_t *conn = (conn_t *)aux;
buff_chunk_t *exwb = conn->write_buf; buff_chunk_t *exwb = conn->write_buf;
do_append( conn, conn->append_buf ); do_flush( conn );
conn->append_buf = 0;
/* If no writes were queued before, ensure that flushing commences. */ /* If no writes were queued before, ensure that flushing commences. */
if (!exwb) if (!exwb)
do_queued_write( conn ); do_queued_write( conn );
} }
#ifdef HAVE_LIBZ
static void
z_fake_cb( void *aux )
{
conn_t *conn = (conn_t *)aux;
socket_fill_z( conn );
}
#endif
#ifdef HAVE_LIBSSL #ifdef HAVE_LIBSSL
static void static void
ssl_fake_cb( void *aux ) ssl_fake_cb( void *aux )

View File

@ -25,6 +25,10 @@
#include "common.h" #include "common.h"
#ifdef HAVE_LIBZ
#include <zlib.h>
#endif
#ifdef HAVE_LIBSSL #ifdef HAVE_LIBSSL
typedef struct ssl_st SSL; typedef struct ssl_st SSL;
typedef struct ssl_ctx_st SSL_CTX; typedef struct ssl_ctx_st SSL_CTX;
@ -76,6 +80,10 @@ typedef struct {
SSL *ssl; SSL *ssl;
wakeup_t ssl_fake; wakeup_t ssl_fake;
#endif #endif
#ifdef HAVE_LIBZ
z_streamp in_z, out_z;
wakeup_t z_fake;
#endif
void (*bad_callback)( void *aux ); /* async fail while sending or listening */ void (*bad_callback)( void *aux ); /* async fail while sending or listening */
void (*read_callback)( void *aux ); /* data available for reading */ void (*read_callback)( void *aux ); /* data available for reading */
@ -92,6 +100,9 @@ typedef struct {
/* writing */ /* writing */
buff_chunk_t *append_buf; /* accumulating buffer */ buff_chunk_t *append_buf; /* accumulating buffer */
buff_chunk_t *write_buf, **write_buf_append; /* buffer head & tail */ buff_chunk_t *write_buf, **write_buf_append; /* buffer head & tail */
#ifdef HAVE_LIBZ
int append_avail; /* space left in accumulating buffer */
#endif
int write_offset; /* offset into buffer head */ int write_offset; /* offset into buffer head */
/* reading */ /* reading */
@ -99,6 +110,9 @@ typedef struct {
int bytes; /* number of filled bytes in buffer */ int bytes; /* number of filled bytes in buffer */
int scanoff; /* offset to continue scanning for newline at, relative to 'offset' */ int scanoff; /* offset to continue scanning for newline at, relative to 'offset' */
char buf[100000]; char buf[100000];
#ifdef HAVE_LIBZ
char z_buf[100000];
#endif
} conn_t; } conn_t;
/* call this before doing anything with the socket */ /* call this before doing anything with the socket */
@ -120,6 +134,7 @@ static INLINE void socket_init( conn_t *conn,
} }
void socket_connect( conn_t *conn, void (*cb)( int ok, void *aux ) ); void socket_connect( conn_t *conn, void (*cb)( int ok, void *aux ) );
void socket_start_tls(conn_t *conn, void (*cb)( int ok, void *aux ) ); void socket_start_tls(conn_t *conn, void (*cb)( int ok, void *aux ) );
void socket_start_deflate( conn_t *conn );
void socket_close( conn_t *sock ); void socket_close( conn_t *sock );
int socket_read( conn_t *sock, char *buf, int len ); /* never waits */ int socket_read( conn_t *sock, char *buf, int len ); /* never waits */
char *socket_read_line( conn_t *sock ); /* don't free return value; never waits */ char *socket_read_line( conn_t *sock ); /* don't free return value; never waits */