document defaults of referenced XDG env vars

so users don't have to google the spec.

.gitignore

@@ -14,6 +14,7 @@
 /config.status
 /config.sub
 /configure
+/configure~
 /configure.lineno
 /configure-stamp
 /cov-int/
@@ -22,11 +23,13 @@
 /isync.spec
 /isync-*.tar.gz
 /isync-*.tar.gz.asc
+/isync-cov.tar.xz
 /missing
 /patch-stamp
 /stamp-h
 /stamp-h.in
 /stamp-h1
+/test-driver
 
 Makefile
 Makefile.in

AUTHORS

@@ -1,17 +1,101 @@
-Oswald Buddenhagen <ossi@users.sf.net>
-	* Contributor, current maintainer
-
-Theodore Ts'o <tytso@mit.edu>
-	* Contributor, Debian package co-maintainer
-
-Nicolas Boullis <nboullis@debian.org>
-	* Debian package maintainer and minor upstream contributions
-
-Michael Elkins <me@mutt.org>
-	* Original author
+Contact
+=======
 
 Send questions and bug reports to the isync-devel@lists.sourceforge.net
 mailing list.
 
-_DON'T_ report bugs to Michael, not even in a CC: - he is not actively
+Do _NOT_ report bugs to Michael, not even in a CC: - he is not actively
 involved in isync development any more.
+
+Lead Developers
+===============
+
+Oswald Buddenhagen <ossi@users.sf.net>
+- Current maintainer
+
+Michael Elkins <me@mutt.org>
+- Original author
+
+Contributors
+============
+
+(Some of these people also contributed bugfixes and optimizations.)
+(In chronological order.)
+
+Jeremy Katz <katzj@linuxpower.org>
+- UseNamespace & UseSSL* options
+
+Daniel Resare <noa@metamatrix.se>
+- Numerous SSL handling improvements
+
+Eivind Eklund <eivind@FreeBSD.org>
+- MaxMessages option
+
+Theodore Ts'o <tytso@mit.edu>
+- get-cert script
+- Maildir UID mapping improvements
+- Initial version of partial async IMAP support
+
+Marc Hoersken <info@marc-hoersken.de>
+- CopyArrivalDate option
+
+Jack Stone <jwjstone@fastmail.fm>
+Jan Synacek <jsynacek@redhat.com>
+- SASL support
+
+Jesse Weaver <pianohacker@gmail.com>
+- IMAP stream compression support
+
+Anton Khirnov <anton@khirnov.net>
+- ClientKey & ClientCertificate options
+
+Michael J Gruber <github@grubix.eu>
+- Support for the $Forwarded/Passed flag
+
+Patrick Steinhardt <ps@pks.im>
+- UserCmd option
+
+Oliver Runge <oliver.runge@gmail.com>
+- UseKeychain option
+
+Georgy Kibardin <georgy@kibardin.name>
+- Support for UTF-7 IMAP mailbox names
+
+Honorary Contributors
+=====================
+
+(These people contributed patches that were too small or obvious
+ to claim copyright, or were rewritten from scratch.)
+(In alphabetical order.)
+
+Alessandro Ghedini <ghedo@debian.org>
+Andreas Grapentin <andreas@grapentin.org>
+Aurélien Francillon <aurelien.francillon@eurecom.fr>
+Ben Kibbey <bjk@luxsci.net>
+Caspar Schutijser <caspar@schutijser.com>
+Cedric Ware <cedric.ware__bml@normalesup.org>
+Dmitrij D. Czarkoff <czarkoff@gmail.com>
+Dmitry Torokhov <dtor@chromium.org>
+Felipe Contreras <felipe.contreras@gmail.com>
+Felix Janda <felix.janda@posteo.de>
+Gergely Risko <gergely@risko.hu>
+Sung Pae "guns" <self@sungpae.com>
+Helmut Grohne <helmut@subdivi.de>
+Hugo Haas <hugo@larve.net>
+Jaroslav Suchanek <jaroslav.suchanek@gmail.com>
+Jeremie Courreges-Anglas <jca@openbsd.org>
+Klemens Nanni <kn@openbsd.org>
+Lorenzo Martignoni <lorenzo.martignoni@technologist.com>
+Magnus Jonsson <bigfoot@acc.umu.se>
+Marcin Niestroj <macius1990w@gmail.com>
+Martin Stenberg <martin@gnutiken.se>
+Mike Delaney <mdelan@lusars.net>
+Nicolas Boullis <nboullis@debian.org>
+Nihal Jere <nihal@nihaljere.xyz>
+Reimar Döffinger <Reimar.Doeffinger@gmx.de>
+Remko Tronçon <remko@el-tramo.be>
+sbfnk@users.sf.net
+Thomas Roessler <roessler@does-not-exist.org>
+Todd T. Fries <todd@fries.net>
+Vincent Bernat <vincent@bernat.ch>
+Yuri D'Elia <wavexx@thregr.org>

COPYING

@@ -1,340 +0,0 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-     59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-			    Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-			    NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-	    How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year  name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.

COPYING

@@ -0,0 +1 @@
+LICENSES/GPL-2.0-or-later.txt

LICENSES/GPL-2.0-or-later.txt

@@ -0,0 +1,359 @@
+Valid-License-Identifier: GPL-2.0
+Valid-License-Identifier: GPL-2.0-only
+Valid-License-Identifier: GPL-2.0+
+Valid-License-Identifier: GPL-2.0-or-later
+SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU General Public License (GPL) version 2 only' use:
+    SPDX-License-Identifier: GPL-2.0
+  or
+    SPDX-License-Identifier: GPL-2.0-only
+  For 'GNU General Public License (GPL) version 2 or any later version' use:
+    SPDX-License-Identifier: GPL-2.0+
+  or
+    SPDX-License-Identifier: GPL-2.0-or-later
+License-Text:
+
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                       51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.

LICENSES/LicenseRef-isync-GPL-exception.txt

@@ -0,0 +1,12 @@
+SPDX-Exception-Identifier: LicenseRef-isync-GPL-exception
+SPDX-Licenses: GPL-2.0-or-later
+Usage-Guide:
+  This exception is used together with the above SPDX-License to
+  allow linking the compiled version of code to non GPL compliant code.
+  To use this exception add it with the keyword WITH to one of the
+  identifiers in the SPDX-Licenses tag:
+    SPDX-License-Identifier: <SPDX-License> WITH LicenseRef-isync-GPL-exception
+License-Text:
+
+As a special exception, mbsync may be linked with the OpenSSL library,
+despite that library's more restrictive license.

Makefile.am

@@ -1,6 +1,10 @@
+# SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+# SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+# SPDX-License-Identifier: GPL-2.0-or-later
+
 SUBDIRS = src
 bin_SCRIPTS = mbsync-get-cert
-EXTRA_DIST = debian isync.spec $(bin_SCRIPTS)
+EXTRA_DIST = LICENSES debian isync.spec $(bin_SCRIPTS)
 
 LOG_PL = \
     use POSIX qw(strftime); \

NEWS

@@ -1,3 +1,41 @@
+[1.5.0]
+
+Changed default config & state locations to follow the XDG basedir spec.
+The old locations remain supported.
+
+The reference point for relative local paths in the configuration file
+is now the file's containing directory.
+
+Placeholders will be now created for messages exceeding MaxSize even if
+they are flagged on the source side.
+
+Renamed the ReNew/--renew/-N options to Upgrade/--upgrade/-u
+and Delete/--delete/-d to Gone/--gone/-g.
+
+Superseded SSLVersions option with TLSVersions, and disabled TLS v1.0
+and v1.1 by default. Renamed SSLType option to TLSType.
+
+Made the Channel side to expire with MaxMessages configurable.
+
+MaxMessages and MaxSize can be used together now.
+
+Added support for IMAP mailbox names with non-ASCII characters.
+
+Added support for Maildir Paths with suffixes.
+
+The unfiltered list of mailboxes in each Store can be printed now.
+
+A proper summary is now printed prior to exiting.
+This includes expunges, which are now included in the progress as well.
+
+Added new sync operation 'Old'.
+
+Added support for mirroring deletions more accurately.
+
+Added --dry-run option.
+
+Added --ext-exit option.
+
 [1.4.0]
 
 The 'isync' compatibility wrapper was removed.

TODO

@@ -1,22 +1,23 @@
 f{,data}sync() usage could be optimized by batching the calls.
 
-make SSL (connect) timeouts produce a bit more than "Unidentified socket error".
-
 automatically resume upon transient errors, e.g. "connection reset by peer"
 or timeout after some data was already transmitted.
 possibly also try to handle Exchange's "glitches" somehow.
 
-add support for IMAP UTF-7 (for internationalized mailbox names).
-
 uidvalidity lock timeout handling would be a good idea.
 
 should complain when multiple Channels match the same folders.
 
+should complain about nonsensical combinations like Sync Pull + Create Both.
+
 propagate folder deletions even when the folders are non-empty.
 - verify that "most" of the folders in the Channel are still there.
 - refuse to delete unpropagated messages when trashing on the remote side.
 - refuse to delete far side if it has unpropagated messages. symmetry?
 
+add option to suppress complaints about folders that would need creation
+(but not deleted ones).
+
 add message expiration based on arrival date (message date would be too
 unreliable). MaxAge; probably mutually exclusive to MaxMessages.
 
@@ -25,17 +26,16 @@ add alternative treatments of expired messages. ExpiredMessageMode: Prune
 separate folder - ArchiveSuffix, default .archive).
 
 add support for event notification callbacks.
-it would be also possible to report more differentiated exit codes, but
-that seems too limiting in the general case.
 
 make it possible to have different mailbox names for far and near side in
 Patterns.
 - use far:near for the pattern
-  - for quoting, use more colons: the longest sequence of colons is the
-    separator
+  - supporting names with colons requires and extension of the parser to
+    report which parts of an argument were quoted
 - this makes Groups mostly useless, as they are mostly a workaround for this
   function being missing so far
 - this is needed for move detection, which would work only within one Channel
+- this supersedes MapInbox
 
 add regexp-based mailbox path rewriting to the drivers. user would provide
 expressions for both directions. every transformation would be immediately
@@ -48,18 +48,22 @@ also: idling mode.
 parallel fetching of multiple mailboxes.
 TLS session resumption becomes interesting then as well.
 
-imap_set_flags(): group commands for efficiency, don't call back until
-imap_commit().
+imap_set_msg_flags() & imap_trash_msg(): group commands for efficiency.
 
 add streaming from fetching to storing.
+this is complicated by the IMAP target needing the final size in advance,
+which we can't know in a single pass when newline translation is necessary.
 
 handle custom flags (keywords).
+this is impeded by there being no Maildir-side standard.
 
 make use of IMAP CONDSTORE extension (rfc4551; CHANGEDSINCE FETCH Modifier);
 make use of IMAP QRESYNC extension (rfc5162) to avoid SEARCH to find vanished
 messages.
 
-use MULTIAPPEND and FETCH with multiple messages.
+make use of IMAP CAPABILITY APPENDLIMIT= extension (rfc7889; fastmail & gmail).
+this is really useful only for IMAP <=> IMAP syncs: saves FETCH BODY.
+the message could still become oversized due to conversion.
 
 dummy messages resulting from MaxSize should contain a dump of the original
 message's MIME structure and its (reasonably sized) text parts.

configure.ac

@@ -1,4 +1,8 @@
-AC_INIT([isync], [1.4.4])
+# SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+# SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+dnl SPDX-License-Identifier: GPL-2.0-or-later
+
+AC_INIT([isync], [1.5.0])
 AC_CONFIG_HEADERS([autodefs.h])
 
 AC_CANONICAL_TARGET
@@ -79,7 +83,7 @@ if test "x$ob_cv_strftime_z" = x"no"; then
 fi
 
 AC_CHECK_HEADERS(poll.h sys/select.h)
-AC_CHECK_FUNCS(vasprintf strnlen memrchr timegm)
+AC_CHECK_FUNCS(vasprintf strnlen memrchr timegm fwrite_unlocked)
 
 AC_CHECK_LIB(socket, socket, [SOCK_LIBS="-lsocket"])
 AC_CHECK_LIB(nsl, inet_ntoa, [SOCK_LIBS="$SOCK_LIBS -lnsl"])

debian/copyright

@@ -4,7 +4,7 @@ Source: http://isync.sourceforge.net
 
 Files: *
 Copyright: 2000-2002, Michael R. Elkins <me@mutt.org>
- 2002-2017, Oswald Buddenhagen <ossi@users.sf.net>
+ 2002-2022, Oswald Buddenhagen <ossi@users.sf.net>
  2004, Theodore Y. Ts'o <tytso@mit.edu>
 License: GPL-2+
 

mbsync-get-cert

@@ -1,25 +1,13 @@
 #!/bin/sh
 #
+# SPDX-FileCopyrightText: 2003 Theodore Ts'o <tytso@alum.mit.edu>
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
 # This script will extract the necessary certificate from the IMAP server
 # It assumes that an attacker isn't trying to spoof you when you connect
 # to the IMAP server!  You're better off downloading the certificate
 # from a trusted source.
 #
-# Copyright (C) 2003 Theodore Ts'o <tytso@alum.mit.edu>
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
-#
 
 if [ $# != 1 ]; then
 	echo "Usage: $0 <host>" >&2

src/.gitignore

@@ -1,8 +1,14 @@
+/*_enum.h
 /drv_proxy.inc
 /mbsync
 /mdconvert
+/tst_imap_msgs
+/tst_imap_utf7
+/tst_msg_cvt
 /tst_timers
-/tmp/
+/tmp
 
 .deps/
+*.log
 *.o
+*.trs

src/Makefile.am

@@ -1,11 +1,47 @@
-mbsync_SOURCES = main.c sync.c config.c util.c socket.c driver.c drv_imap.c drv_maildir.c drv_proxy.c
+# SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+# SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+mbsync_SOURCES = \
+	util.c config.c socket.c \
+	driver.c drv_proxy.c \
+	drv_imap.c imap_msgs.c imap_utf7.c \
+	drv_maildir.c \
+	sync.c sync_state.c sync_msg_cvt.c \
+	main.c main_sync.c main_list.c
+noinst_HEADERS = \
+	common.h config.h socket.h \
+	driver.h imap_p.h \
+	sync.h sync_p.h \
+	main_p.h
 mbsync_LDADD = $(DB_LIBS) $(SSL_LIBS) $(SOCK_LIBS) $(SASL_LIBS) $(Z_LIBS) $(KEYCHAIN_LIBS)
-noinst_HEADERS = common.h config.h driver.h sync.h socket.h
 
 drv_proxy.$(OBJEXT): drv_proxy.inc
 drv_proxy.inc: $(srcdir)/driver.h $(srcdir)/drv_proxy.c $(srcdir)/drv_proxy_gen.pl
 	perl $(srcdir)/drv_proxy_gen.pl $(srcdir)/driver.h $(srcdir)/drv_proxy.c drv_proxy.inc
 
+ENUM_GEN = $(srcdir)/bit_enum_gen.pl
+
+$(mbsync_OBJECTS): common_enum.h
+common_enum.h: common.h $(ENUM_GEN)
+	perl $(ENUM_GEN) < $< > $@
+
+$(mbsync_OBJECTS): driver_enum.h
+driver_enum.h: driver.h $(ENUM_GEN)
+	perl $(ENUM_GEN) < $< > $@
+
+$(mbsync_OBJECTS): sync_enum.h
+sync_enum.h: sync.h $(ENUM_GEN)
+	perl $(ENUM_GEN) < $< > $@
+
+sync.$(OBJEXT): sync_c_enum.h
+sync_c_enum.h: sync.c $(ENUM_GEN)
+	perl $(ENUM_GEN) < $< > $@
+
+sync.$(OBJEXT) sync_state.$(OBJEXT): sync_p_enum.h
+sync_p_enum.h: sync_p.h $(ENUM_GEN)
+	perl $(ENUM_GEN) < $< > $@
+
 mdconvert_SOURCES = mdconvert.c
 mdconvert_LDADD = $(DB_LIBS)
 if with_mdconvert
@@ -13,16 +49,26 @@ mdconvert_prog = mdconvert
 mdconvert_man = mdconvert.1
 endif
 
-EXTRA_PROGRAMS = tst_timers
+bin_PROGRAMS = mbsync $(mdconvert_prog)
+man_MANS = mbsync.1 $(mdconvert_man)
+
+tst_imap_msgs_SOURCES = tst_imap_msgs.c imap_msgs.c util.c
+
+tst_imap_utf7_SOURCES = tst_imap_utf7.c imap_utf7.c util.c
+
+tst_msg_cvt_SOURCES = tst_msg_cvt.c sync_msg_cvt.c util.c
+tst_msg_cvt_CFLAGS = -DQPRINTF_BUFF=10000
+
+check_PROGRAMS = tst_imap_msgs tst_imap_utf7 tst_msg_cvt
+TESTS = $(check_PROGRAMS)
 
 tst_timers_SOURCES = tst_timers.c util.c
 
-bin_PROGRAMS = mbsync $(mdconvert_prog)
-man_MANS = mbsync.1 $(mdconvert_man)
+EXTRA_PROGRAMS = tst_timers
 
 exampledir = $(docdir)/examples
 example_DATA = mbsyncrc.sample
 
-EXTRA_DIST = drv_proxy_gen.pl run-tests.pl $(example_DATA) $(man_MANS)
+EXTRA_DIST = bit_enum_gen.pl drv_proxy_gen.pl run-tests.pl $(example_DATA) $(man_MANS)
 
-CLEANFILES = drv_proxy.inc
+CLEANFILES = *_enum.h drv_proxy.inc

src/bit_enum_gen.pl

@@ -0,0 +1,70 @@
+#!/usr/bin/perl
+#
+# SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# mbsync - mailbox synchronizer
+#
+
+use strict;
+use warnings;
+
+my $in_enum = 0;
+my $conts;
+while (<>) {
+	s,\s*(?://.*)?$,,;
+	if ($in_enum) {
+		if (/^\)$/) {
+			$conts =~ s/\s//g;
+			$conts =~ s/,$//;
+			my @vals = split(/,/, $conts);
+			my ($pfx, $pfx1);
+			for my $e (@vals) {
+				if (!defined($pfx)) {
+					$pfx1 = $pfx = ($e =~ /^([A-Z]+_)/) ? $1 : "";
+				} elsif (length($pfx)) {
+					$pfx = "" if ((($e =~ /^([A-Z]+_)/) ? $1 : "") ne $pfx);
+				}
+			}
+			my $bit = 1;
+			my $bitn = 0;
+			my (@names, @nameos);
+			my $nameo = 0;
+			for my $e (@vals) {
+				my $bits = ($e =~ s/\((\d+)\)$//) ? $1 : 1;
+				my $n = substr($e, length($pfx));
+				if ($bits != 1) {
+					die("Unsupported field size $bits\n") if ($bits != 2);
+					print "#define $e(b) ($bit << (b))\n";
+					push @names, "F-".$n, "N-".$n;
+					my $nl = length($n) + 3;
+					push @nameos, $nameo, $nameo + $nl;
+					$nameo += $nl * 2;
+				} else {
+					print "#define $e $bit\n";
+					push @names, $n;
+					push @nameos, $nameo;
+					$nameo += length($n) + 1;
+				}
+				$bit <<= $bits;
+				$bitn += $bits;
+			}
+			if (length($pfx)) {
+				print "#define ${pfx}_NUM_BITS $bitn\n";
+			}
+			if (length($pfx1)) {
+				print "#define ${pfx1}_STRINGS \"".join("\\0", @names)."\"\n";
+				print "#define ${pfx1}_OFFSETS ".join(", ", @nameos)."\n";
+			}
+			print "\n";
+			$in_enum = 0;
+		} else {
+			$conts .= $_;
+		}
+	} else {
+		if (/^BIT_ENUM\($/) {
+			$conts = "";
+			$in_enum = 1;
+		}
+	}
+}

src/common.h

@@ -1,23 +1,8 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #ifndef COMMON_H
@@ -26,10 +11,18 @@
 #include <autodefs.h>
 
 #include <sys/types.h>
+#include <assert.h>
+#include <limits.h>
 #include <stdarg.h>
+#include <stddef.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
 #include <time.h>
 
+#include "common_enum.h"
+
 typedef unsigned char uchar;
 typedef unsigned short ushort;
 typedef unsigned int uint;
@@ -50,14 +43,22 @@ typedef unsigned long ulong;
 #define shifted_bit(in, from, to) \
 	((int)(((uint)(in) / (from > to ? from / to : 1) * (to > from ? to / from : 1)) & to))
 
+#define BIT_ENUM(...)
+
+#define static_assert_bits(pfx, type, field) \
+	static_assert( pfx##__NUM_BITS <= sizeof(((type){ 0 }).field) * 8, \
+	               stringify(type) "::" stringify(field) " is too small" )
+
 #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
 # define ATTR_UNUSED __attribute__((unused))
 # define ATTR_NORETURN __attribute__((noreturn))
 # define ATTR_PRINTFLIKE(fmt,var) __attribute__((format(printf,fmt,var)))
+# define ATTR_OPTIMIZE __attribute__((optimize("2")))
 #else
 # define ATTR_UNUSED
 # define ATTR_NORETURN
 # define ATTR_PRINTFLIKE(fmt,var)
+# define ATTR_OPTIMIZE
 #endif
 
 #if defined(__clang__)
@@ -92,44 +93,72 @@ typedef unsigned long ulong;
 
 /* main.c */
 
-#define DEBUG_CRASH     0x01
-#define DEBUG_MAILDIR   0x02
-#define DEBUG_NET       0x04
-#define DEBUG_NET_ALL   0x08
-#define DEBUG_SYNC      0x10
-#define DEBUG_MAIN      0x20
-#define DEBUG_DRV       0x40
-#define DEBUG_DRV_ALL   0x80
-#define DEBUG_ALL       (0xFF & ~(DEBUG_NET_ALL | DEBUG_DRV_ALL))
-#define QUIET           0x100
-#define VERYQUIET       0x200
-#define PROGRESS        0x400
-#define VERBOSE         0x800
-#define KEEPJOURNAL     0x1000
-#define ZERODELAY       0x2000
-#define FORCEASYNC      0x4000
+enum {
+	VERYQUIET,
+	QUIET,
+	TERSE,
+	VERBOSE,
+};
 
+BIT_ENUM(
+	DEBUG_MAILDIR,
+	DEBUG_NET,
+	DEBUG_NET_ALL,
+	DEBUG_SYNC,
+	DEBUG_MAIN,
+	DEBUG_DRV,
+	DEBUG_DRV_ALL,
+
+	DEBUG_CRASH,
+
+	PROGRESS,
+
+	DRYRUN,
+
+	EXT_EXIT,
+
+	ZERODELAY,
+	KEEPJOURNAL,
+	FORCEJOURNAL,
+	FORCEASYNC(2),
+	FAKEEXPUNGE,
+	FAKEDUMBSTORE,
+)
+
+#define DEBUG_ANY (DEBUG_MAILDIR | DEBUG_NET | DEBUG_SYNC | DEBUG_MAIN | DEBUG_DRV)
+#define DEBUG_ALL (DEBUG_ANY | DEBUG_CRASH)
+
+// Global options
+extern int Verbosity;
 extern int DFlags;
-extern int JLimit;
+extern int JLimit, JCount;
 extern int UseFSync;
-extern char FieldDelimiter;
 
+// Global constants (inited by main())
 extern int Pid;
 extern char Hostname[256];
 extern const char *Home;
 
-extern uint BufferLimit;
-
-extern int new_total[2], new_done[2];
-extern int flags_total[2], flags_done[2];
-extern int trash_total[2], trash_done[2];
-
+void countStep( void );
 void stats( void );
 
 /* util.c */
 
-void ATTR_PRINTFLIKE(2, 0) vdebug( int, const char *, va_list va );
-void ATTR_PRINTFLIKE(2, 0) vdebugn( int, const char *, va_list va );
+#ifdef DEBUG_FLAG
+#  define debug(...) \
+	do { \
+		if (DFlags & DEBUG_FLAG) \
+			print( __VA_ARGS__ ); \
+	} while (0)
+#  define debugn(...) \
+	do { \
+		if (DFlags & DEBUG_FLAG) \
+			printn( __VA_ARGS__ ); \
+	} while (0)
+#endif
+
+void ATTR_PRINTFLIKE(1, 2) print( const char *, ... );
+void ATTR_PRINTFLIKE(1, 2) printn( const char *, ... );
 void ATTR_PRINTFLIKE(1, 2) info( const char *, ... );
 void ATTR_PRINTFLIKE(1, 2) infon( const char *, ... );
 void ATTR_PRINTFLIKE(1, 2) progress( const char *, ... );
@@ -140,6 +169,17 @@ void ATTR_PRINTFLIKE(1, 0) vsys_error( const char *, va_list va );
 void ATTR_PRINTFLIKE(1, 2) sys_error( const char *, ... );
 void flushn( void );
 
+char *xvasprintf( const char *fmt, va_list ap );
+void xprintf( const char *fmt, ... );
+
+#if !defined(_POSIX_SYNCHRONIZED_IO) || _POSIX_SYNCHRONIZED_IO <= 0
+# define fdatasync fsync
+#endif
+
+void ATTR_PRINTFLIKE(2, 0) vFprintf( FILE *f, const char *msg, va_list va );
+void ATTR_PRINTFLIKE(2, 3) Fprintf( FILE *f, const char *msg, ... );
+void Fclose( FILE *f, int safe );
+
 typedef struct string_list {
 	struct string_list *next;
 	char string[1];
@@ -156,16 +196,50 @@ void *memrchr( const void *s, int c, size_t n );
 size_t strnlen( const char *str, size_t maxlen );
 #endif
 
+void to_upper( char *str, uint len );
+
 int starts_with( const char *str, int strl, const char *cmp, uint cmpl );
 int starts_with_upper( const char *str, int strl, const char *cmp, uint cmpl );
 int equals( const char *str, int strl, const char *cmp, uint cmpl );
+int equals_upper( const char *str, int strl, const char *cmp, uint cmpl );
 
 #ifndef HAVE_TIMEGM
 time_t timegm( struct tm *tm );
 #endif
 
+void fmt_bits( uint bits, uint num_bits, const char *bit_str, const int *bit_off, char *buf );
+
+#define BIT_FORMATTER_RET(name, pfx) \
+	struct name##_str { char str[sizeof(pfx##__STRINGS)]; };
+
+#define BIT_FORMATTER_PROTO(name, pfx, storage) \
+	storage struct name##_str ATTR_OPTIMIZE  /* force RVO */ \
+	fmt_##name( uint bits )
+
+#define BIT_FORMATTER_IMPL(name, pfx, storage) \
+	BIT_FORMATTER_PROTO(name, pfx, storage) \
+	{ \
+		static const char strings[] = pfx##__STRINGS; \
+		static const int offsets[] = { pfx##__OFFSETS }; \
+		\
+		struct name##_str buf; \
+		fmt_bits( bits, as(offsets), strings, offsets, buf.str ); \
+		return buf; \
+	}
+
+#define BIT_FORMATTER_FUNCTION(name, pfx) \
+	BIT_FORMATTER_RET(name, pfx) \
+	BIT_FORMATTER_IMPL(name, pfx, static)
+
+#define DECL_BIT_FORMATTER_FUNCTION(name, pfx) \
+	BIT_FORMATTER_RET(name, pfx) \
+	BIT_FORMATTER_PROTO(name, pfx, );
+
+#define DEF_BIT_FORMATTER_FUNCTION(name, pfx) \
+	BIT_FORMATTER_IMPL(name, pfx, )
+
 void *nfmalloc( size_t sz );
-void *nfcalloc( size_t sz );
+void *nfzalloc( size_t sz );
 void *nfrealloc( void *mem, size_t sz );
 char *nfstrndup( const char *str, size_t nchars );
 char *nfstrdup( const char *str );
@@ -175,9 +249,9 @@ int ATTR_PRINTFLIKE(3, 4) nfsnprintf( char *buf, int blen, const char *fmt, ...
 void ATTR_NORETURN oob( void );
 void ATTR_NORETURN oom( void );
 
-char *expand_strdup( const char *s );
+int map_name( const char *arg, int argl, char **result, uint reserve, const char *in, const char *out );
 
-int map_name( const char *arg, char **result, uint reserve, const char *in, const char *out );
+int mkdir_p( char *path, int len );
 
 #define DEFINE_ARRAY_TYPE(T) \
 	typedef struct { \
@@ -251,9 +325,11 @@ typedef struct {
 	list_head_t links;
 	void (*cb)( void *aux );
 	void *aux;
-	time_t timeout;
+	int64_t timeout;
 } wakeup_t;
 
+void init_timers( void );
+int64_t get_now( void );
 void init_wakeup( wakeup_t *tmr, void (*cb)( void * ), void *aux );
 void conf_wakeup( wakeup_t *tmr, int timeout );
 void wipe_wakeup( wakeup_t *tmr );

src/config.c

@@ -1,40 +1,65 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2011 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
+#define DEBUG_FLAG DEBUG_MAIN
+
 #include "config.h"
 
 #include "sync.h"
 
-#include <assert.h>
-#include <unistd.h>
-#include <limits.h>
 #include <pwd.h>
-#include <sys/types.h>
 #include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
+#include <errno.h>
+#include <sys/stat.h>
 
-static store_conf_t *stores;
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) || defined(__CYGWIN__)
+char FieldDelimiter = ';';
+#else
+char FieldDelimiter = ':';
+#endif
+
+DEF_BIT_FORMATTER_FUNCTION(ops, OP)
+
+char *
+expand_strdup( const char *s, const conffile_t *cfile )
+{
+	struct passwd *pw;
+	const char *p, *q;
+	char *r;
+
+	if (*s == '~') {
+		s++;
+		if (!*s) {
+			p = NULL;
+			q = Home;
+		} else if (*s == '/') {
+			p = s;
+			q = Home;
+		} else {
+			if ((p = strchr( s, '/' ))) {
+				r = nfstrndup( s, (size_t)(p - s) );
+				pw = getpwnam( r );
+				free( r );
+			} else {
+				pw = getpwnam( s );
+			}
+			if (!pw)
+				return NULL;
+			q = pw->pw_dir;
+		}
+		nfasprintf( &r, "%s%s", q, p ? p : "" );
+		return r;
+	} else if (*s != '/') {
+		nfasprintf( &r, "%.*s%s", cfile->path_len, cfile->file, s );
+		return r;
+	} else {
+		return nfstrdup( s );
+	}
+}
 
 char *
 get_arg( conffile_t *cfile, int required, int *comment )
@@ -61,14 +86,15 @@ get_arg( conffile_t *cfile, int required, int *comment )
 			if (escaped && c >= 32) {
 				escaped = 0;
 				*t++ = c;
-			} else if (c == '\\')
+			} else if (c == '\\') {
 				escaped = 1;
-			else if (c == '"')
+			} else if (c == '"') {
 				quoted ^= 1;
-			else if (!quoted && isspace( (uchar)c ))
+			} else if (!quoted && isspace( (uchar)c )) {
 				break;
-			else
+			} else {
 				*t++ = c;
+			}
 		}
 		*t = 0;
 		if (escaped) {
@@ -148,6 +174,7 @@ static const struct {
 	const char *name;
 } boxOps[] = {
 	{ OP_EXPUNGE, "Expunge" },
+	{ OP_EXPUNGE_SOLO, "ExpungeSolo" },
 	{ OP_CREATE, "Create" },
 	{ OP_REMOVE, "Remove" },
 };
@@ -160,53 +187,94 @@ getopt_helper( conffile_t *cfile, int *cops, channel_conf_t *conf )
 
 	if (!strcasecmp( "Sync", cfile->cmd )) {
 		arg = cfile->val;
-		do
-			if (!strcasecmp( "Push", arg ))
+		do {
+			if (!strcasecmp( "Push", arg )) {
 				*cops |= XOP_PUSH;
-			else if (!strcasecmp( "Pull", arg ))
+			} else if (!strcasecmp( "Pull", arg )) {
 				*cops |= XOP_PULL;
-			else if (!strcasecmp( "ReNew", arg ))
-				*cops |= OP_RENEW;
-			else if (!strcasecmp( "New", arg ))
+			} else if (!strcasecmp( "Upgrade", arg )) {
+				*cops |= OP_UPGRADE;
+			} else if (!strcasecmp( "ReNew", arg )) {
+				cfile->renew_warn = 1;
+				*cops |= OP_UPGRADE;
+			} else if (!strcasecmp( "New", arg )) {
 				*cops |= OP_NEW;
-			else if (!strcasecmp( "Delete", arg ))
-				*cops |= OP_DELETE;
-			else if (!strcasecmp( "Flags", arg ))
+			} else if (!strcasecmp( "Old", arg )) {
+				*cops |= OP_OLD;
+			} else if (!strcasecmp( "Gone", arg )) {
+				*cops |= OP_GONE;
+			} else if (!strcasecmp( "Delete", arg )) {
+				cfile->delete_warn = 1;
+				*cops |= OP_GONE;
+			} else if (!strcasecmp( "Flags", arg )) {
 				*cops |= OP_FLAGS;
-			else if (!strcasecmp( "PullReNew", arg ))
-				conf->ops[N] |= OP_RENEW;
-			else if (!strcasecmp( "PullNew", arg ))
+			} else if (!strcasecmp( "All", arg ) || !strcasecmp( "Full", arg )) {
+				*cops |= OP_MASK_TYPE;
+			} else if (!strcasecmp( "PullUpgrade", arg )) {
+				conf->ops[N] |= OP_UPGRADE;
+			} else if (!strcasecmp( "PullReNew", arg )) {
+				cfile->renew_warn = 1;
+				conf->ops[N] |= OP_UPGRADE;
+			} else if (!strcasecmp( "PullNew", arg )) {
 				conf->ops[N] |= OP_NEW;
-			else if (!strcasecmp( "PullDelete", arg ))
-				conf->ops[N] |= OP_DELETE;
-			else if (!strcasecmp( "PullFlags", arg ))
+			} else if (!strcasecmp( "PullOld", arg )) {
+				conf->ops[N] |= OP_OLD;
+			} else if (!strcasecmp( "PullGone", arg )) {
+				conf->ops[N] |= OP_GONE;
+			} else if (!strcasecmp( "PullDelete", arg )) {
+				cfile->delete_warn = 1;
+				conf->ops[N] |= OP_GONE;
+			} else if (!strcasecmp( "PullFlags", arg )) {
 				conf->ops[N] |= OP_FLAGS;
-			else if (!strcasecmp( "PushReNew", arg ))
-				conf->ops[F] |= OP_RENEW;
-			else if (!strcasecmp( "PushNew", arg ))
+			} else if (!strcasecmp( "PullFull", arg )) {
+				conf->ops[N] |= OP_MASK_TYPE;
+			} else if (!strcasecmp( "PushUpgrade", arg )) {
+				conf->ops[F] |= OP_UPGRADE;
+			} else if (!strcasecmp( "PushReNew", arg )) {
+				cfile->renew_warn = 1;
+				conf->ops[F] |= OP_UPGRADE;
+			} else if (!strcasecmp( "PushNew", arg )) {
 				conf->ops[F] |= OP_NEW;
-			else if (!strcasecmp( "PushDelete", arg ))
-				conf->ops[F] |= OP_DELETE;
-			else if (!strcasecmp( "PushFlags", arg ))
+			} else if (!strcasecmp( "PushOld", arg )) {
+				conf->ops[F] |= OP_OLD;
+			} else if (!strcasecmp( "PushGone", arg )) {
+				conf->ops[F] |= OP_GONE;
+			} else if (!strcasecmp( "PushDelete", arg )) {
+				cfile->delete_warn = 1;
+				conf->ops[F] |= OP_GONE;
+			} else if (!strcasecmp( "PushFlags", arg )) {
 				conf->ops[F] |= OP_FLAGS;
-			else if (!strcasecmp( "All", arg ) || !strcasecmp( "Full", arg ))
-				*cops |= XOP_PULL|XOP_PUSH;
-			else if (strcasecmp( "None", arg ) && strcasecmp( "Noop", arg )) {
+			} else if (!strcasecmp( "PushFull", arg )) {
+				conf->ops[F] |= OP_MASK_TYPE;
+			} else if (!strcasecmp( "None", arg ) || !strcasecmp( "Noop", arg )) {
+				conf->ops[F] |= XOP_TYPE_NOOP;
+			} else {
 				error( "%s:%d: invalid Sync arg '%s'\n",
 				       cfile->file, cfile->line, arg );
 				cfile->err = 1;
 			}
-		while ((arg = get_arg( cfile, ARG_OPTIONAL, NULL )));
+		} while ((arg = get_arg( cfile, ARG_OPTIONAL, NULL )));
 		conf->ops[F] |= XOP_HAVE_TYPE;
-	} else if (!strcasecmp( "SyncState", cfile->cmd ))
-		conf->sync_state = expand_strdup( cfile->val );
-	else if (!strcasecmp( "CopyArrivalDate", cfile->cmd ))
+	} else if (!strcasecmp( "SyncState", cfile->cmd )) {
+		conf->sync_state = !strcmp( cfile->val, "*" ) ? "*" : expand_strdup( cfile->val, cfile );
+	} else if (!strcasecmp( "CopyArrivalDate", cfile->cmd )) {
 		conf->use_internal_date = parse_bool( cfile );
-	else if (!strcasecmp( "MaxMessages", cfile->cmd ))
+	} else if (!strcasecmp( "MaxMessages", cfile->cmd )) {
 		conf->max_messages = parse_int( cfile );
-	else if (!strcasecmp( "ExpireUnread", cfile->cmd ))
+	} else if (!strcasecmp( "ExpireSide", cfile->cmd )) {
+		arg = cfile->val;
+		if (!strcasecmp( "Far", arg )) {
+			conf->expire_side = F;
+		} else if (!strcasecmp( "Near", arg )) {
+			conf->expire_side = N;
+		} else {
+			error( "%s:%d: invalid ExpireSide argument '%s'\n",
+			       cfile->file, cfile->line, arg );
+			cfile->err = 1;
+		}
+	} else if (!strcasecmp( "ExpireUnread", cfile->cmd )) {
 		conf->expire_unread = parse_bool( cfile );
-	else {
+	} else {
 		for (i = 0; i < as(boxOps); i++) {
 			if (!strcasecmp( boxOps[i].name, cfile->cmd )) {
 				int op = boxOps[i].op;
@@ -224,7 +292,9 @@ getopt_helper( conffile_t *cfile, int *cops, channel_conf_t *conf )
 					} else if (!strcasecmp( "Slave", arg )) {  // Pre-1.4 legacy
 						conf->ops[N] |= op;
 						cfile->ms_warn = 1;
-					} else if (strcasecmp( "None", arg )) {
+					} else if (!strcasecmp( "None", arg )) {
+						conf->ops[F] |= op * (XOP_EXPUNGE_NOOP / OP_EXPUNGE);
+					} else {
 						error( "%s:%d: invalid %s arg '%s'\n",
 						       cfile->file, cfile->line, boxOps[i].name, arg );
 						cfile->err = 1;
@@ -264,38 +334,76 @@ getcline( conffile_t *cfile )
 	return 0;
 }
 
-/* XXX - this does not detect None conflicts ... */
+static const char *
+channel_str( const char *chan_name )
+{
+	if (!chan_name)
+		return "on the command line";
+
+	if (!*chan_name)
+		return "in global config section";
+
+	static char buf[100];
+	nfsnprintf( buf, sizeof(buf), "in Channel '%s'", chan_name );
+	return buf;
+}
+
 int
-merge_ops( int cops, int ops[] )
+merge_ops( int cops, int ops[], const char *chan_name )
 {
 	int aops, op;
 	uint i;
 
+	if (!cops && !ops[F] && !ops[N])  // Only to denoise the debug output
+		return 0;
+	debug( "merge ops (%s):\n  common: %s\n  far: %s\n  near: %s\n",
+	       channel_str( chan_name ), fmt_ops( cops ).str, fmt_ops( ops[F] ).str, fmt_ops( ops[N] ).str );
 	aops = ops[F] | ops[N];
 	if (ops[F] & XOP_HAVE_TYPE) {
-		if (aops & OP_MASK_TYPE) {
-			if (aops & cops & OP_MASK_TYPE) {
+		if (aops & OP_MASK_TYPE) {  // PullNew, etc.
+			if (ops[F] & XOP_TYPE_NOOP) {
 			  cfl:
-				error( "Conflicting Sync args specified.\n" );
+				error( "Conflicting Sync options specified %s.\n", channel_str( chan_name ) );
 				return 1;
 			}
-			ops[F] |= cops & OP_MASK_TYPE;
-			ops[N] |= cops & OP_MASK_TYPE;
+			if (aops & cops & OP_MASK_TYPE) {  // Overlapping New, etc.
+			  ovl:
+				error( "Redundant Sync options specified %s.\n", channel_str( chan_name ) );
+				return 1;
+			}
+			// Mix in non-overlapping Push/Pull or New, etc.
 			if (cops & XOP_PULL) {
-				if (ops[N] & OP_MASK_TYPE)
-					goto cfl;
-				ops[N] |= OP_MASK_TYPE;
+				if (cops & (XOP_PUSH | OP_MASK_TYPE)) {
+					// Mixing instant effect flags with row/column flags would be confusing,
+					// so instead everything is instant effect. This implies that mixing
+					// direction with type would cause overlaps, so PullNew Push Gone, etc.
+					// is invalid.
+					// Pull Push covers everything, so makes no sense to combine.
+				  ivl:
+					error( "Invalid combination of simple and compound Sync options %s.\n",
+					       channel_str( chan_name ) );
+					return 1;
+				}
+				if (ops[N] & OP_DFLT_TYPE)
+					goto ovl;
+				ops[N] |= OP_DFLT_TYPE;
+			} else if (cops & XOP_PUSH) {
+				if (cops & OP_MASK_TYPE)
+					goto ivl;
+				if (ops[F] & OP_DFLT_TYPE)
+					goto ovl;
+				ops[F] |= OP_DFLT_TYPE;
+			} else {
+				ops[F] |= cops & OP_MASK_TYPE;
+				ops[N] |= cops & OP_MASK_TYPE;
 			}
-			if (cops & XOP_PUSH) {
-				if (ops[F] & OP_MASK_TYPE)
-					goto cfl;
-				ops[F] |= OP_MASK_TYPE;
-			}
-		} else if (cops & (OP_MASK_TYPE|XOP_MASK_DIR)) {
+		} else if (cops & (OP_MASK_TYPE | XOP_MASK_DIR)) {  // Pull New, etc.
+			if (ops[F] & XOP_TYPE_NOOP)
+				goto cfl;
 			if (!(cops & OP_MASK_TYPE))
-				cops |= OP_MASK_TYPE;
+				cops |= OP_DFLT_TYPE;
 			else if (!(cops & XOP_MASK_DIR))
-				cops |= XOP_PULL|XOP_PUSH;
+				cops |= XOP_PULL | XOP_PUSH;
 			if (cops & XOP_PULL)
 				ops[N] |= cops & OP_MASK_TYPE;
 			if (cops & XOP_PUSH)
@@ -305,14 +413,19 @@ merge_ops( int cops, int ops[] )
 	for (i = 0; i < as(boxOps); i++) {
 		op = boxOps[i].op;
 		if (ops[F] & (op * (XOP_HAVE_EXPUNGE / OP_EXPUNGE))) {
+			if (((aops | cops) & op) && (ops[F] & (op * (XOP_EXPUNGE_NOOP / OP_EXPUNGE)))) {
+				error( "Conflicting %s options specified %s.\n", boxOps[i].name, channel_str( chan_name ) );
+				return 1;
+			}
 			if (aops & cops & op) {
-				error( "Conflicting %s args specified.\n", boxOps[i].name );
+				error( "Redundant %s options specified %s.\n", boxOps[i].name, channel_str( chan_name ) );
 				return 1;
 			}
 			ops[F] |= cops & op;
 			ops[N] |= cops & op;
 		}
 	}
+	debug( "  => far: %s\n  => near: %s\n", fmt_ops( ops[F] ).str, fmt_ops( ops[N] ).str );
 	return 0;
 }
 
@@ -327,14 +440,39 @@ load_config( const char *where )
 	char *arg, *p;
 	uint len, max_size;
 	int cops, gcops, glob_ok, fn, i;
-	char path[_POSIX_PATH_MAX];
+	char path[_POSIX_PATH_MAX], path2[_POSIX_PATH_MAX];
 	char buf[1024];
 
 	if (!where) {
-		nfsnprintf( path, sizeof(path), "%s/." EXE "rc", Home );
-		cfile.file = path;
-	} else
-		cfile.file = where;
+		int path_len, path_len2;
+		const char *config_home = getenv( "XDG_CONFIG_HOME" );
+		if (config_home)
+			nfsnprintf( path, sizeof(path), "%s/%nisyncrc", config_home, &path_len );
+		else
+			nfsnprintf( path, sizeof(path), "%s/.config/%nisyncrc", Home, &path_len );
+		nfsnprintf( path2, sizeof(path2), "%s/%n.mbsyncrc", Home, &path_len2 );
+		struct stat st;
+		int ex = !lstat( path, &st );
+		int ex2 = !lstat( path2, &st );
+		if (ex2 && !ex) {
+			cfile.file = path2;
+			cfile.path_len = path_len2;
+		} else {
+			if (ex && ex2)
+				warn( "Both %s and %s exist; using the former.\n", path, path2 );
+			cfile.file = path;
+			cfile.path_len = path_len;
+		}
+	} else {
+		const char *sl = strrchr( where, '/' );
+		if (!sl) {
+			nfsnprintf( path, sizeof(path), "./%n%s", &cfile.path_len, where );
+			cfile.file = path;
+		} else {
+			cfile.path_len = sl - where + 1;
+			cfile.file = where;
+		}
+	}
 
 	info( "Reading configuration file %s\n", cfile.file );
 
@@ -348,16 +486,19 @@ load_config( const char *where )
 	cfile.line = 0;
 	cfile.err = 0;
 	cfile.ms_warn = 0;
+	cfile.renew_warn = 0;
+	cfile.delete_warn = 0;
 	cfile.rest = NULL;
 
 	gcops = 0;
 	glob_ok = 1;
+	global_conf.expire_side = N;
 	global_conf.expire_unread = -1;
   reloop:
 	while (getcline( &cfile )) {
 		if (!cfile.cmd)
 			continue;
-		for (i = 0; i < N_DRIVERS; i++)
+		for (i = 0; i < N_DRIVERS; i++) {
 			if (drivers[i]->parse_store( &cfile, &store )) {
 				if (store) {
 					if (!store->max_size)
@@ -371,27 +512,26 @@ load_config( const char *where )
 				glob_ok = 0;
 				goto reloop;
 			}
-		if (!strcasecmp( "Channel", cfile.cmd ))
-		{
-			channel = nfcalloc( sizeof(*channel) );
+		}
+		if (!strcasecmp( "Channel", cfile.cmd )) {
+			channel = nfzalloc( sizeof(*channel) );
 			channel->name = nfstrdup( cfile.val );
 			channel->max_messages = global_conf.max_messages;
+			channel->expire_side = global_conf.expire_side;
 			channel->expire_unread = global_conf.expire_unread;
 			channel->use_internal_date = global_conf.use_internal_date;
 			cops = 0;
 			max_size = UINT_MAX;
 			while (getcline( &cfile ) && cfile.cmd) {
-				if (!strcasecmp( "MaxSize", cfile.cmd ))
+				if (!strcasecmp( "MaxSize", cfile.cmd )) {
 					max_size = parse_size( &cfile );
-				else if (!strcasecmp( "Pattern", cfile.cmd ) ||
-				         !strcasecmp( "Patterns", cfile.cmd ))
-				{
+				} else if (!strcasecmp( "Pattern", cfile.cmd ) ||
+				           !strcasecmp( "Patterns", cfile.cmd )) {
 					arg = cfile.val;
-					do
+					do {
 						add_string_list( &channel->patterns, arg );
-					while ((arg = get_arg( &cfile, ARG_OPTIONAL, NULL )));
-				}
-				else if (!strcasecmp( "Far", cfile.cmd )) {
+					} while ((arg = get_arg( &cfile, ARG_OPTIONAL, NULL )));
+				} else if (!strcasecmp( "Far", cfile.cmd )) {
 					fn = F;
 					goto linkst;
 				} else if (!strcasecmp( "Master", cfile.cmd )) {  // Pre-1.4 legacy
@@ -412,11 +552,13 @@ load_config( const char *where )
 						continue;
 					}
 					*p = 0;
-					for (store = stores; store; store = store->next)
+					for (store = stores; store; store = store->next) {
 						if (!strcmp( store->name, cfile.val + 1 )) {
 							channel->stores[fn] = store;
 							goto stpcom;
 						}
+					}
+					channel->stores[fn] = (void *)~0;
 					error( "%s:%d: unknown store '%s'\n",
 					       cfile.file, cfile.line, cfile.val + 1 );
 					cfile.err = 1;
@@ -427,31 +569,30 @@ load_config( const char *where )
 				} else if (!getopt_helper( &cfile, &cops, channel )) {
 					error( "%s:%d: keyword '%s' is not recognized in Channel sections\n",
 					       cfile.file, cfile.line, cfile.cmd );
+					cfile.rest = NULL;
 					cfile.err = 1;
 				}
 			}
 			if (!channel->stores[F]) {
 				error( "channel '%s' refers to no far side store\n", channel->name );
 				cfile.err = 1;
-			} else if (!channel->stores[N]) {
+			}
+			if (!channel->stores[N]) {
 				error( "channel '%s' refers to no near side store\n", channel->name );
 				cfile.err = 1;
-			} else if (merge_ops( cops, channel->ops ))
-				cfile.err = 1;
-			else {
-				if (max_size != UINT_MAX) {
-					if (!max_size)
-						max_size = UINT_MAX;
-					channel->stores[F]->max_size = channel->stores[N]->max_size = max_size;
-				}
-				*channelapp = channel;
-				channelapp = &channel->next;
 			}
+			if (merge_ops( cops, channel->ops, channel->name ))
+				cfile.err = 1;
+			if (max_size != UINT_MAX && !cfile.err) {
+				if (!max_size)
+					max_size = UINT_MAX;
+				channel->stores[F]->max_size = channel->stores[N]->max_size = max_size;
+			}
+			*channelapp = channel;
+			channelapp = &channel->next;
 			glob_ok = 0;
 			goto reloop;
-		}
-		else if (!strcasecmp( "Group", cfile.cmd ))
-		{
+		} else if (!strcasecmp( "Group", cfile.cmd )) {
 			group = nfmalloc( sizeof(*group) );
 			group->name = nfstrdup( cfile.val );
 			*groupapp = group;
@@ -470,27 +611,21 @@ load_config( const char *where )
 			}
 			while (getcline( &cfile ) && cfile.cmd) {
 				if (!strcasecmp( "Channel", cfile.cmd ) ||
-				    !strcasecmp( "Channels", cfile.cmd ))
-				{
+				    !strcasecmp( "Channels", cfile.cmd )) {
 					arg = cfile.val;
 					goto addone;
-				}
-				else
-				{
+				} else {
 					error( "%s:%d: keyword '%s' is not recognized in Group sections\n",
 					       cfile.file, cfile.line, cfile.cmd );
+					cfile.rest = NULL;
 					cfile.err = 1;
 				}
 			}
 			glob_ok = 0;
 			goto reloop;
-		}
-		else if (!strcasecmp( "FSync", cfile.cmd ))
-		{
+		} else if (!strcasecmp( "FSync", cfile.cmd )) {
 			UseFSync = parse_bool( &cfile );
-		}
-		else if (!strcasecmp( "FieldDelimiter", cfile.cmd ))
-		{
+		} else if (!strcasecmp( "FieldDelimiter", cfile.cmd )) {
 			if (strlen( cfile.val ) != 1) {
 				error( "%s:%d: Field delimiter must be exactly one character long\n", cfile.file, cfile.line );
 				cfile.err = 1;
@@ -501,20 +636,17 @@ load_config( const char *where )
 					cfile.err = 1;
 				}
 			}
-		}
-		else if (!strcasecmp( "BufferLimit", cfile.cmd ))
-		{
+		} else if (!strcasecmp( "BufferLimit", cfile.cmd )) {
 			BufferLimit = parse_size( &cfile );
 			if (!BufferLimit) {
 				error( "%s:%d: BufferLimit cannot be zero\n", cfile.file, cfile.line );
 				cfile.err = 1;
 			}
-		}
-		else if (!getopt_helper( &cfile, &gcops, &global_conf ))
-		{
+		} else if (!getopt_helper( &cfile, &gcops, &global_conf )) {
 			error( "%s:%d: '%s' is not a recognized section-starting or global keyword\n",
 			       cfile.file, cfile.line, cfile.cmd );
 			cfile.err = 1;
+			cfile.rest = NULL;
 			while (getcline( &cfile ))
 				if (!cfile.cmd)
 					goto reloop;
@@ -529,8 +661,30 @@ load_config( const char *where )
 	fclose (cfile.fp);
 	if (cfile.ms_warn)
 		warn( "Notice: Master/Slave are deprecated; use Far/Near instead.\n" );
-	cfile.err |= merge_ops( gcops, global_conf.ops );
-	if (!global_conf.sync_state)
-		global_conf.sync_state = expand_strdup( "~/." EXE "/" );
+	if (cfile.renew_warn)
+		warn( "Notice: ReNew is deprecated; use Upgrade instead.\n" );
+	if (cfile.delete_warn)
+		warn( "Notice: Delete is deprecated; use Gone instead.\n" );
+	cfile.err |= merge_ops( gcops, global_conf.ops, "" );
+	if (!global_conf.sync_state) {
+		const char *state_home = getenv( "XDG_STATE_HOME" );
+		if (state_home)
+			nfsnprintf( path, sizeof(path), "%s/isync/", state_home );
+		else
+			nfsnprintf( path, sizeof(path), "%s/.local/state/isync/", Home );
+		nfsnprintf( path2, sizeof(path2), "%s/.mbsync/", Home );
+		struct stat st;
+		int ex = !lstat( path, &st );
+		int ex2 = !lstat( path2, &st );
+		if (ex2 && !ex) {
+			global_conf.sync_state = nfstrdup( path2 );
+		} else {
+			if (ex && ex2) {
+				error( "Error: both %s and %s exist; delete one or set SyncState globally.\n", path, path2 );
+				cfile.err = 1;
+			}
+			global_conf.sync_state = nfstrdup( path );
+		}
+	}
 	return cfile.err;
 }

src/config.h

@@ -1,23 +1,8 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #ifndef CONFIG_H
@@ -32,20 +17,25 @@ typedef struct {
 	int bufl;
 	int line;
 	int err;
-	int ms_warn;
+	int ms_warn, renew_warn, delete_warn;
+	int path_len;
 	char *cmd, *val, *rest;
 } conffile_t;
 
+extern char FieldDelimiter;
+
 #define ARG_OPTIONAL 0
 #define ARG_REQUIRED 1
 
+char *expand_strdup( const char *s, const conffile_t *cfile );
+
 char *get_arg( conffile_t *cfile, int required, int *comment );
 
 char parse_bool( conffile_t *cfile );
 int parse_int( conffile_t *cfile );
 uint parse_size( conffile_t *cfile );
 int getcline( conffile_t *cfile );
-int merge_ops( int cops, int ops[] );
+int merge_ops( int cops, int ops[], const char *chan_name );
 int load_config( const char *filename );
 
 #endif

src/driver.c

@@ -1,50 +1,68 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #include "driver.h"
 
-#include <stdlib.h>
-#include <string.h>
+store_conf_t *stores;
 
 driver_t *drivers[N_DRIVERS] = { &maildir_driver, &imap_driver };
 
-uint
-count_generic_messages( message_t *msgs )
+void
+cleanup_drivers( void )
 {
-	uint count = 0;
-	for (; msgs; msgs = msgs->next)
-		count++;
-	return count;
+	for (int t = 0; t < N_DRIVERS; t++)
+		drivers[t]->cleanup();
+}
+
+// Keep the MESSAGE_FLAGS in sync (grep that)!
+const char MsgFlags[] = { 'D', 'F', 'P', 'R', 'S', 'T' };
+
+static void
+make_flags( uchar flags, char *buf )
+{
+	uint i, d;
+
+	for (i = d = 0; i < as(MsgFlags); i++)
+		if (flags & (1 << i))
+			buf[d++] = MsgFlags[i];
+	buf[d] = 0;
+}
+
+flag_str_t
+fmt_flags( uchar flags )
+{
+	flag_str_t buf;
+
+	make_flags( flags, buf.str );
+	return buf;
+}
+
+flag_str_t
+fmt_lone_flags( uchar flags )
+{
+	flag_str_t buf;
+
+	if (!flags) {
+		buf.str[0] = '-';
+		buf.str[1] = 0;
+	} else {
+		make_flags( flags, buf.str );
+	}
+	return buf;
 }
 
 void
 free_generic_messages( message_t *msgs )
 {
-	message_t *tmsg;
-
-	for (; msgs; msgs = tmsg) {
-		tmsg = msgs->next;
+	while (msgs) {
+		message_t *tmsg = msgs->next;
 		free( msgs->msgid );
 		free( msgs );
+		msgs = tmsg;
 	}
 }
 
@@ -73,6 +91,7 @@ parse_generic_store( store_conf_t *store, conffile_t *cfg, const char *type )
 		store->flat_delim = nfstrdup( cfg->val );
 	} else {
 		error( "%s:%d: keyword '%s' is not recognized in %s sections\n", cfg->file, cfg->line, cfg->cmd, type );
+		cfg->rest = NULL;
 		cfg->err = 1;
 	}
 }

src/driver.h

@@ -1,29 +1,15 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #ifndef DRIVER_H
 #define DRIVER_H
 
 #include "config.h"
+#include "driver_enum.h"
 
 typedef struct driver driver_t;
 
@@ -45,27 +31,37 @@ typedef struct store_conf {
 	STORE_CONF
 } store_conf_t;
 
+extern store_conf_t *stores;
+
 /* For message->flags */
-/* Keep the mailbox driver flag definitions in sync: */
-/* grep for MAILBOX_DRIVER_FLAG */
+// Keep the MESSAGE_FLAGS in sync (grep that)!
 /* The order is according to alphabetical maildir flag sort */
-#define F_DRAFT	     (1<<0) /* Draft */
-#define F_FLAGGED    (1<<1) /* Flagged */
-#define F_FORWARDED  (1<<2) /* Passed */
-#define F_ANSWERED   (1<<3) /* Replied */
-#define F_SEEN       (1<<4) /* Seen */
-#define F_DELETED    (1<<5) /* Trashed */
-#define NUM_FLAGS 6
+BIT_ENUM(
+	F_DRAFT,      // Draft
+	F_FLAGGED,    // Flagged
+	F_FORWARDED,  // Passed
+	F_ANSWERED,   // Replied
+	F_SEEN,       // Seen
+	F_DELETED,    // Trashed
+)
+
+extern const char MsgFlags[F__NUM_BITS];
+typedef struct { char str[F__NUM_BITS + 1]; } flag_str_t;
+flag_str_t ATTR_OPTIMIZE /* force RVO */ fmt_flags( uchar flags );
+flag_str_t ATTR_OPTIMIZE /* force RVO */ fmt_lone_flags( uchar flags );
 
 /* For message->status */
-#define M_RECENT       (1<<0) /* unsyncable flag; maildir_* depend on this being 1<<0 */
-#define M_DEAD         (1<<1) /* expunged */
-#define M_FLAGS        (1<<2) /* flags fetched */
-// The following are only for IMAP FETCH response parsing
-#define M_DATE         (1<<3)
-#define M_SIZE         (1<<4)
-#define M_BODY         (1<<5)
-#define M_HEADER       (1<<6)
+BIT_ENUM(
+	M_RECENT,   // unsyncable flag; maildir_*() depend on this being bit 0
+	M_DEAD,     // expunged
+	M_EXPUNGE,  // for driver_t->close_box()
+	M_FLAGS,    // flags are valid
+	// The following are only for IMAP FETCH response parsing
+	M_DATE,
+	M_SIZE,
+	M_BODY,
+	M_HEADER,
+)
 
 #define TUIDL 12
 
@@ -83,26 +79,37 @@ typedef struct message {
 	MESSAGE(struct message)
 } message_t;
 
+static_assert_bits(F, message_t, flags);
+static_assert_bits(M, message_t, status);
+
 // For driver_t->prepare_load_box(), which may amend the passed flags.
-// The drivers don't use the first two, but may set them if loading the
+// The drivers don't use the first three, but may set them if loading the
 // particular range is required to handle some other flag; note that these
 // ranges may overlap.
-#define OPEN_OLD        (1<<0)  // Paired messages *in* this store.
-#define OPEN_NEW        (1<<1)  // Messages (possibly) not yet propagated *from* this store.
-#define OPEN_FLAGS      (1<<2)  // Note that fetch_msg() gets the flags regardless.
-#define OPEN_NEW_SIZE   (1<<4)
-#define OPEN_EXPUNGE    (1<<5)
-#define OPEN_SETFLAGS   (1<<6)
-#define OPEN_APPEND     (1<<7)
-#define OPEN_FIND       (1<<8)
-#define OPEN_OLD_IDS    (1<<9)
+BIT_ENUM(
+	OPEN_PAIRED,      // Paired messages *in* this store.
+	OPEN_OLD,         // Messages that should be already propagated *from* this store.
+	OPEN_NEW,         // Messages (possibly) not yet propagated *from* this store.
+	OPEN_FIND,
+	OPEN_FLAGS,       // Note that fetch_msg() gets the flags regardless.
+	OPEN_OLD_SIZE,
+	OPEN_NEW_SIZE,
+	OPEN_PAIRED_IDS,
+	OPEN_APPEND,
+	OPEN_SETFLAGS,
+	OPEN_EXPUNGE,
+	// Expunge only deleted messages we know about. Relies on OPEN_{OLD,NEW,FLAGS}
+	// being set externally. The driver may unset it if it can't handle it.
+	OPEN_UID_EXPUNGE,
+)
 
 #define UIDVAL_BAD ((uint)-1)
 
 #define STORE(store) \
 	store *next; \
 	driver_t *driver; \
-	store##_conf *conf;  /* foreign */
+	store##_conf *conf;  /* foreign */ \
+	uchar racy_trash;
 
 typedef struct store {
 	STORE(struct store)
@@ -115,6 +122,8 @@ typedef struct {
 	uchar flags;
 } msg_data_t;
 
+static_assert_bits(F, msg_data_t, flags);
+
 #define DRV_OK          0
 /* Message went missing, or mailbox is full, etc. */
 #define DRV_MSG_BAD     1
@@ -122,13 +131,12 @@ typedef struct {
 #define DRV_BOX_BAD     2
 /* Failed to connect store. */
 #define DRV_STORE_BAD   3
-/* The command has been cancel()ed or cancel_store()d. */
+/* The command has been cancel_cmds()d or cancel_store()d. */
 #define DRV_CANCELED    4
 
 /* All memory belongs to the driver's user, unless stated otherwise. */
-// If the driver is NOT DRV_ASYNC, memory owned by the driver returned
-// through callbacks MUST remain valid until a related subsequent command
-// is invoked, as the proxy driver may deliver these pointers with delay.
+// All memory passed to driver functions must remain valid until the
+// respective result callback is invoked.
 
 /*
    This flag says that the driver CAN store messages with CRLFs,
@@ -137,7 +145,7 @@ typedef struct {
 */
 #define DRV_CRLF        1
 /*
-   This flag says that the driver will act upon (DFlags & VERBOSE).
+   This flag says that the driver will act upon (Verbosity >= VERBOSE).
 */
 #define DRV_VERBOSE     2
 /*
@@ -165,9 +173,14 @@ struct driver {
 	 * return quickly, and must not fail. */
 	store_t *(*alloc_store)( store_conf_t *conf, const char *label );
 
-	/* When this callback is invoked (at most once per store), the store is fubar;
-	 * call cancel_store() to dispose of it. */
-	void (*set_bad_callback)( store_t *ctx, void (*cb)( void *aux ), void *aux );
+	// When exp_cb is invoked, the passed message has been expunged;
+	// its status is M_DEAD now.
+	// When bad_cb is invoked (at most once per store), the store is fubar;
+	// call cancel_store() to dispose of it.
+	void (*set_callbacks)( store_t *ctx,
+	                       void (*exp_cb)( message_t *msg, void *aux ),
+	                       void (*bad_cb)( void *aux ),
+	                       void *aux );
 
 	/* Open/connect the store. This may recycle existing server connections. */
 	void (*connect_store)( store_t *ctx,
@@ -229,10 +242,13 @@ struct driver {
 	 * and those named in the excs array (smaller than minuid).
 	 * The driver takes ownership of the excs array.
 	 * Messages starting with finduid need to have the TUID populated when OPEN_FIND is set.
-	 * Messages up to pairuid need to have the Message-Id populated when OPEN_OLD_IDS is set.
+	 * Messages up to pairuid need to have the Message-Id populated when OPEN_PAIRED_IDS is set.
 	 * Messages up to newuid need to have the size populated when OPEN_OLD_SIZE is set;
 	 * likewise messages above newuid when OPEN_NEW_SIZE is set.
-	 * The returned message list remains owned by the driver. */
+	 * The returned message list remains owned by the driver and remains valid
+	 * until a new box is selected or the store is freed. New messages within
+	 * the specified range may be added to the list as a result of invoking
+	 * other driver functions. */
 	void (*load_box)( store_t *ctx, uint minuid, uint maxuid, uint finduid, uint pairuid, uint newuid, uint_array_t excs,
 	                  void (*cb)( int sts, message_t *msgs, int total_msgs, int recent_msgs, void *aux ), void *aux );
 
@@ -256,8 +272,8 @@ struct driver {
 
 	/* Add/remove the named flags to/from the given message. The message may be either
 	 * a pre-fetched one (in which case the in-memory representation is updated),
-	 * or it may be identifed by UID only. The operation may be delayed until commit()
-	 * is called. */
+	 * or it may be identifed by UID only.
+	 * The operation may be delayed until commit_cmds() is called. */
 	void (*set_msg_flags)( store_t *ctx, message_t *msg, uint uid, int add, int del,
 	                       void (*cb)( int sts, void *aux ), void *aux );
 
@@ -268,8 +284,9 @@ struct driver {
 
 	/* Expunge deleted messages from the current mailbox and close it.
 	 * There is no need to explicitly close a mailbox if no expunge is needed. */
+	// If reported is true, the expunge callback was called reliably.
 	void (*close_box)( store_t *ctx,
-	                   void (*cb)( int sts, void *aux ), void *aux );
+	                   void (*cb)( int sts, int reported, void *aux ), void *aux );
 
 	/* Cancel queued commands which are not in flight yet; they will have their
 	 * callbacks invoked with DRV_CANCELED. Afterwards, wait for the completion of
@@ -288,15 +305,16 @@ struct driver {
 	int (*get_fail_state)( store_conf_t *conf );
 };
 
-uint count_generic_messages( message_t * );
 void free_generic_messages( message_t * );
 
 void parse_generic_store( store_conf_t *store, conffile_t *cfg, const char *type );
 
-store_t *proxy_alloc_store( store_t *real_ctx, const char *label );
+store_t *proxy_alloc_store( store_t *real_ctx, const char *label, int force_async );
 
 #define N_DRIVERS 2
 extern driver_t *drivers[N_DRIVERS];
 extern driver_t maildir_driver, imap_driver, proxy_driver;
 
+void cleanup_drivers( void );
+
 #endif

src/drv_maildir.c

@@ -1,40 +1,20 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-FileCopyrightText: 2004 Theodore Y. Ts'o <tytso@mit.edu>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2013 Oswald Buddenhagen <ossi@users.sf.net>
- * Copyright (C) 2004 Theodore Y. Ts'o <tytso@mit.edu>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
+#define DEBUG_FLAG DEBUG_MAILDIR
+
 #include "driver.h"
 
-#include <assert.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <string.h>
 #include <ctype.h>
 #include <dirent.h>
 #include <fcntl.h>
-#include <stdio.h>
-#include <unistd.h>
 #include <sys/stat.h>
 #include <errno.h>
-#include <time.h>
 #include <utime.h>
 
 #if !defined(_POSIX_SYNCHRONIZED_IO) || _POSIX_SYNCHRONIZED_IO <= 0
@@ -55,6 +35,7 @@ typedef union maildir_store_conf {
 	struct {
 		STORE_CONF
 		char *path;
+		char *path_sfx;
 		char *inbox;
 #ifdef USE_DB
 		int alt_map;
@@ -93,10 +74,12 @@ typedef union maildir_store {
 		// but mailbox totals. also, don't trust them beyond the initial load.
 		int total_msgs, recent_msgs;
 		maildir_message_t *msgs;
+		maildir_message_t **app_msgs;  // Only for testing find_new()
 		wakeup_t lcktmr;
 
+		void (*expunge_callback)( message_t *msg, void *aux );
 		void (*bad_callback)( void *aux );
-		void *bad_callback_aux;
+		void *callback_aux;
 	};
 } maildir_store_t;
 
@@ -107,21 +90,6 @@ static struct flock lck;
 
 static int MaildirCount;
 
-static void ATTR_PRINTFLIKE(1, 2)
-debug( const char *msg, ... )
-{
-	va_list va;
-
-	va_start( va, msg );
-	vdebug( DEBUG_SYNC, msg, va );
-	va_end( va );
-}
-
-/* Keep the mailbox driver flag definitions in sync: */
-/* grep for MAILBOX_DRIVER_FLAG */
-/* The order is according to alphabetical maildir flag sort */
-static const char Flags[] = { 'D', 'F', 'P', 'R', 'S', 'T' };
-
 static uchar
 maildir_parse_flags( const char *info_prefix, const char *base )
 {
@@ -131,8 +99,8 @@ maildir_parse_flags( const char *info_prefix, const char *base )
 
 	flags = 0;
 	if ((s = strstr( base, info_prefix )))
-		for (s += 3, i = 0; i < as(Flags); i++)
-			if (strchr( s, Flags[i] ))
+		for (s += 3, i = 0; i < as(MsgFlags); i++)
+			if (strchr( s, MsgFlags[i] ))
 				flags |= (1 << i);
 	return flags;
 }
@@ -154,19 +122,23 @@ static char *
 maildir_join_path( maildir_store_conf_t *conf, int in_inbox, const char *box )
 {
 	char *out, *p;
-	const char *prefix;
-	uint pl, bl, n;
+	const char *prefix, *infix;
+	uint pl, il, bl, n;
 	char c;
 
 	if (in_inbox || conf->sub_style == SUB_MAILDIRPP) {
 		prefix = conf->inbox;
+		infix = NULL;
+		il = 0;
 	} else {
 		if (maildir_ensure_path( conf ) < 0)
 			return NULL;
 		prefix = conf->path;
+		infix = conf->path_sfx;
+		il = strlen( infix ) + 1;
 	}
 	pl = strlen( prefix );
-	for (bl = 0, n = 0; (c = box[bl]); bl++)
+	for (bl = 0, n = 0; (c = box[bl]); bl++) {
 		if (c == '/') {
 			if (conf->sub_style == SUB_UNSET) {
 				error( "Maildir error: accessing subfolder '%s', but store '%s' does not specify SubFolders style\n",
@@ -179,6 +151,7 @@ maildir_join_path( maildir_store_conf_t *conf, int in_inbox, const char *box )
 			       conf->name, box );
 			return NULL;
 		}
+	}
 	switch (conf->sub_style) {
 	case SUB_VERBATIM:
 		n = 0;
@@ -189,12 +162,16 @@ maildir_join_path( maildir_store_conf_t *conf, int in_inbox, const char *box )
 	default: /* SUB_LEGACY and SUB_UNSET */
 		break;
 	}
-	out = nfmalloc( pl + bl + n + 1 );
+	out = nfmalloc( pl + il + bl + n + 1 );
 	memcpy( out, prefix, pl );
 	p = out + pl;
 	if (conf->sub_style == SUB_MAILDIRPP) {
 		*p++ = '/';
 		*p++ = '.';
+	} else if (il) {
+		*p++ = '/';
+		memcpy( p, infix, il - 1 );
+		p += il - 1;
 	}
 	while ((c = *box++)) {
 		if (c == '/') {
@@ -237,7 +214,7 @@ maildir_alloc_store( store_conf_t *gconf, const char *label ATTR_UNUSED )
 {
 	maildir_store_t *ctx;
 
-	ctx = nfcalloc( sizeof(*ctx) );
+	ctx = nfzalloc( sizeof(*ctx) );
 	ctx->driver = &maildir_driver;
 	ctx->gen.conf = gconf;
 	ctx->uvfd = -1;
@@ -266,10 +243,12 @@ maildir_connect_store( store_t *gctx,
 static void
 free_maildir_messages( maildir_message_t *msg )
 {
-	for (maildir_message_t *tmsg; (tmsg = msg); msg = tmsg) {
-		tmsg = msg->next;
+	while (msg) {
+		maildir_message_t *tmsg = msg->next;
 		free( msg->base );
+		free( msg->msgid );
 		free( msg );
+		msg = tmsg;
 	}
 }
 
@@ -309,18 +288,20 @@ maildir_cleanup_drv( void )
 }
 
 static void
-maildir_set_bad_callback( store_t *gctx, void (*cb)( void *aux ), void *aux )
+maildir_set_callbacks( store_t *gctx, void (*exp_cb)( message_t *, void * ),
+                       void (*bad_cb)( void * ), void *aux )
 {
 	maildir_store_t *ctx = (maildir_store_t *)gctx;
 
-	ctx->bad_callback = cb;
-	ctx->bad_callback_aux = aux;
+	ctx->expunge_callback = exp_cb;
+	ctx->bad_callback = bad_cb;
+	ctx->callback_aux = aux;
 }
 
 static void
 maildir_invoke_bad_callback( maildir_store_t *ctx )
 {
-	ctx->bad_callback( ctx->bad_callback_aux );
+	ctx->bad_callback( ctx->callback_aux );
 }
 
 static int
@@ -391,12 +372,10 @@ maildir_list_maildirpp( maildir_store_t *ctx, int flags, const char *inbox )
 	return 0;
 }
 
-static int maildir_list_inbox( maildir_store_t *ctx, int flags, const char *basePath );
-static int maildir_list_path( maildir_store_t *ctx, int flags, const char *inbox );
-
 static int
-maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
-                      const char *inbox, uint inboxLen, const char *basePath, uint basePathLen,
+maildir_list_recurse( maildir_store_t *ctx, int isBox,
+                      const char *inbox, uint inboxLen,
+                      char *suffix, int suffixLen,
                       char *path, int pathLen, char *name, int nameLen )
 {
 	DIR *dir;
@@ -417,7 +396,7 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
 		closedir( dir );
 		return -1;
 	}
-	if (++depth > 10) {
+	if (isBox > 10) {
 		// We do the other checks first to avoid confusing error messages for files.
 		error( "Maildir error: path %s is too deeply nested. Symlink loop?\n", path );
 		closedir( dir );
@@ -430,19 +409,19 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
 		pl = nfsnprintf( path + pathLen, _POSIX_PATH_MAX - pathLen, "%s", ent );
 		if (pl == 3 && (!memcmp( ent, "cur", 3 ) || !memcmp( ent, "new", 3 ) || !memcmp( ent, "tmp", 3 )))
 			continue;
+		if (suffixLen) {
+			if (!starts_with( ent, pl, suffix, suffixLen ))
+				continue;
+			if (pl == suffixLen) {
+				error( "Maildir error: empty mailbox name under %s - did you forget the trailing slash?\n", path );
+				closedir( dir );
+				return -1;
+			}
+			ent += suffixLen;
+		}
 		pl += pathLen;
 		if (inbox && equals( path, pl, inbox, inboxLen )) {
 			// Inbox nested into Path.
-			if (maildir_list_inbox( ctx, flags, NULL ) < 0) {
-				closedir( dir );
-				return -1;
-			}
-		} else if (basePath && equals( path, pl, basePath, basePathLen )) {
-			// Path nested into Inbox.
-			if (maildir_list_path( ctx, flags, NULL ) < 0) {
-				closedir( dir );
-				return -1;
-			}
 		} else {
 			if (style == SUB_LEGACY) {
 				if (*ent == '.') {
@@ -470,7 +449,7 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
 				add_string_list( &ctx->boxes, name );
 			path[pl] = 0;
 			name[nl++] = '/';
-			if (maildir_list_recurse( ctx, isBox + 1, flags, depth, inbox, inboxLen, basePath, basePathLen, path, pl, name, nl ) < 0) {
+			if (maildir_list_recurse( ctx, isBox + 1, inbox, inboxLen, NULL, 0, path, pl, name, nl ) < 0) {
 				closedir( dir );
 				return -1;
 			}
@@ -481,7 +460,7 @@ maildir_list_recurse( maildir_store_t *ctx, int isBox, int flags, int depth,
 }
 
 static int
-maildir_list_inbox( maildir_store_t *ctx, int flags, const char *basePath )
+maildir_list_inbox( maildir_store_t *ctx )
 {
 	char path[_POSIX_PATH_MAX], name[_POSIX_PATH_MAX];
 
@@ -491,13 +470,13 @@ maildir_list_inbox( maildir_store_t *ctx, int flags, const char *basePath )
 
 	add_string_list( &ctx->boxes, "INBOX" );
 	return maildir_list_recurse(
-	        ctx, 1, flags, 0, NULL, 0, basePath, basePath ? strlen( basePath ) - 1 : 0,
+	        ctx, 1, NULL, 0, NULL, 0,
 	        path, nfsnprintf( path, _POSIX_PATH_MAX, "%s/", ctx->conf->inbox ),
 	        name, nfsnprintf( name, _POSIX_PATH_MAX, "INBOX/" ) );
 }
 
 static int
-maildir_list_path( maildir_store_t *ctx, int flags, const char *inbox )
+maildir_list_path( maildir_store_t *ctx )
 {
 	char path[_POSIX_PATH_MAX], name[_POSIX_PATH_MAX];
 
@@ -507,9 +486,11 @@ maildir_list_path( maildir_store_t *ctx, int flags, const char *inbox )
 
 	if (maildir_ensure_path( ctx->conf ) < 0)
 		return -1;
+	const char *inbox = ctx->conf->inbox;
 	return maildir_list_recurse(
-	        ctx, 0, flags, 0, inbox, inbox ? strlen( inbox ) : 0, NULL, 0,
-	        path, nfsnprintf( path, _POSIX_PATH_MAX, "%s", ctx->conf->path ),
+	        ctx, 0, inbox, strlen( inbox ),
+	        ctx->conf->path_sfx, strlen( ctx->conf->path_sfx ),
+	        path, nfsnprintf( path, _POSIX_PATH_MAX, "%s/", ctx->conf->path ),
 	        name, 0 );
 }
 
@@ -523,9 +504,9 @@ maildir_list_store( store_t *gctx, int flags,
 	if (conf->sub_style == SUB_MAILDIRPP
 	        ? maildir_list_maildirpp( ctx, flags, conf->inbox ) < 0
 	        : ((((flags & LIST_PATH) || ((flags & LIST_PATH_MAYBE) && conf->path))
-	            && maildir_list_path( ctx, flags, conf->inbox ) < 0) ||
+	            && maildir_list_path( ctx ) < 0) ||
 	           ((flags & LIST_INBOX)
-	            && maildir_list_inbox( ctx, flags, conf->path ) < 0))) {
+	            && maildir_list_inbox( ctx ) < 0))) {
 		maildir_invoke_bad_callback( ctx );
 		cb( DRV_CANCELED, NULL, aux );
 	} else {
@@ -552,8 +533,10 @@ maildir_free_scan( msg_t_array_alloc_t *msglist )
 	uint i;
 
 	if (msglist->array.data) {
-		for (i = 0; i < msglist->array.size; i++)
+		for (i = 0; i < msglist->array.size; i++) {
 			free( msglist->array.data[i].base );
+			free( msglist->array.data[i].msgid );
+		}
 		free( msglist->array.data );
 	}
 }
@@ -593,21 +576,6 @@ maildir_clear_tmp( char *buf, int bufsz, int bl )
 	return DRV_OK;
 }
 
-static int
-make_box_dir( char *buf, int bl )
-{
-	char *p;
-
-	if (!mkdir( buf, 0700 ) || errno == EEXIST)
-		return 0;
-	p = memrchr( buf, '/', (size_t)bl - 1 );
-	*p = 0;
-	if (make_box_dir( buf, (int)(p - buf) ))
-		return -1;
-	*p = '/';
-	return mkdir( buf, 0700 );
-}
-
 static int
 maildir_validate( const char *box, int create, maildir_store_t *ctx )
 {
@@ -618,20 +586,20 @@ maildir_validate( const char *box, int create, maildir_store_t *ctx )
 	bl = nfsnprintf( buf, sizeof(buf) - 4, "%s/", box );
 	if (stat( buf, &st )) {
 		if (errno != ENOENT) {
-			sys_error( "Maildir error: cannot access mailbox '%s'", box );
+			sys_error( "Maildir error: cannot access mailbox '%s'", buf );
 			return DRV_BOX_BAD;
 		}
 		if (!create)
 			return DRV_BOX_BAD;
-		if (make_box_dir( buf, bl )) {
-			sys_error( "Maildir error: cannot create mailbox '%s'", box );
+		if (mkdir_p( buf, bl - 1 )) {
+			sys_error( "Maildir error: cannot create mailbox '%s'", buf );
 			ctx->conf->failed = FAIL_FINAL;
 			maildir_invoke_bad_callback( ctx );
 			return DRV_CANCELED;
 		}
 	} else if (!S_ISDIR(st.st_mode)) {
 	  notdir:
-		error( "Maildir error: '%s' is no valid mailbox\n", box );
+		error( "Maildir error: '%s' is no valid mailbox\n", buf );
 		return DRV_BOX_BAD;
 	}
 	for (i = 0; i < 3; i++) {
@@ -702,11 +670,11 @@ maildir_store_uidval( maildir_store_t *ctx )
 		n = sprintf( buf, "%u\n%u\n", ctx->uidvalidity, ctx->nuid );
 		lseek( ctx->uvfd, 0, SEEK_SET );
 		if (write( ctx->uvfd, buf, (uint)n ) != n || ftruncate( ctx->uvfd, n ) || (UseFSync && fdatasync( ctx->uvfd ))) {
-			error( "Maildir error: cannot write UIDVALIDITY.\n" );
+			error( "Maildir error: cannot write UIDVALIDITY in %s\n", ctx->path );
 			return DRV_BOX_BAD;
 		}
 	}
-	conf_wakeup( &ctx->lcktmr, 2 );
+	conf_wakeup( &ctx->lcktmr, 2000 );
 	return DRV_OK;
 }
 
@@ -728,7 +696,7 @@ maildir_init_uidval( maildir_store_t *ctx )
 static int
 maildir_init_uidval_new( maildir_store_t *ctx )
 {
-	notice( "Maildir notice: no UIDVALIDITY, creating new.\n" );
+	notice( "Maildir notice: no UIDVALIDITY in %s, creating new.\n", ctx->path );
 	return maildir_init_uidval( ctx );
 }
 
@@ -753,14 +721,14 @@ maildir_uidval_lock( maildir_store_t *ctx )
 #endif
 	lck.l_type = F_WRLCK;
 	if (fcntl( ctx->uvfd, F_SETLKW, &lck )) {
-		error( "Maildir error: cannot fcntl lock UIDVALIDITY.\n" );
+		error( "Maildir error: cannot fcntl lock UIDVALIDITY in %s.\n", ctx->path );
 		return DRV_BOX_BAD;
 	}
 
 #ifdef USE_DB
 	if (ctx->usedb) {
 		if (fstat( ctx->uvfd, &st )) {
-			sys_error( "Maildir error: cannot fstat UID database" );
+			sys_error( "Maildir error: cannot fstat UID database in %s", ctx->path );
 			return DRV_BOX_BAD;
 		}
 		if (db_create( &ctx->db, NULL, 0 )) {
@@ -794,7 +762,7 @@ maildir_uidval_lock( maildir_store_t *ctx )
 			 * But this would mess up the sync state completely. So better bail out and
 			 * give the user a chance to fix the mailbox. */
 			if (n) {
-				error( "Maildir error: cannot read UIDVALIDITY.\n" );
+				error( "Maildir error: cannot read UIDVALIDITY in %s.\n", ctx->path );
 				return DRV_BOX_BAD;
 			}
 #endif
@@ -802,7 +770,7 @@ maildir_uidval_lock( maildir_store_t *ctx )
 		}
 	}
 	ctx->uvok = 1;
-	conf_wakeup( &ctx->lcktmr, 2 );
+	conf_wakeup( &ctx->lcktmr, 2000 );
 	return DRV_OK;
 }
 
@@ -1057,11 +1025,10 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 		}
 #ifdef USE_DB
 		if (ctx->usedb) {
-			if (maildir_uidval_lock( ctx ) != DRV_OK)
-				;
-			else if ((ret = ctx->db->cursor( ctx->db, NULL, &dbc, 0 )))
+			if (maildir_uidval_lock( ctx ) != DRV_OK) {
+			} else if ((ret = ctx->db->cursor( ctx->db, NULL, &dbc, 0 ))) {
 				ctx->db->err( ctx->db, ret, "Maildir error: db->cursor()" );
-			else {
+			} else {
 				for (;;) {
 					if ((ret = dbc->c_get( dbc, &key, &value, DB_NEXT ))) {
 						if (ret != DB_NOTFOUND)
@@ -1092,11 +1059,11 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 				if (uid == entry->uid) {
 #if 1
 					/* See comment in maildir_uidval_lock() why this is fatal. */
-					error( "Maildir error: duplicate UID %u.\n", uid );
+					error( "Maildir error: duplicate UID %u in %s.\n", uid, ctx->path );
 					maildir_free_scan( msglist );
 					return DRV_BOX_BAD;
 #else
-					notice( "Maildir notice: duplicate UID; changing UIDVALIDITY.\n");
+					notice( "Maildir notice: duplicate UID in %s; changing UIDVALIDITY.\n", ctx->path );
 					if ((ret = maildir_init_uid( ctx )) != DRV_OK) {
 						maildir_free_scan( msglist );
 						return ret;
@@ -1109,7 +1076,8 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 				if (uid > ctx->nuid) {
 					/* In principle, we could just warn and top up nuid. However, getting into this
 					 * situation might indicate some serious trouble, so let's not make it worse. */
-					error( "Maildir error: UID %u is beyond highest assigned UID %u.\n", uid, ctx->nuid );
+					error( "Maildir error: UID %u is beyond highest assigned UID %u in %s.\n",
+					       uid, ctx->nuid, ctx->path );
 					maildir_free_scan( msglist );
 					return DRV_BOX_BAD;
 				}
@@ -1153,9 +1121,10 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 				free( entry->base );
 				entry->base = nfstrndup( buf + bl + 4, (size_t)fnl );
 			}
-			int want_size = ((ctx->opts & OPEN_NEW_SIZE) && uid > ctx->newuid);
+			int want_size = ((ctx->opts & OPEN_OLD_SIZE) && uid <= ctx->newuid) ||
+			                ((ctx->opts & OPEN_NEW_SIZE) && uid > ctx->newuid);
 			int want_tuid = ((ctx->opts & OPEN_FIND) && uid >= ctx->finduid);
-			int want_msgid = ((ctx->opts & OPEN_OLD_IDS) && uid <= ctx->pairuid);
+			int want_msgid = ((ctx->opts & OPEN_PAIRED_IDS) && uid <= ctx->pairuid);
 			if (!want_size && !want_tuid && !want_msgid)
 				continue;
 			if (!fnl)
@@ -1191,7 +1160,7 @@ maildir_scan( maildir_store_t *ctx, msg_t_array_alloc_t *msglist )
 						break;
 					if (want_tuid && starts_with( lnbuf, bufl, "X-TUID: ", 8 )) {
 						if (bufl < 8 + TUIDL) {
-							error( "Maildir error: malformed X-TUID header (UID %u)\n", uid );
+							error( "Maildir error: malformed X-TUID header in %s\n", buf );
 							continue;
 						}
 						memcpy( entry->tuid, lnbuf + 8, TUIDL );
@@ -1236,15 +1205,15 @@ maildir_init_msg( maildir_store_t *ctx, maildir_message_t *msg, msg_t *entry )
 	msg->msgid = entry->msgid;
 	entry->msgid = NULL; /* prevent deletion */
 	msg->size = entry->size;
-	msg->srec = NULL;
 	memcpy( msg->tuid, entry->tuid, TUIDL );
 	if (entry->recent)
 		msg->status |= M_RECENT;
 	if (ctx->opts & OPEN_FLAGS) {
 		msg->status |= M_FLAGS;
 		msg->flags = maildir_parse_flags( ctx->conf->info_prefix, msg->base );
-	} else
+	} else {
 		msg->flags = 0;
+	}
 }
 
 static void
@@ -1256,6 +1225,7 @@ maildir_app_msg( maildir_store_t *ctx, maildir_message_t ***msgapp, msg_t *entry
 	*msgapp = &msg->next;
 	msg->uid = entry->uid;
 	msg->status = 0;
+	msg->srec = NULL;
 	maildir_init_msg( ctx, msg, entry );
 }
 
@@ -1267,6 +1237,7 @@ maildir_select_box( store_t *gctx, const char *name )
 
 	maildir_cleanup( gctx );
 	ctx->msgs = NULL;
+	ctx->app_msgs = &ctx->msgs;
 	ctx->excs.data = NULL;
 	ctx->uvfd = -1;
 #ifdef USE_DB
@@ -1309,7 +1280,7 @@ maildir_open_box( store_t *gctx,
 
 	nfsnprintf( uvpath, sizeof(uvpath), "%s/.uidvalidity", ctx->path );
 #ifndef USE_DB
-	if ((ctx->uvfd = open( uvpath, O_RDWR|O_CREAT, 0600 )) < 0) {
+	if ((ctx->uvfd = open( uvpath, O_RDWR | O_CREAT, 0600 )) < 0) {
 		sys_error( "Maildir error: cannot write %s", uvpath );
 		cb( DRV_BOX_BAD, UIDVAL_BAD, aux );
 		return;
@@ -1320,11 +1291,11 @@ maildir_open_box( store_t *gctx,
 		nfsnprintf( uvpath, sizeof(uvpath), "%s/.isyncuidmap.db", ctx->path );
 		if ((ctx->uvfd = open( uvpath, O_RDWR, 0600 )) < 0) {
 			if (ctx->conf->alt_map) {
-				if ((ctx->uvfd = open( uvpath, O_RDWR|O_CREAT, 0600 )) >= 0)
+				if ((ctx->uvfd = open( uvpath, O_RDWR | O_CREAT, 0600 )) >= 0)
 					goto dbok;
 			} else {
 				nfsnprintf( uvpath, sizeof(uvpath), "%s/.uidvalidity", ctx->path );
-				if ((ctx->uvfd = open( uvpath, O_RDWR|O_CREAT, 0600 )) >= 0)
+				if ((ctx->uvfd = open( uvpath, O_RDWR | O_CREAT, 0600 )) >= 0)
 					goto fnok;
 			}
 			sys_error( "Maildir error: cannot write %s", uvpath );
@@ -1442,9 +1413,9 @@ maildir_prepare_load_box( store_t *gctx, uint opts )
 	maildir_store_t *ctx = (maildir_store_t *)gctx;
 
 	if (opts & OPEN_SETFLAGS)
-		opts |= OPEN_OLD;
+		opts |= OPEN_PAIRED;
 	if (opts & OPEN_EXPUNGE)
-		opts |= OPEN_OLD|OPEN_NEW|OPEN_FLAGS;
+		opts |= OPEN_PAIRED | OPEN_OLD | OPEN_NEW | OPEN_FLAGS | OPEN_UID_EXPUNGE;
 	ctx->opts = opts;
 	return opts;
 }
@@ -1466,6 +1437,7 @@ maildir_load_box( store_t *gctx, uint minuid, uint maxuid, uint finduid, uint pa
 	ARRAY_SQUEEZE( &excs );
 	ctx->excs = excs;
 
+	assert( !ctx->msgs );
 	if (maildir_scan( ctx, &msglist ) != DRV_OK) {
 		cb( DRV_BOX_BAD, NULL, 0, 0, aux );
 		return;
@@ -1473,6 +1445,7 @@ maildir_load_box( store_t *gctx, uint minuid, uint maxuid, uint finduid, uint pa
 	msgapp = &ctx->msgs;
 	for (i = 0; i < msglist.array.size; i++)
 		maildir_app_msg( ctx, &msgapp, msglist.array.data + i );
+	ctx->app_msgs = msgapp;
 	maildir_free_scan( &msglist );
 
 	cb( DRV_OK, &ctx->msgs->gen, ctx->total_msgs, ctx->recent_msgs, aux );
@@ -1488,35 +1461,30 @@ maildir_rescan( maildir_store_t *ctx )
 	ctx->fresh[0] = ctx->fresh[1] = 0;
 	if (maildir_scan( ctx, &msglist ) != DRV_OK)
 		return DRV_BOX_BAD;
+	debug( "Maildir processing rescan of %s:\n", ctx->path );
 	for (msgapp = &ctx->msgs, i = 0; (msg = *msgapp) || i < msglist.array.size; ) {
 		if (!msg) {
-#if 0
-			debug( "adding new message %u\n", msglist.array.data[i].uid );
+			debug( "  adding new message %u\n", msglist.array.data[i].uid );
 			maildir_app_msg( ctx, &msgapp, msglist.array.data + i );
-#else
-			debug( "ignoring new message %u\n", msglist.array.data[i].uid );
-#endif
 			i++;
 		} else if (i >= msglist.array.size) {
-			debug( "purging deleted message %u\n", msg->uid );
+			debug( "  purging deleted message %u\n", msg->uid );
 			msg->status = M_DEAD;
+			ctx->expunge_callback( &msg->gen, ctx->callback_aux );
 			msgapp = &msg->next;
 		} else if (msglist.array.data[i].uid < msg->uid) {
 			/* this should not happen, actually */
-#if 0
-			debug( "adding new message %u\n", msglist.array.data[i].uid );
+			debug( "  adding new message %u\n", msglist.array.data[i].uid );
 			maildir_app_msg( ctx, &msgapp, msglist.array.data + i );
-#else
-			debug( "ignoring new message %u\n", msglist.array.data[i].uid );
-#endif
 			i++;
 		} else if (msglist.array.data[i].uid > msg->uid) {
-			debug( "purging deleted message %u\n", msg->uid );
+			debug( "  purging deleted message %u\n", msg->uid );
 			msg->status = M_DEAD;
+			ctx->expunge_callback( &msg->gen, ctx->callback_aux );
 			msgapp = &msg->next;
 		} else {
-			debug( "updating message %u\n", msg->uid );
-			msg->status &= ~(M_FLAGS|M_RECENT);
+			debug( "  updating message %u\n", msg->uid );
+			msg->status &= ~(M_FLAGS | M_RECENT);
 			free( msg->base );
 			free( msg->msgid );
 			maildir_init_msg( ctx, msg, msglist.array.data + i );
@@ -1593,9 +1561,9 @@ maildir_make_flags( char info_delimiter, uchar flags, char *buf )
 	buf[0] = info_delimiter;
 	buf[1] = '2';
 	buf[2] = ',';
-	for (d = 3, i = 0; i < (int)as(Flags); i++)
+	for (d = 3, i = 0; i < (int)as(MsgFlags); i++)
 		if (flags & (1 << i))
-			buf[d++] = Flags[i];
+			buf[d++] = MsgFlags[i];
 	buf[d] = 0;
 	return d;
 }
@@ -1608,7 +1576,7 @@ maildir_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
 	const char *box;
 	int ret, fd, bl;
 	uint uid;
-	char buf[_POSIX_PATH_MAX], nbuf[_POSIX_PATH_MAX], fbuf[NUM_FLAGS + 3], base[128];
+	char buf[_POSIX_PATH_MAX], nbuf[_POSIX_PATH_MAX], fbuf[as(MsgFlags) + 3], base[128];
 
 	bl = nfsnprintf( base, sizeof(base), "%lld.%d_%d.%s", (long long)time( NULL ), Pid, ++MaildirCount, Hostname );
 	if (!to_trash) {
@@ -1637,7 +1605,7 @@ maildir_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
 
 	maildir_make_flags( ctx->conf->info_delimiter, data->flags, fbuf );
 	nfsnprintf( buf, sizeof(buf), "%s/tmp/%s%s", box, base, fbuf );
-	if ((fd = open( buf, O_WRONLY|O_CREAT|O_EXCL, 0600 )) < 0) {
+	if ((fd = open( buf, O_WRONLY | O_CREAT | O_EXCL, 0600 )) < 0) {
 		if (errno != ENOENT || !to_trash) {
 			sys_error( "Maildir error: cannot create %s", buf );
 			free( data->data );
@@ -1649,7 +1617,7 @@ maildir_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
 			cb( ret, 0, aux );
 			return;
 		}
-		if ((fd = open( buf, O_WRONLY|O_CREAT|O_EXCL, 0600 )) < 0) {
+		if ((fd = open( buf, O_WRONLY | O_CREAT | O_EXCL, 0600 )) < 0) {
 			sys_error( "Maildir error: cannot create %s", buf );
 			free( data->data );
 			cb( DRV_BOX_BAD, 0, aux );
@@ -1692,9 +1660,24 @@ maildir_store_msg( store_t *gctx, msg_data_t *data, int to_trash,
 		cb( DRV_BOX_BAD, 0, aux );
 		return;
 	}
+	if (DFlags & FAKEDUMBSTORE)
+		uid = 0;
 	cb( DRV_OK, uid, aux );
 }
 
+static void
+maildir_find_new_msgs( store_t *gctx, uint newuid,
+                       void (*cb)( int sts, message_t *msgs, void *aux ), void *aux )
+{
+	maildir_store_t *ctx = (maildir_store_t *)gctx;
+
+	assert( DFlags & FAKEDUMBSTORE );
+	ctx->opts |= OPEN_FIND;
+	ctx->finduid = newuid;
+	int ret = maildir_rescan( ctx );
+	cb( ret, &(*ctx->app_msgs)->gen, aux );
+}
+
 static void
 maildir_set_msg_flags( store_t *gctx, message_t *gmsg, uint uid ATTR_UNUSED, int add, int del,
                        void (*cb)( int sts, void *aux ), void *aux )
@@ -1713,24 +1696,24 @@ maildir_set_msg_flags( store_t *gctx, message_t *gmsg, uint uid ATTR_UNUSED, int
 	for (;;) {
 		bl = bbl + nfsnprintf( buf + bbl, _POSIX_PATH_MAX - bbl, "%s/", subdirs[gmsg->status & M_RECENT] );
 		ol = strlen( msg->base );
-		if (_POSIX_PATH_MAX - bl < ol + 3 + NUM_FLAGS)
+		if (_POSIX_PATH_MAX - bl < ol + 3 + (int)as(MsgFlags))
 			oob();
 		memcpy( buf + bl, msg->base, (size_t)ol + 1 );
 		memcpy( nbuf + bl, msg->base, (size_t)ol + 1 );
 		if ((s = strstr( nbuf + bl, conf->info_prefix ))) {
 			s += 3;
 			fl = ol - (s - (nbuf + bl));
-			for (i = 0; i < as(Flags); i++) {
-				if ((p = strchr( s, Flags[i] ))) {
+			for (i = 0; i < as(MsgFlags); i++) {
+				if ((p = strchr( s, MsgFlags[i] ))) {
 					if (del & (1 << i)) {
 						memmove( p, p + 1, (size_t)fl - (size_t)(p - s) );
 						fl--;
 					}
 				} else if (add & (1 << i)) {
-					for (j = 0; j < fl && Flags[i] > s[j]; j++);
+					for (j = 0; j < fl && MsgFlags[i] > s[j]; j++);
 					fl++;
 					memmove( s + j + 1, s + j, (size_t)(fl - j) );
-					s[j] = Flags[i];
+					s[j] = MsgFlags[i];
 				}
 			}
 			tl = ol + 3 + fl;
@@ -1808,6 +1791,7 @@ maildir_trash_msg( store_t *gctx, message_t *gmsg,
 	}
 	gmsg->status |= M_DEAD;
 	ctx->total_msgs--;
+	ctx->expunge_callback( gmsg, ctx->callback_aux );
 
 #ifdef USE_DB
 	if (ctx->usedb) {
@@ -1820,7 +1804,7 @@ maildir_trash_msg( store_t *gctx, message_t *gmsg,
 
 static void
 maildir_close_box( store_t *gctx,
-                   void (*cb)( int sts, void *aux ), void *aux )
+                   void (*cb)( int sts, int reported, void *aux ), void *aux )
 {
 	maildir_store_t *ctx = (maildir_store_t *)gctx;
 	maildir_message_t *msg;
@@ -1830,8 +1814,8 @@ maildir_close_box( store_t *gctx,
 	for (;;) {
 		retry = 0;
 		basel = nfsnprintf( buf, sizeof(buf), "%s/", ctx->path );
-		for (msg = ctx->msgs; msg; msg = msg->next)
-			if (!(msg->status & M_DEAD) && (msg->flags & F_DELETED)) {
+		for (msg = ctx->msgs; msg; msg = msg->next) {
+			if (!(msg->status & M_DEAD) && (msg->status & M_EXPUNGE)) {
 				nfsnprintf( buf + basel, _POSIX_PATH_MAX - basel, "%s/%s", subdirs[msg->status & M_RECENT], msg->base );
 				if (unlink( buf )) {
 					if (errno == ENOENT)
@@ -1841,20 +1825,22 @@ maildir_close_box( store_t *gctx,
 				} else {
 					msg->status |= M_DEAD;
 					ctx->total_msgs--;
+					ctx->expunge_callback( &msg->gen, ctx->callback_aux );
 #ifdef USE_DB
 					if (ctx->db && (ret = maildir_purge_msg( ctx, msg->base )) != DRV_OK) {
-						cb( ret, aux );
+						cb( ret, 1, aux );
 						return;
 					}
 #endif /* USE_DB */
 				}
 			}
+		}
 		if (!retry) {
-			cb( DRV_OK, aux );
+			cb( DRV_OK, 1, aux );
 			return;
 		}
-		if ((ret = maildir_rescan( (maildir_store_t *)gctx )) != DRV_OK) {
-			cb( ret, aux );
+		if ((ret = maildir_rescan( ctx )) != DRV_OK) {
+			cb( ret, 1, aux );
 			return;
 		}
 	}
@@ -1892,21 +1878,26 @@ maildir_parse_store( conffile_t *cfg, store_conf_t **storep )
 
 	if (strcasecmp( "MaildirStore", cfg->cmd ))
 		return 0;
-	store = nfcalloc( sizeof(*store) );
+	store = nfzalloc( sizeof(*store) );
 	store->info_delimiter = FieldDelimiter;
 	store->driver = &maildir_driver;
 	store->name = nfstrdup( cfg->val );
 
-	while (getcline( cfg ) && cfg->cmd)
-		if (!strcasecmp( "Inbox", cfg->cmd ))
-			store->inbox = expand_strdup( cfg->val );
-		else if (!strcasecmp( "Path", cfg->cmd ))
-			store->path = expand_strdup( cfg->val );
+	while (getcline( cfg ) && cfg->cmd) {
+		if (!strcasecmp( "Inbox", cfg->cmd )) {
+			store->inbox = expand_strdup( cfg->val, cfg );
+		} else if (!strcasecmp( "Path", cfg->cmd )) {
+			store->path = expand_strdup( cfg->val, cfg );
+		} else if (!strcasecmp( "AltMap", cfg->cmd )) {
 #ifdef USE_DB
-		else if (!strcasecmp( "AltMap", cfg->cmd ))
 			store->alt_map = parse_bool( cfg );
+#else
+			if (parse_bool( cfg )) {
+				error( "Error: AltMap=true is not supported by this build.\n" );
+				cfg->err = 1;
+			}
 #endif /* USE_DB */
-		else if (!strcasecmp( "InfoDelimiter", cfg->cmd )) {
+		} else if (!strcasecmp( "InfoDelimiter", cfg->cmd )) {
 			if (strlen( cfg->val ) != 1) {
 				error( "%s:%d: Info delimiter must be exactly one character long\n", cfg->file, cfg->line );
 				cfg->err = 1;
@@ -1929,13 +1920,28 @@ maildir_parse_store( conffile_t *cfg, store_conf_t **storep )
 				error( "%s:%d: Unrecognized SubFolders style\n", cfg->file, cfg->line );
 				cfg->err = 1;
 			}
-		} else
+		} else {
 			parse_generic_store( &store->gen, cfg, "MaildirStore" );
+		}
+	}
 	if (!store->inbox)
-		store->inbox = expand_strdup( "~/Maildir" );
-	if (store->sub_style == SUB_MAILDIRPP && store->path) {
-		error( "Maildir store '%s': Setting Path is incompatible with 'SubFolders Maildir++'\n", store->name );
-		cfg->err = 1;
+		store->inbox = expand_strdup( "~/Maildir", NULL );
+	if (store->path) {
+		if (store->sub_style == SUB_MAILDIRPP) {
+			error( "Maildir store '%s': Setting Path is incompatible with 'SubFolders Maildir++'\n", store->name );
+			cfg->err = 1;
+		} else {
+			uint inboxLen = strlen( store->inbox );
+			if (starts_with( store->path, -1, store->inbox, inboxLen ) && store->path[inboxLen] == '/') {
+				error( "Maildir store '%s': Path cannot be nested under Inbox\n", store->name );
+				cfg->err = 1;
+			} else {
+				char *s = strrchr( store->path, '/' );
+				assert( s );  // due to expand_strdup()
+				store->path_sfx = s + 1;
+				*s = 0;
+			}
+		}
 	}
 	nfasprintf( &store->info_prefix, "%c2,", store->info_delimiter );
 	nfasprintf( &store->info_stop, "%c,", store->info_delimiter );
@@ -1954,7 +1960,7 @@ struct driver maildir_driver = {
 	maildir_parse_store,
 	maildir_cleanup_drv,
 	maildir_alloc_store,
-	maildir_set_bad_callback,
+	maildir_set_callbacks,
 	maildir_connect_store,
 	maildir_free_store,
 	maildir_free_store, /* _cancel_, but it's the same */
@@ -1972,7 +1978,7 @@ struct driver maildir_driver = {
 	maildir_load_box,
 	maildir_fetch_msg,
 	maildir_store_msg,
-	NULL,  // find_new_msgs
+	maildir_find_new_msgs,
 	maildir_set_msg_flags,
 	maildir_trash_msg,
 	maildir_close_box,

src/drv_proxy.c

@@ -1,29 +1,14 @@
+// SPDX-FileCopyrightText: 2017-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2017 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
+#define DEBUG_FLAG DEBUG_DRV
+
 #include "driver.h"
 
-#include <assert.h>
-#include <limits.h>
-#include <stdlib.h>
+BIT_FORMATTER_FUNCTION(opts, OPEN)
 
 typedef struct gen_cmd gen_cmd_t;
 
@@ -35,52 +20,19 @@ typedef union proxy_store {
 		uint ref_count;
 		driver_t *real_driver;
 		store_t *real_store;
-		gen_cmd_t *done_cmds, **done_cmds_append;
+		gen_cmd_t *pending_cmds, **pending_cmds_append;
 		gen_cmd_t *check_cmds, **check_cmds_append;
 		wakeup_t wakeup;
+		uint fake_nextuid;
+		char is_fake;  // Was "created" by dry-run
+		char force_async;
 
+		void (*expunge_callback)( message_t *msg, void *aux );
 		void (*bad_callback)( void *aux );
-		void *bad_callback_aux;
+		void *callback_aux;
 	};
 } proxy_store_t;
 
-static void ATTR_PRINTFLIKE(1, 2)
-debug( const char *msg, ... )
-{
-	va_list va;
-
-	va_start( va, msg );
-	vdebug( DEBUG_DRV, msg, va );
-	va_end( va );
-}
-
-static void ATTR_PRINTFLIKE(1, 2)
-debugn( const char *msg, ... )
-{
-	va_list va;
-
-	va_start( va, msg );
-	vdebugn( DEBUG_DRV, msg, va );
-	va_end( va );
-}
-
-/* Keep the mailbox driver flag definitions in sync: */
-/* grep for MAILBOX_DRIVER_FLAG */
-/* The order is according to alphabetical maildir flag sort */
-static const char Flags[] = { 'D', 'F', 'P', 'R', 'S', 'T' };
-
-static char *
-proxy_make_flags( uchar flags, char *buf )
-{
-	uint i, d;
-
-	for (d = 0, i = 0; i < as(Flags); i++)
-		if (flags & (1 << i))
-			buf[d++] = Flags[i];
-	buf[d] = 0;
-	return buf;
-}
-
 static void
 proxy_store_deref( proxy_store_t *ctx )
 {
@@ -103,17 +55,6 @@ struct gen_cmd {
 	GEN_CMD
 };
 
-#define GEN_STS_CMD \
-	GEN_CMD \
-	int sts;
-
-typedef union {
-	gen_cmd_t gen;
-	struct {
-		GEN_STS_CMD
-	};
-} gen_sts_cmd_t;
-
 static gen_cmd_t *
 proxy_cmd_new( proxy_store_t *ctx, uint sz )
 {
@@ -139,10 +80,10 @@ proxy_wakeup( void *aux )
 {
 	proxy_store_t *ctx = (proxy_store_t *)aux;
 
-	gen_cmd_t *cmd = ctx->done_cmds;
+	gen_cmd_t *cmd = ctx->pending_cmds;
 	assert( cmd );
-	if (!(ctx->done_cmds = cmd->next))
-		ctx->done_cmds_append = &ctx->done_cmds;
+	if (!(ctx->pending_cmds = cmd->next))
+		ctx->pending_cmds_append = &ctx->pending_cmds;
 	else
 		conf_wakeup( &ctx->wakeup, 0 );
 	cmd->queued_cb( cmd );
@@ -150,22 +91,22 @@ proxy_wakeup( void *aux )
 }
 
 static void
-proxy_invoke_cb( gen_cmd_t *cmd, void (*cb)( gen_cmd_t * ), int checked, const char *name )
+proxy_invoke( gen_cmd_t *cmd, int checked, const char *name )
 {
-	if (DFlags & FORCEASYNC) {
-		debug( "%s[% 2d] Callback queue %s%s\n", cmd->ctx->label, cmd->tag, name, checked ? " (checked)" : "" );
-		cmd->queued_cb = cb;
+	proxy_store_t *ctx = cmd->ctx;
+	if (ctx->force_async) {
+		debug( "%s[% 2d] Queue %s%s\n", ctx->label, cmd->tag, name, checked ? " (checked)" : "" );
 		cmd->next = NULL;
 		if (checked) {
-			*cmd->ctx->check_cmds_append = cmd;
-			cmd->ctx->check_cmds_append = &cmd->next;
+			*ctx->check_cmds_append = cmd;
+			ctx->check_cmds_append = &cmd->next;
 		} else {
-			*cmd->ctx->done_cmds_append = cmd;
-			cmd->ctx->done_cmds_append = &cmd->next;
-			conf_wakeup( &cmd->ctx->wakeup, 0 );
+			*ctx->pending_cmds_append = cmd;
+			ctx->pending_cmds_append = &cmd->next;
+			conf_wakeup( &ctx->wakeup, 0 );
 		}
 	} else {
-		cb( cmd );
+		cmd->queued_cb( cmd );
 		proxy_cmd_done( cmd );
 	}
 }
@@ -174,8 +115,8 @@ static void
 proxy_flush_checked_cmds( proxy_store_t *ctx )
 {
 	if (ctx->check_cmds) {
-		*ctx->done_cmds_append = ctx->check_cmds;
-		ctx->done_cmds_append = ctx->check_cmds_append;
+		*ctx->pending_cmds_append = ctx->check_cmds;
+		ctx->pending_cmds_append = ctx->check_cmds_append;
 		ctx->check_cmds_append = &ctx->check_cmds;
 		ctx->check_cmds = NULL;
 		conf_wakeup( &ctx->wakeup, 0 );
@@ -183,15 +124,14 @@ proxy_flush_checked_cmds( proxy_store_t *ctx )
 }
 
 static void
-proxy_cancel_checked_cmds( proxy_store_t *ctx )
+proxy_cancel_queued_cmds( proxy_store_t *ctx )
 {
-	gen_cmd_t *cmd;
-
-	while ((cmd = ctx->check_cmds)) {
-		if (!(ctx->check_cmds = cmd->next))
-			ctx->check_cmds_append = &ctx->check_cmds;
-		((gen_sts_cmd_t *)cmd)->sts = DRV_CANCELED;
-		cmd->queued_cb( cmd );
+	if (ctx->pending_cmds || ctx->check_cmds) {
+		// This would involve directly invoking the result callbacks with
+		// DRV_CANCEL, for which we'd need another set of dispatch functions.
+		// The autotest doesn't need that, so save the effort.
+		error( "Fatal: Faking asynchronous cancelation is not supported.\n" );
+		abort();
 	}
 }
 
@@ -201,8 +141,11 @@ static @type@proxy_@name@( store_t *gctx )
 {
 	proxy_store_t *ctx = (proxy_store_t *)gctx;
 
-	@type@rv = ctx->real_driver->@name@( ctx->real_store );
-	debug( "%sCalled @name@, ret=@fmt@\n", ctx->label, rv );
+	@type@rv;
+	@pre_invoke@
+	@indent_invoke@rv = ctx->real_driver->@name@( ctx->real_store );
+	@post_invoke@
+	debug( "%sCalled @name@@print_fmt_dry@, ret=@fmt@\n", ctx->label@print_pass_dry@, rv );
 	return rv;
 }
 //# END
@@ -213,10 +156,13 @@ static @type@proxy_@name@( store_t *gctx@decl_args@ )
 	proxy_store_t *ctx = (proxy_store_t *)gctx;
 
 	@pre_print_args@
-	debug( "%sEnter @name@@print_fmt_args@\n", ctx->label@print_pass_args@ );
+	debug( "%sEnter @name@@print_fmt_dry@@print_fmt_args@\n", ctx->label@print_pass_dry@@print_pass_args@ );
 	@print_args@
-	@type@rv = ctx->real_driver->@name@( ctx->real_store@pass_args@ );
-	debug( "%sLeave @name@, ret=@fmt@\n", ctx->label, rv );
+	@type@rv;
+	@pre_invoke@
+	@indent_invoke@rv = ctx->real_driver->@name@( ctx->real_store@pass_args@ );
+	@post_invoke@
+	debug( "%sLeave @name@, ret=@print_fmt_ret@\n", ctx->label, @print_pass_ret@ );
 	return rv;
 }
 //# END
@@ -227,45 +173,75 @@ static @type@proxy_@name@( store_t *gctx@decl_args@ )
 	proxy_store_t *ctx = (proxy_store_t *)gctx;
 
 	@pre_print_args@
-	debug( "%sEnter @name@@print_fmt_args@\n", ctx->label@print_pass_args@ );
+	debug( "%sEnter @name@@print_fmt_dry@@print_fmt_args@\n", ctx->label@print_pass_dry@@print_pass_args@ );
 	@print_args@
-	ctx->real_driver->@name@( ctx->real_store@pass_args@ );
+	@pre_invoke@
+	@indent_invoke@ctx->real_driver->@name@( ctx->real_store@pass_args@ );
+	@post_invoke@
 	debug( "%sLeave @name@\n", ctx->label );
 	@action@
 }
 //# END
 
+//# TEMPLATE CALLBACK_VOID
+	debug( "%s[% 2d] Callback enter @name@\n", ctx->label, cmd->tag );
+	@print_cb_args@
+//# END
+//# TEMPLATE CALLBACK_STS
+	debug( "%s[% 2d] Callback enter @name@, sts=%d\n", ctx->label, cmd->tag, sts );
+//# END
+//# TEMPLATE CALLBACK_STS_PRN
+	debug( "%s[% 2d] Callback enter @name@, sts=%d\n", ctx->label, cmd->tag, sts );
+	if (sts == DRV_OK) {
+		@print_cb_args@
+	}
+//# END
+//# TEMPLATE CALLBACK_STS_FMT
+	if (sts == DRV_OK) {
+		debug( "%s[% 2d] Callback enter @name@, sts=" stringify(DRV_OK) "@print_fmt_cb_args@\n", ctx->label, cmd->tag@print_pass_cb_args@ );
+		@print_cb_args@
+	} else {
+		debug( "%s[% 2d] Callback enter @name@, sts=%d\n", ctx->label, cmd->tag, sts );
+	}
+//# END
+
 //# TEMPLATE CALLBACK
 typedef union {
-	@gen_cmd_t@ gen;
+	gen_cmd_t gen;
 	struct {
-		@GEN_CMD@
-		@decl_cb_state@
+		GEN_CMD
 		void (*callback)( @decl_cb_args@void *aux );
 		void *callback_aux;
 		@decl_state@
 	};
 } @name@_cmd_t;
 
-static void
-proxy_do_@name@_cb( gen_cmd_t *gcmd )
-{
-	@name@_cmd_t *cmd = (@name@_cmd_t *)gcmd;
-
-	@pre_print_cb_args@
-	debug( "%s[% 2d] Callback enter @name@@print_fmt_cb_args@\n", cmd->ctx->label, cmd->tag@print_pass_cb_args@ );
-	@print_cb_args@
-	cmd->callback( @pass_cb_args@cmd->callback_aux );
-	debug( "%s[% 2d] Callback leave @name@\n", cmd->ctx->label, cmd->tag );
-}
-
 static void
 proxy_@name@_cb( @decl_cb_args@void *aux )
 {
 	@name@_cmd_t *cmd = (@name@_cmd_t *)aux;
+	proxy_store_t *ctx = cmd->ctx;
 
-	@save_cb_args@
-	proxy_invoke_cb( @gen_cmd@, proxy_do_@name@_cb, @checked@, "@name@" );
+	@count_step@
+	@print_cb_args_tpl@
+	cmd->callback( @pass_cb_args@cmd->callback_aux );
+	debug( "%s[% 2d] Callback leave @name@\n", ctx->label, cmd->tag );
+	proxy_cmd_done( &cmd->gen );
+}
+
+static void
+proxy_do_@name@( gen_cmd_t *gcmd )
+{
+	@name@_cmd_t *cmd = (@name@_cmd_t *)gcmd;
+	proxy_store_t *ctx = cmd->ctx;
+
+	@pre_print_args@
+	debug( "%s[% 2d] Enter @name@@print_fmt_dry@@print_fmt_args@\n", ctx->label, cmd->tag@print_pass_dry@@print_pass_args@ );
+	@print_args@
+	@pre_invoke@
+	@indent_invoke@ctx->real_driver->@name@( ctx->real_store@pass_args@, proxy_@name@_cb, cmd );
+	@post_invoke@
+	debug( "%s[% 2d] Leave @name@\n", ctx->label, cmd->tag );
 }
 
 static @type@proxy_@name@( store_t *gctx@decl_args@, void (*cb)( @decl_cb_args@void *aux ), void *aux )
@@ -273,76 +249,118 @@ static @type@proxy_@name@( store_t *gctx@decl_args@, void (*cb)( @decl_cb_args@v
 	proxy_store_t *ctx = (proxy_store_t *)gctx;
 
 	@name@_cmd_t *cmd = (@name@_cmd_t *)proxy_cmd_new( ctx, sizeof(@name@_cmd_t) );
+	cmd->queued_cb = proxy_do_@name@;
 	cmd->callback = cb;
 	cmd->callback_aux = aux;
 	@assign_state@
-	@pre_print_args@
-	debug( "%s[% 2d] Enter @name@@print_fmt_args@\n", ctx->label, cmd->tag@print_pass_args@ );
-	@print_args@
-	ctx->real_driver->@name@( ctx->real_store@pass_args@, proxy_@name@_cb, cmd );
-	debug( "%s[% 2d] Leave @name@\n", ctx->label, cmd->tag );
-	proxy_cmd_done( @gen_cmd@ );
+	proxy_invoke( &cmd->gen, @checked@, "@name@" );
 }
 //# END
 
 //# UNDEFINE list_store_print_fmt_cb_args
 //# UNDEFINE list_store_print_pass_cb_args
 //# DEFINE list_store_print_cb_args
-	if (cmd->sts == DRV_OK) {
-		for (string_list_t *box = cmd->boxes; box; box = box->next)
-			debug( "  %s\n", box->string );
-	}
+	for (string_list_t *box = boxes; box; box = box->next)
+		debug( "  %s\n", box->string );
 //# END
 
+//# DEFINE select_box_pre_invoke
+	ctx->is_fake = 0;
+//# END
+
+//# DEFINE create_box_driable 1
+//# DEFINE create_box_fake_invoke
+	ctx->is_fake = 1;
+//# END
+//# DEFINE create_box_counted 1
+
+//# DEFINE open_box_fakeable 1
+//# DEFINE open_box_fake_invoke
+	ctx->fake_nextuid = 1;
+//# END
+//# DEFINE open_box_fake_cb_args , 1
+
+//# DEFINE get_uidnext_fakeable 1
+//# DEFINE get_uidnext_fake_invoke
+	rv = ctx->fake_nextuid;
+//# END
+//# DEFINE get_uidnext_post_real_invoke
+	ctx->fake_nextuid = rv;
+//# END
+
+//# DEFINE get_supported_flags_fakeable 1
+//# DEFINE get_supported_flags_fake_invoke
+	rv = 255;
+//# END
+
+//# DEFINE confirm_box_empty_fakeable 1
+//# DEFINE confirm_box_empty_fake_invoke
+	rv = 1;
+//# END
+
+//# DEFINE delete_box_driable 1
+//# DEFINE delete_box_fake_invoke
+	ctx->is_fake = 0;
+//# END
+//# DEFINE delete_box_counted 1
+
+//# DEFINE finish_delete_box_driable 1
+//# DEFINE finish_delete_box_fake_invoke
+	rv = DRV_OK;
+//# END
+
+//# DEFINE prepare_load_box_print_fmt_args , opts=%s
+//# DEFINE prepare_load_box_print_pass_args , fmt_opts( opts ).str
+//# DEFINE prepare_load_box_print_fmt_ret %s
+//# DEFINE prepare_load_box_print_pass_ret fmt_opts( rv ).str
+
 //# DEFINE load_box_pre_print_args
-	static char ubuf[12];
+	char ubuf[12];
 //# END
 //# DEFINE load_box_print_fmt_args , [%u,%s] (find >= %u, paired <= %u, new > %u)
-//# DEFINE load_box_print_pass_args , minuid, (maxuid == UINT_MAX) ? "inf" : (nfsnprintf( ubuf, sizeof(ubuf), "%u", maxuid ), ubuf), finduid, pairuid, newuid
+//# DEFINE load_box_print_pass_args , cmd->minuid, (cmd->maxuid == UINT_MAX) ? "inf" : (nfsnprintf( ubuf, sizeof(ubuf), "%u", cmd->maxuid ), ubuf), cmd->finduid, cmd->pairuid, cmd->newuid
 //# DEFINE load_box_print_args
-	if (excs.size) {
+	if (cmd->excs.size) {
 		debugn( "  excs:" );
-		for (uint t = 0; t < excs.size; t++)
-			debugn( " %u", excs.data[t] );
+		for (uint t = 0; t < cmd->excs.size; t++)
+			debugn( " %u", cmd->excs.data[t] );
 		debug( "\n" );
 	}
 //# END
-//# DEFINE load_box_print_fmt_cb_args , sts=%d, total=%d, recent=%d
-//# DEFINE load_box_print_pass_cb_args , cmd->sts, cmd->total_msgs, cmd->recent_msgs
+//# DEFINE load_box_fakeable 1
+//# DEFINE load_box_fake_cb_args , NULL, 0, 0
+//# DEFINE load_box_print_fmt_cb_args , total=%d, recent=%d
+//# DEFINE load_box_print_pass_cb_args , total_msgs, recent_msgs
 //# DEFINE load_box_print_cb_args
-	if (cmd->sts == DRV_OK) {
-		static char fbuf[as(Flags) + 1];
-		for (message_t *msg = cmd->msgs; msg; msg = msg->next)
-			debug( "  uid=%-5u flags=%-4s size=%-6u tuid=%." stringify(TUIDL) "s\n",
-			       msg->uid, (msg->status & M_FLAGS) ? (proxy_make_flags( msg->flags, fbuf ), fbuf) : "?", msg->size, *msg->tuid ? msg->tuid : "?" );
+	for (message_t *msg = msgs; msg; msg = msg->next) {
+		if (msg->status & M_DEAD)
+			continue;
+		debug( "  uid=%-5u flags=%-4s size=%-6u tuid=%." stringify(TUIDL) "s\n",
+		       msg->uid, (msg->status & M_FLAGS) ? fmt_flags( msg->flags ).str : "?", msg->size, *msg->tuid ? msg->tuid : "?" );
 	}
 //# END
 
-//# DEFINE find_new_msgs_print_fmt_cb_args , sts=%d
-//# DEFINE find_new_msgs_print_pass_cb_args , cmd->sts
+//# UNDEFINE find_new_msgs_print_fmt_cb_args
+//# UNDEFINE find_new_msgs_print_pass_cb_args
 //# DEFINE find_new_msgs_print_cb_args
-	if (cmd->sts == DRV_OK) {
-		for (message_t *msg = cmd->msgs; msg; msg = msg->next)
-			debug( "  uid=%-5u tuid=%." stringify(TUIDL) "s\n", msg->uid, msg->tuid );
+	for (message_t *msg = msgs; msg; msg = msg->next) {
+		if (msg->status & M_DEAD)
+			continue;
+		debug( "  uid=%-5u tuid=%." stringify(TUIDL) "s\n", msg->uid, msg->tuid );
 	}
 //# END
 
-//# DEFINE fetch_msg_decl_state
-	msg_data_t *data;
-//# END
-//# DEFINE fetch_msg_assign_state
-	cmd->data = data;
-//# END
 //# DEFINE fetch_msg_print_fmt_args , uid=%u, want_flags=%s, want_date=%s
-//# DEFINE fetch_msg_print_pass_args , msg->uid, !(msg->status & M_FLAGS) ? "yes" : "no", data->date ? "yes" : "no"
-//# DEFINE fetch_msg_pre_print_cb_args
-	static char fbuf[as(Flags) + 1];
-	proxy_make_flags( cmd->data->flags, fbuf );
+//# DEFINE fetch_msg_print_pass_args , cmd->msg->uid, !(cmd->msg->status & M_FLAGS) ? "yes" : "no", cmd->data->date ? "yes" : "no"
+//# DEFINE fetch_msg_driable 1
+//# DEFINE fetch_msg_fake_invoke
+	cmd->data->data = strdup( "" );
+	cmd->data->len = 0;
 //# END
 //# DEFINE fetch_msg_print_fmt_cb_args , flags=%s, date=%lld, size=%u
-//# DEFINE fetch_msg_print_pass_cb_args , fbuf, (long long)cmd->data->date, cmd->data->len
+//# DEFINE fetch_msg_print_pass_cb_args , fmt_flags( cmd->data->flags ).str, (long long)cmd->data->date, cmd->data->len
 //# DEFINE fetch_msg_print_cb_args
-	if (cmd->sts == DRV_OK && (DFlags & DEBUG_DRV_ALL)) {
+	if (DFlags & DEBUG_DRV_ALL) {
 		printf( "%s=========\n", cmd->ctx->label );
 		fwrite( cmd->data->data, cmd->data->len, 1, stdout );
 		printf( "%s=========\n", cmd->ctx->label );
@@ -350,64 +368,78 @@ static @type@proxy_@name@( store_t *gctx@decl_args@, void (*cb)( @decl_cb_args@v
 	}
 //# END
 
-//# DEFINE store_msg_pre_print_args
-	static char fbuf[as(Flags) + 1];
-	proxy_make_flags( data->flags, fbuf );
-//# END
 //# DEFINE store_msg_print_fmt_args , flags=%s, date=%lld, size=%u, to_trash=%s
-//# DEFINE store_msg_print_pass_args , fbuf, (long long)data->date, data->len, to_trash ? "yes" : "no"
+//# DEFINE store_msg_print_pass_args , fmt_flags( cmd->data->flags ).str, (long long)cmd->data->date, cmd->data->len, cmd->to_trash ? "yes" : "no"
 //# DEFINE store_msg_print_args
 	if (DFlags & DEBUG_DRV_ALL) {
 		printf( "%s>>>>>>>>>\n", ctx->label );
-		fwrite( data->data, data->len, 1, stdout );
+		fwrite( cmd->data->data, cmd->data->len, 1, stdout );
 		printf( "%s>>>>>>>>>\n", ctx->label );
 		fflush( stdout );
 	}
 //# END
+//# DEFINE store_msg_driable 1
+//# DEFINE store_msg_fake_cb_args , cmd->to_trash ? 0 : ctx->fake_nextuid++
+//# DEFINE store_msg_counted 1
 
-//# DEFINE set_msg_flags_pre_print_args
-	static char fbuf1[as(Flags) + 1], fbuf2[as(Flags) + 1];
-	proxy_make_flags( add, fbuf1 );
-	proxy_make_flags( del, fbuf2 );
-//# END
+//# DEFINE set_msg_flags_checked 1
 //# DEFINE set_msg_flags_print_fmt_args , uid=%u, add=%s, del=%s
-//# DEFINE set_msg_flags_print_pass_args , uid, fbuf1, fbuf2
-//# DEFINE set_msg_flags_checked sts == DRV_OK
+//# DEFINE set_msg_flags_print_pass_args , cmd->uid, fmt_flags( cmd->add ).str, fmt_flags( cmd->del ).str
+//# DEFINE set_msg_flags_driable 1
+//# DEFINE set_msg_flags_counted 1
 
 //# DEFINE trash_msg_print_fmt_args , uid=%u
-//# DEFINE trash_msg_print_pass_args , msg->uid
+//# DEFINE trash_msg_print_pass_args , cmd->msg->uid
+//# DEFINE trash_msg_driable 1
+//# DEFINE trash_msg_counted 1
+
+//# DEFINE close_box_driable 1
+//# DEFINE close_box_fake_cb_args , 0
+//# DEFINE close_box_counted 1
 
 //# DEFINE commit_cmds_print_args
 	proxy_flush_checked_cmds( ctx );
 //# END
 
 //# DEFINE cancel_cmds_print_cb_args
-	proxy_cancel_checked_cmds( cmd->ctx );
+	proxy_cancel_queued_cmds( ctx );
 //# END
 
 //# DEFINE free_store_print_args
-	proxy_cancel_checked_cmds( ctx );
+	proxy_cancel_queued_cmds( ctx );
 //# END
 //# DEFINE free_store_action
 	proxy_store_deref( ctx );
 //# END
 
 //# DEFINE cancel_store_print_args
-	proxy_cancel_checked_cmds( ctx );
+	proxy_cancel_queued_cmds( ctx );
 //# END
 //# DEFINE cancel_store_action
 	proxy_store_deref( ctx );
 //# END
 #endif
 
-//# SPECIAL set_bad_callback
+//# SPECIAL set_callbacks
 static void
-proxy_set_bad_callback( store_t *gctx, void (*cb)( void *aux ), void *aux )
+proxy_set_callbacks( store_t *gctx, void (*exp_cb)( message_t *, void * ),
+                     void (*bad_cb)( void * ), void *aux )
 {
 	proxy_store_t *ctx = (proxy_store_t *)gctx;
 
-	ctx->bad_callback = cb;
-	ctx->bad_callback_aux = aux;
+	ctx->expunge_callback = exp_cb;
+	ctx->bad_callback = bad_cb;
+	ctx->callback_aux = aux;
+}
+
+static void
+proxy_invoke_expunge_callback( message_t *msg, proxy_store_t *ctx )
+{
+	ctx->ref_count++;
+	debug( "%sCallback enter expunged message %u\n", ctx->label, msg->uid );
+	ctx->expunge_callback( msg, ctx->callback_aux );
+	debug( "%sCallback leave expunged message %u\n", ctx->label, msg->uid );
+	proxy_store_deref( ctx );
 }
 
 static void
@@ -415,27 +447,30 @@ proxy_invoke_bad_callback( proxy_store_t *ctx )
 {
 	ctx->ref_count++;
 	debug( "%sCallback enter bad store\n", ctx->label );
-	ctx->bad_callback( ctx->bad_callback_aux );
+	ctx->bad_callback( ctx->callback_aux );
 	debug( "%sCallback leave bad store\n", ctx->label );
 	proxy_store_deref( ctx );
 }
 
 //# EXCLUDE alloc_store
 store_t *
-proxy_alloc_store( store_t *real_ctx, const char *label )
+proxy_alloc_store( store_t *real_ctx, const char *label, int force_async )
 {
 	proxy_store_t *ctx;
 
-	ctx = nfcalloc( sizeof(*ctx) );
+	ctx = nfzalloc( sizeof(*ctx) );
 	ctx->driver = &proxy_driver;
 	ctx->gen.conf = real_ctx->conf;
 	ctx->ref_count = 1;
 	ctx->label = label;
-	ctx->done_cmds_append = &ctx->done_cmds;
+	ctx->force_async = force_async;
+	ctx->pending_cmds_append = &ctx->pending_cmds;
 	ctx->check_cmds_append = &ctx->check_cmds;
 	ctx->real_driver = real_ctx->driver;
 	ctx->real_store = real_ctx;
-	ctx->real_driver->set_bad_callback( ctx->real_store, (void (*)(void *))proxy_invoke_bad_callback, ctx );
+	ctx->real_driver->set_callbacks( ctx->real_store,
+	                                 (void (*)( message_t *, void * ))proxy_invoke_expunge_callback,
+	                                 (void (*)( void * ))proxy_invoke_bad_callback, ctx );
 	init_wakeup( &ctx->wakeup, proxy_wakeup, ctx );
 	return &ctx->gen;
 }

src/drv_proxy_gen.pl

@@ -1,23 +1,9 @@
 #!/usr/bin/perl
 #
+# SPDX-FileCopyrightText: 2017-2022 Oswald Buddenhagen <ossi@users.sf.net>
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
 # mbsync - mailbox synchronizer
-# Copyright (C) 2017 Oswald Buddenhagen <ossi@users.sf.net>
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# As a special exception, mbsync may be linked with the OpenSSL library,
-# despite that library's more restrictive license.
 #
 
 use strict;
@@ -91,7 +77,7 @@ while (<$inh>) {
 }
 close($inh);
 
-$cont =~ s,\n, ,g;
+$cont =~ s,(?://.*)?\n, ,g;
 $cont =~ s,/\*.*?\*/, ,g;
 $cont =~ s,\h+, ,g;
 my @ptypes = map { s,^ ,,r } split(/;/, $cont);
@@ -140,6 +126,7 @@ for (@ptypes) {
 	}
 	push @cmd_table, "proxy_$cmd_name";
 	next if (defined($special{$cmd_name}));
+	my $inc_tpl = "";
 	my %replace;
 	$replace{'name'} = $cmd_name;
 	$replace{'type'} = $cmd_type;
@@ -148,44 +135,106 @@ for (@ptypes) {
 		$template = "GETTER";
 		$replace{'fmt'} = type_to_format($cmd_type);
 	} else {
+		my $pass_args;
 		if ($cmd_type eq "void " && $cmd_args =~ s/, void \(\*cb\)\( (.*)void \*aux \), void \*aux$//) {
 			my $cmd_cb_args = $1;
+			$replace{'decl_cb_args'} = $cmd_cb_args;
+			$replace{'pass_cb_args'} = make_args($cmd_cb_args);
 			if (length($cmd_cb_args)) {
-				$replace{'decl_cb_args'} = $cmd_cb_args;
 				my $r_cmd_cb_args = $cmd_cb_args;
 				$r_cmd_cb_args =~ s/^int sts, // or die("Callback arguments of $cmd_name don't start with sts.\n");
-				$replace{'decl_cb_state'} = $r_cmd_cb_args =~ s/, /\;\n/gr;
-				my $pass_cb_args = make_args($cmd_cb_args);
-				$replace{'save_cb_args'} = $pass_cb_args =~ s/([^,]+), /cmd->$1 = $1\;\n/gr;
-				$pass_cb_args =~ s/([^, ]+)/cmd->$1/g;
-				$replace{'pass_cb_args'} = $pass_cb_args;
-				$replace{'print_pass_cb_args'} = $pass_cb_args =~ s/(.*), $/, $1/r;
-				$replace{'print_fmt_cb_args'} = make_format($cmd_cb_args =~ s/(.*), $/, $1/r);
-				$replace{'gen_cmd_t'} = "gen_sts_cmd_t";
-				$replace{'GEN_CMD'} = "GEN_STS_CMD\n";
-				$replace{'gen_cmd'} = "&cmd->gen.gen";
+				$r_cmd_cb_args =~ s/^(.*), $/, $1/;
+				$replace{'print_pass_cb_args'} = make_args($r_cmd_cb_args);
+				$replace{'print_fmt_cb_args'} = make_format($r_cmd_cb_args);
+				$inc_tpl = 'CALLBACK_STS';
 			} else {
-				$replace{'gen_cmd_t'} = "gen_cmd_t";
-				$replace{'GEN_CMD'} = "GEN_CMD\n";
-				$replace{'gen_cmd'} = "&cmd->gen";
+				$inc_tpl = 'CALLBACK_VOID';
 			}
-			$replace{'checked'} //= '0';
+
+			$pass_args = make_args($cmd_args);
+			$pass_args =~ s/([^, ]+)/cmd->$1/g;
+			my $r_cmd_args = $cmd_args =~ s/, (.*)$/$1, /r;
+			$replace{'decl_state'} = $r_cmd_args =~ s/, /\;\n/gr;
+			my $r_pass_args = make_args($r_cmd_args);
+			$replace{'assign_state'} = $r_pass_args =~ s/([^,]+), /cmd->$1 = $1\;\n/gr;
+
+			$replace{'checked'} = '0';
 			$template = "CALLBACK";
-		} elsif ($cmd_type eq "void ") {
-			$template = "REGULAR_VOID";
 		} else {
-			$template = "REGULAR";
-			$replace{'fmt'} = type_to_format($cmd_type);
+			$pass_args = make_args($cmd_args);
+
+			if ($cmd_type eq "void ") {
+				$template = "REGULAR_VOID";
+			} else {
+				$template = "REGULAR";
+				$replace{'print_fmt_ret'} = type_to_format($cmd_type);
+				$replace{'print_pass_ret'} = "rv";
+			}
 		}
 		$replace{'decl_args'} = $cmd_args;
-		$replace{'print_pass_args'} = $replace{'pass_args'} = make_args($cmd_args);
+		$replace{'print_pass_args'} = $replace{'pass_args'} = $pass_args;
 		$replace{'print_fmt_args'} = make_format($cmd_args);
 	}
+	my ($fake_cond, $fake_invoke, $fake_cb_args, $post_invoke) = (undef, "", "", "");
 	for (keys %defines) {
-		$replace{$1} = delete $defines{$_} if (/^${cmd_name}_(.*)$/);
+		next if (!/^${cmd_name}_(.*)$/);
+		my ($key, $val) = ($1, delete $defines{$_});
+		if ($key eq 'counted') {
+			$replace{count_step} = "countStep();\n";
+		} elsif ($key eq 'fakeable') {
+			$fake_cond = "ctx->is_fake";
+			$replace{print_pass_dry} = ', '.$fake_cond.' ? " [FAKE]" : ""';
+		} elsif ($key eq 'driable') {
+			$fake_cond = "DFlags & DRYRUN";
+			$replace{print_pass_dry} = ', ('.$fake_cond.') ? " [DRY]" : ""';
+		} elsif ($key eq 'fake_invoke') {
+			$fake_invoke = $val;
+		} elsif ($key eq 'fake_cb_args') {
+			$fake_cb_args = $val;
+		} elsif ($key eq 'post_real_invoke') {
+			$post_invoke = $val;
+		} else {
+			$replace{$key} = $val;
+		}
+	}
+	if (defined($fake_cond)) {
+		$replace{print_fmt_dry} = '%s';
+		if ($inc_tpl eq 'CALLBACK_STS') {
+			$fake_invoke .= "proxy_${cmd_name}_cb( DRV_OK${fake_cb_args}, cmd );\n";
+		} elsif (length($fake_cb_args)) {
+			die("Unexpected fake callback arguments to $cmd_name\n");
+		}
+		my $num_fake = $fake_invoke =~ s/^(?=.)/\t/gsm;
+		my $num_real = $post_invoke =~ s/^(?=.)/\t/gsm;
+		my $pre_invoke = "if (".$fake_cond.")";
+		if ($num_fake > 1 || $num_real) {
+			$pre_invoke .= " {";
+			$fake_invoke .= "} else {\n";
+			$post_invoke .= "}\n";
+		} else {
+			$fake_invoke .= "else\n";
+		}
+		$replace{pre_invoke} = $pre_invoke."\n".$fake_invoke;
+		$replace{indent_invoke} = "\t";
+		$replace{post_invoke} = $post_invoke;
 	}
 	my %used;
 	my $text = $templates{$template};
+	if ($inc_tpl) {
+		if ($inc_tpl eq 'CALLBACK_STS') {
+			if ($replace{print_fmt_cb_args}) {
+				$inc_tpl .= '_FMT';
+			} else {
+				if ($replace{print_cb_args}) {
+					$inc_tpl .= '_PRN';
+				}
+				# These may be defined but empty; that's not an error.
+				delete $replace{print_fmt_cb_args};
+				delete $replace{print_pass_cb_args};
+			}
+		}
+		$text =~ s/^\t\@print_cb_args_tpl\@\n/$templates{$inc_tpl}/sm;
+	}
 	$text =~ s/^(\h*)\@(\w+)\@\n/$used{$2} = 1; indent($replace{$2} \/\/ "", $1)/smeg;
 	$text =~ s/\@(\w+)\@/$used{$1} = 1; $replace{$1} \/\/ ""/eg;
 	print $outh $text."\n";

src/imap_msgs.c

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#include "imap_p.h"
+
+#ifdef DEBUG_IMAP_MSGS
+# define dbg(...) print(__VA_ARGS__)
+#else
+# define dbg(...) do { } while (0)
+#endif
+
+imap_message_t *
+imap_new_msg( imap_messages_t *msgs )
+{
+	imap_message_t *msg = nfzalloc( sizeof(*msg) );
+	*msgs->tail = msg;
+	msgs->tail = &msg->next;
+	msgs->count++;
+	return msg;
+}
+
+void
+reset_imap_messages( imap_messages_t *msgs )
+{
+	free_generic_messages( &msgs->head->gen );
+	msgs->head = NULL;
+	msgs->tail = &msgs->head;
+	msgs->count = 0;
+	msgs->cursor_ptr = NULL;
+	msgs->cursor_seq = 0;
+}
+
+static int
+imap_compare_msgs( const void *a_, const void *b_ )
+{
+	const imap_message_t *a = *(const imap_message_t * const *)a_;
+	const imap_message_t *b = *(const imap_message_t * const *)b_;
+
+	if (a->uid < b->uid)
+		return -1;
+	if (a->uid > b->uid)
+		return 1;
+	return 0;
+}
+
+void
+imap_ensure_relative( imap_messages_t *msgs )
+{
+	if (msgs->cursor_ptr)
+		return;
+	uint count = msgs->count;
+	if (!count)
+		return;
+	if (count > 1) {
+		imap_message_t **t = nfmalloc( sizeof(*t) * count );
+
+		imap_message_t *m = msgs->head;
+		for (uint i = 0; i < count; i++) {
+			t[i] = m;
+			m = m->next;
+		}
+
+		qsort( t, count, sizeof(*t), imap_compare_msgs );
+
+		imap_message_t *nm = t[0];
+		msgs->head = nm;
+		nm->prev = NULL;
+		uint seq, nseq = nm->seq;
+		for (uint j = 0; m = nm, seq = nseq, j < count - 1; j++) {
+			nm = t[j + 1];
+			m->next = nm;
+			m->next->prev = m;
+			nseq = nm->seq;
+			nm->seq = nseq - seq;
+		}
+		msgs->tail = &m->next;
+		*msgs->tail = NULL;
+
+		free( t );
+	}
+	msgs->cursor_ptr = msgs->head;
+	msgs->cursor_seq = msgs->head->seq;
+}
+
+void
+imap_ensure_absolute( imap_messages_t *msgs )
+{
+	if (!msgs->cursor_ptr)
+		return;
+	uint seq = 0;
+	for (imap_message_t *msg = msgs->head; msg; msg = msg->next) {
+		seq += msg->seq;
+		msg->seq = seq;
+	}
+	msgs->cursor_ptr = NULL;
+	msgs->cursor_seq = 0;
+}
+
+imap_message_t *
+imap_expunge_msg( imap_messages_t *msgs, uint fseq )
+{
+	dbg( "expunge %u\n", fseq );
+	imap_ensure_relative( msgs );
+	imap_message_t *ret = NULL, *msg = msgs->cursor_ptr;
+	if (msg) {
+		uint seq = msgs->cursor_seq;
+		for (;;) {
+			dbg( "  now on message %u (uid %u), %sdead\n", seq, msg->uid, (msg->status & M_DEAD) ? "" : "not " );
+			if (seq == fseq && !(msg->status & M_DEAD)) {
+				dbg( "  => expunging\n" );
+				msg->status = M_DEAD;
+				ret = msg;
+				break;
+			}
+			if (seq < fseq) {
+				dbg( "    is below\n" );
+				if (!msg->next) {
+					dbg( "    no next\n" );
+					goto done;
+				}
+				msg = msg->next;
+				seq += msg->seq;
+			} else {
+				dbg( "    is not below\n" );
+				if (!msg->prev) {
+					dbg( "    no prev\n" );
+					break;
+				}
+				uint pseq = seq - msg->seq;
+				if (pseq < fseq) {
+					dbg( "    prev too low\n" );
+					break;
+				}
+				seq = pseq;
+				msg = msg->prev;
+			}
+		}
+		dbg( "  => lowering\n" );
+		assert( msg->seq );
+		msg->seq--;
+		seq--;
+	  done:
+		dbg( "  saving cursor on %u (uid %u)\n", seq, msg->uid );
+		msgs->cursor_ptr = msg;
+		msgs->cursor_seq = seq;
+	} else {
+		dbg( "  => no messages\n" );
+	}
+	return ret;
+}

src/imap_p.h

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#ifndef IMAP_P_H
+#define IMAP_P_H
+
+#include "driver.h"
+
+//#define DEBUG_IMAP_MSGS
+//#define DEBUG_IMAP_UTF7
+
+typedef union imap_message {
+	message_t gen;
+	struct {
+		MESSAGE(union imap_message)
+
+		union imap_message *prev;  // Used to optimize lookup by seq.
+		// This is made relative once the fetches complete - to avoid that
+		// each expunge re-enumerates all subsequent messages. Dead messages
+		// "occupy" no sequence number themselves, but may still jump a gap.
+		// Note that use of sequence numbers to address messages in commands
+		// imposes limitations on permissible pipelining. We don't do that,
+		// so this is of no concern; however, we might miss the closing of
+		// a gap, which would result in a tiny performance hit.
+		uint seq;
+	};
+} imap_message_t;
+
+typedef struct {
+	imap_message_t *head;
+	imap_message_t **tail;
+	// Bulk changes (which is where performance matters) are assumed to be
+	// reported sequentially (be it forward or reverse), so walking the
+	// sorted linked list from the previously used message is efficient.
+	imap_message_t *cursor_ptr;
+	uint cursor_seq;
+	uint count;
+} imap_messages_t;
+
+imap_message_t *imap_new_msg( imap_messages_t *msgs );
+imap_message_t *imap_expunge_msg( imap_messages_t *msgs, uint fseq );
+void reset_imap_messages( imap_messages_t *msgs );
+void imap_ensure_relative( imap_messages_t *msgs );
+void imap_ensure_absolute( imap_messages_t *msgs );
+
+char *imap_utf8_to_utf7( const char *buf );
+int imap_utf7_to_utf8( const char *buf, int argl, char *outbuf );
+
+#endif

src/imap_utf7.c

@@ -0,0 +1,288 @@
+// SPDX-FileCopyrightText: 2018-2021 Georgy Kibardin <georgy@kibardin.name>
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#include "imap_p.h"
+
+#ifdef DEBUG_IMAP_UTF7
+# define dbg(...) print(__VA_ARGS__)
+#else
+# define dbg(...) do { } while (0)
+#endif
+
+struct bit_fifo {
+	unsigned long long value;
+	uint bits;
+};
+
+static void
+add_bits( struct bit_fifo *fifo, uint bits, uint size )
+{
+	fifo->value = (fifo->value << size) | bits;
+	fifo->bits += size;
+	assert( fifo->bits <= sizeof(fifo->value) * 8 );
+}
+
+static uint
+eat_bits( struct bit_fifo *fifo, uint size )
+{
+	fifo->bits -= size;
+	return (fifo->value >> fifo->bits) & ((1LL << size) - 1);
+}
+
+static uint
+peek_bits( struct bit_fifo *fifo, uint size )
+{
+	return (fifo->value >> (fifo->bits - size)) & ((1LL << size) - 1);
+}
+
+static void
+add_char( char **p, uint chr )
+{
+	*((*p)++) = (char)chr;
+}
+
+static uchar
+eat_char( const char **p )
+{
+	return (uchar)*((*p)++);
+}
+
+static uint
+read_as_utf8( const char **utf8_buf_p )
+{
+	uchar chr = eat_char( utf8_buf_p );
+	if (chr < 0x80)
+		return chr;
+	if ((chr & 0xf8) == 0xf0) {
+		uchar chr2 = eat_char( utf8_buf_p );
+		if ((chr2 & 0xc0) != 0x80)
+			return ~0;
+		uchar chr3 = eat_char( utf8_buf_p );
+		if ((chr3 & 0xc0) != 0x80)
+			return ~0;
+		uchar chr4 = eat_char( utf8_buf_p );
+		if ((chr4 & 0xc0) != 0x80)
+			return ~0;
+		return ((chr & 0x7) << 18) |
+		       ((chr2 & 0x3f) << 12) |
+		       ((chr3 & 0x3f) << 6) |
+		       (chr4 & 0x3f);
+	}
+	if ((chr & 0xf0) == 0xe0) {
+		uchar chr2 = eat_char( utf8_buf_p );
+		if ((chr2 & 0xc0) != 0x80)
+			return ~0;
+		uchar chr3 = eat_char( utf8_buf_p );
+		if ((chr3 & 0xc0) != 0x80)
+			return ~0;
+		return ((chr & 0xf) << 12) |
+		       ((chr2 & 0x3f) << 6) |
+		       (chr3 & 0x3f);
+	}
+	if ((chr & 0xe0) == 0xc0) {
+		uchar chr2 = eat_char( utf8_buf_p );
+		if ((chr2 & 0xc0) != 0x80)
+			return ~0;
+		return (chr & 0x1f) << 6 |
+		       (chr2 & 0x3f);
+	}
+	return ~0;
+}
+
+static int
+needs_encoding( uint chr )
+{
+	return chr && (chr <= 0x1f || chr >= 0x7f);
+}
+
+static uint
+utf16_encode( uint chr )
+{
+	chr -= 0x10000;
+	return (((chr >> 10) + 0xd800) << 16) | ((chr & 0x3ff) + 0xdc00);
+}
+
+static uchar
+b64_encode( uint chr )
+{
+	assert( chr <= 0x3f );
+	return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"[chr];
+}
+
+char *
+imap_utf8_to_utf7( const char *buf )
+{
+	// Size requirements:
+	// - pass-through: l, 1 => 1
+	// - all "&": l * 2, 1 => 2
+	// - 7-bit: (l * 2 * 4 + 2) / 3 + 2, ~ l * 2.7, 1 => 5
+	// - 3-octet: (l / 3 * 2 * 4 + 2) / 3 + 2, ~ l * 0.9, 3 => 5
+	// - 4-octet: (l / 4 * 2 * 2 * 4 + 2) / 3 + 2, ~ l * 1.3, 4 => 8
+	// => worst case: "&" and 7-bit alternating: l * 3.5, 2 => 7
+	int outsz = strlen( buf ) * 7 / 2 + 3;
+	char *result = nfmalloc( outsz );
+	char *outp = result;
+	struct bit_fifo fifo = { 0, 0 };
+	int encoding = 0;
+	uint chr;
+	do {
+		chr = read_as_utf8( &buf );
+		if (chr == ~0U) {
+			dbg( "Error: invalid UTF-8 string\n" );
+			free( result );
+			return NULL;
+		}
+		if (needs_encoding( chr )) {
+			if (!encoding) {
+				add_char( &outp, '&' );
+				encoding = 1;
+			}
+			if (chr <= 0xffff)
+				add_bits( &fifo, chr, 16 );
+			else
+				add_bits( &fifo, utf16_encode( chr ), 32 );
+			while (fifo.bits >= 6)
+				add_char( &outp, b64_encode( eat_bits( &fifo, 6 ) ) );
+		} else {
+			if (encoding) {
+				if (fifo.bits) {
+					uint trailing_bits = 6 - fifo.bits;
+					uchar trail = b64_encode( eat_bits( &fifo, fifo.bits ) << trailing_bits );
+					add_char( &outp, trail );
+				}
+				add_char( &outp, '-' );
+				encoding = 0;
+			}
+			add_char( &outp, chr );
+			if (chr == '&')
+				add_char( &outp, '-' );
+		}
+	} while (chr);
+	assert( (int)(outp - result) <= outsz );
+	return result;
+}
+
+static void
+write_as_utf8( char **outp, uint chr )
+{
+	if (chr <= 0x7f) {
+		add_char( outp, chr );
+	} else if (chr <= 0x7ff) {
+		add_char( outp, (chr >> 6) | 0xc0 );
+		add_char( outp, (chr & 0x3f) | 0x80 );
+	} else if (chr <= 0xffff) {
+		add_char( outp, (chr >> 12) | 0xe0 );
+		add_char( outp, ((chr >> 6) & 0x3f) | 0x80 );
+		add_char( outp, (chr & 0x3f) | 0x80 );
+	} else {
+		assert( chr <= 0xfffff );
+		add_char( outp, (chr >> 18) | 0xf0 );
+		add_char( outp, ((chr >> 12) & 0x3f) | 0x80 );
+		add_char( outp, ((chr >> 6) & 0x3f) | 0x80 );
+		add_char( outp, (chr & 0x3f) | 0x80 );
+	}
+}
+
+static int
+need_another_16bit( uint bits )
+{
+	return (bits & 0xfc00) == 0xd800;
+}
+
+static uint
+utf16_decode( uint subject )
+{
+	return 0x10000 + (((subject >> 16) - 0xd800) << 10) + ((subject & 0xffff) - 0xdc00);
+}
+
+static uint
+b64_decode( uchar chr )
+{
+	static uint lu[128] = {
+		~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0,
+		~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0,
+		~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 62, 63, ~0, ~0, ~0,
+		52, 53, 54, 55, 56, 57, 58, 59, 60, 61, ~0, ~0, ~0, ~0, ~0, ~0,
+		~0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+		15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, ~0, ~0, ~0, ~0, ~0,
+		~0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+		41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, ~0, ~0, ~0, ~0, ~0,
+	};
+	return lu[chr];
+}
+
+int
+imap_utf7_to_utf8( const char *buf, int bufl, char *outbuf )
+{
+	// Size requirements:
+	// - pass-through: l (shortest worst case)
+	// - all "&": l / 2, 2 => 1, * .5
+	// - 7-bit: ((l - 2) * 3 + 1) / 4 / 2, ~ l * .38, 5 => 1, * .2
+	// - 3-octet: ((l - 2) * 3 + 1) / 4 / 2 * 3, ~ l * 1.13, 5 => 3, * .6 (generic worst case)
+	// - 4-octet: ((l - 2) * 3 + 1) / 4 / 2 / 2 * 4, ~ l * .75, 8 => 4, * .5
+	// => reserve bufl * 9 / 8
+	char *outp = outbuf;
+	struct bit_fifo fifo = { 0, 0 };
+	const char *bufe = buf + bufl;
+	while (buf != bufe) {
+		uchar chr = *buf++;
+		if (chr != '&') {
+			if (chr & 0x80) {
+				dbg( "Error: 8-bit char %x\n", chr );
+				return -1;
+			}
+			add_char( &outp, chr );
+			continue;
+		}
+		if (buf == bufe) {
+			dbg( "Error: unterminated shift sequence\n" );
+			return -1;
+		}
+		chr = *buf++;
+		if (chr == '-') {
+			add_char( &outp, '&' );
+			continue;
+		}
+		fifo.bits = 0;
+		do {
+			if (chr & 0x80) {
+				dbg( "Error: 8-bit char %x\n", chr );
+				return -1;
+			}
+			uint bits = b64_decode( chr );
+			if (bits == ~0U) {
+				dbg( "Error: char %x outside alphabet\n", chr );
+				return -1;
+			}
+			add_bits( &fifo, bits, 6 );
+			if (fifo.bits >= 16) {
+				if (need_another_16bit( peek_bits( &fifo, 16 ) )) {
+					if (fifo.bits >= 32) {
+						uint utf16 = eat_bits( &fifo, 32 );
+						if ((utf16 & 0xfc00) != 0xdc00) {
+							dbg( "Error: unpaired UTF-16 surrogate\n" );
+							return -1;
+						}
+						write_as_utf8( &outp, utf16_decode( utf16 ) );
+					}
+				} else {
+					write_as_utf8( &outp, eat_bits( &fifo, 16 ) );
+				}
+			}
+			if (buf == bufe) {
+				dbg( "Error: unterminated shift sequence\n" );
+				return -1;
+			}
+			chr = *buf++;
+		} while (chr != '-');
+		if (fifo.bits > 6) {
+			dbg( "Error: incomplete code point\n" );
+			return -1;
+		}
+	}
+	return (int)(outp - outbuf);
+}

src/main_list.c

@@ -0,0 +1,188 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#include "main_p.h"
+
+typedef struct store_ent {
+	struct store_ent *next;
+	store_conf_t *conf;
+} store_ent_t;
+
+typedef struct {
+	core_vars_t *cvars;
+	store_conf_t *store;
+	driver_t *drv;
+	store_t *ctx;
+	store_ent_t *storeptr;
+	int cben, done;
+} list_vars_t;
+
+static store_ent_t *
+add_store( store_ent_t ***storeapp, store_conf_t *store )
+{
+	store_ent_t *se = nfzalloc( sizeof(*se) );
+	se->conf = store;
+	**storeapp = se;
+	*storeapp = &se->next;
+	return se;
+}
+
+static void do_list_stores( list_vars_t *lvars );
+static void list_next_store( list_vars_t *lvars );
+
+void
+list_stores( core_vars_t *cvars, char **argv )
+{
+	list_vars_t lvars[1];
+	store_ent_t *strs = NULL, **strapp = &strs;
+	store_conf_t *store;
+
+	memset( lvars, 0, sizeof(*lvars) );
+	lvars->cvars = cvars;
+
+	if (!stores) {
+		fputs( "No stores defined.\n", stderr );
+		cvars->ret = 1;
+		return;
+	}
+
+	if (!*argv) {  // Implicit --all
+		for (store = stores; store; store = store->next)
+			add_store( &strapp, store );
+	} else {
+		for (; *argv; argv++) {
+			for (store = stores; store; store = store->next) {
+				if (!strcmp( store->name, *argv )) {
+					add_store( &strapp, store );
+					goto gotstr;
+				}
+			}
+			error( "No store named '%s' defined.\n", *argv );
+			cvars->ret = 1;
+		  gotstr: ;
+		}
+	}
+	if (cvars->ret)
+		return;
+	lvars->storeptr = strs;
+
+	do_list_stores( lvars );
+	main_loop();
+}
+
+static void
+list_store_bad( void *aux )
+{
+	list_vars_t *lvars = (list_vars_t *)aux;
+
+	lvars->drv->cancel_store( lvars->ctx );
+	lvars->cvars->ret = 1;
+	list_next_store( lvars );
+}
+
+static void
+advance_store( list_vars_t *lvars )
+{
+	store_ent_t *nstr = lvars->storeptr->next;
+	free( lvars->storeptr );
+	lvars->storeptr = nstr;
+}
+
+static void list_store_connected( int sts, void *aux );
+
+static void
+do_list_stores( list_vars_t *lvars )
+{
+	while (lvars->storeptr) {
+		lvars->store = lvars->storeptr->conf;
+		lvars->drv = lvars->store->driver;
+		int st = lvars->drv->get_fail_state( lvars->store );
+		if (st != FAIL_TEMP) {
+			info( "Skipping %sfailed store %s.\n",
+				  (st == FAIL_WAIT) ? "temporarily " : "", lvars->store->name );
+			lvars->cvars->ret = 1;
+			goto next;
+		}
+
+		uint dcaps = lvars->drv->get_caps( NULL );
+		store_t *ctx = lvars->drv->alloc_store( lvars->store, "" );
+		if ((DFlags & DEBUG_DRV) || ((DFlags & FORCEASYNC(F)) && !(dcaps & DRV_ASYNC))) {
+			lvars->drv = &proxy_driver;
+			ctx = proxy_alloc_store( ctx, "", DFlags & FORCEASYNC(F) );
+		}
+		lvars->ctx = ctx;
+		lvars->drv->set_callbacks( ctx, NULL, list_store_bad, lvars );
+		info( "Opening store %s...\n", lvars->store->name );
+		lvars->cben = lvars->done = 0;
+		lvars->drv->connect_store( lvars->ctx, list_store_connected, lvars );
+		if (!lvars->done) {
+			lvars->cben = 1;
+			return;
+		}
+
+	  next:
+		advance_store( lvars );
+	}
+	cleanup_mainloop();
+}
+
+static void
+list_next_store( list_vars_t *lvars )
+{
+	if (lvars->cben) {
+		advance_store( lvars );
+		do_list_stores( lvars );
+	}
+}
+
+static void
+list_done_store( list_vars_t *lvars )
+{
+	lvars->done = 1;
+	lvars->drv->free_store( lvars->ctx );
+	list_next_store( lvars );
+}
+
+static void list_store_listed( int sts, string_list_t *boxes, void *aux );
+
+static void
+list_store_connected( int sts, void *aux )
+{
+	list_vars_t *lvars = (list_vars_t *)aux;
+
+	switch (sts) {
+	case DRV_CANCELED:
+		return;
+	case DRV_OK:
+		lvars->drv->list_store( lvars->ctx, LIST_INBOX | LIST_PATH_MAYBE, list_store_listed, lvars );
+		break;
+	default:
+		lvars->cvars->ret = 1;
+		list_done_store( lvars );
+		break;
+	}
+}
+
+static void
+list_store_listed( int sts, string_list_t *boxes, void *aux )
+{
+	list_vars_t *lvars = (list_vars_t *)aux;
+	string_list_t *box;
+
+	switch (sts) {
+	case DRV_CANCELED:
+		return;
+	case DRV_OK:
+		printf( "===== %s:\n", lvars->ctx->conf->name );
+		for (box = boxes; box; box = box->next)
+			puts( box->string );
+		break;
+	default:
+		lvars->cvars->ret = 1;
+		break;
+	}
+	list_done_store( lvars );
+}

src/main_p.h

@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#ifndef MAIN_P_H
+#define MAIN_P_H
+
+#define DEBUG_FLAG DEBUG_MAIN
+
+#include "sync.h"
+
+typedef struct {
+	int ret;
+	int all;
+	int list;
+	int list_stores;
+	int ops[2];
+} core_vars_t;
+
+void sync_chans( core_vars_t *cvars, char **argv );
+void list_stores( core_vars_t *cvars, char **argv );
+void cleanup_mainloop( void );
+
+#endif

src/main_sync.c

@@ -0,0 +1,795 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#include "main_p.h"
+
+#define nz(a, b) ((a) ? (a) : (b))
+
+static int ops_any[2], trash_any[2], expunge_any[2];
+static int chans_total, chans_done;
+static int boxes_total, boxes_done;
+
+static int stats_steps;
+static int64_t stats_stamp;
+static wakeup_t stats_wakeup;
+
+static void
+print_stats( void )
+{
+	char buf[3][80];
+	char *cs;
+	static int cols = -1;
+
+	if (cols < 0 && (!(cs = getenv( "COLUMNS" )) || !(cols = atoi( cs ))))
+		cols = 80;
+	int ll = sprintf( buf[2], "C: %d/%d  B: %d/%d", chans_done, chans_total, boxes_done, boxes_total );
+	int cls = (cols - ll - 10) / 2;
+	for (int t = 0; t < 2; t++) {
+		int l = sprintf( buf[t], "+%d/%d *%d/%d #%d/%d -%d/%d",
+		                 new_done[t], new_total[t],
+		                 flags_done[t], flags_total[t],
+		                 trash_done[t], trash_total[t],
+		                 expunge_done[t], expunge_total[t] );
+		if (l > cls)
+			buf[t][cls - 1] = '~';
+	}
+	progress( "\r%s  F: %.*s  N: %.*s", buf[2], cls, buf[0], cls, buf[1] );
+}
+
+static void
+stats_timeout( void *aux ATTR_UNUSED )
+{
+	if (stats_steps != -1) {
+		stats_steps = -1;
+		print_stats();
+	}
+	conf_wakeup( &stats_wakeup, 200 );
+}
+
+void
+stats( void )
+{
+	if (!(DFlags & PROGRESS))
+		return;
+
+	// If the main loop appears to be running, skip the sync path.
+	if (stats_steps < 0) {
+		stats_steps = -2;
+		return;
+	}
+
+	// Rate-limit the (somewhat) expensive timer queries.
+	if (++stats_steps < 10)
+		return;
+	stats_steps = 0;
+
+	int64_t now = get_now();
+	if (now < stats_stamp + 300)
+		return;
+	stats_stamp = now;
+
+	print_stats();
+}
+
+static void
+summary( void )
+{
+	if (Verbosity < TERSE)
+		return;
+
+	if (!boxes_done)
+		return;  // Shut up if we errored out early.
+
+	printf( "Processed %d box(es) in %d channel(s)", boxes_done, chans_done );
+	for (int t = 2; --t >= 0; ) {
+		if (ops_any[t])
+			printf( (DFlags & DRYRUN) ?
+			            ",\nwould %s %d new message(s) and %d flag update(s)" :
+			            ",\n%sed %d new message(s) and %d flag update(s)",
+			        str_hl[t], new_done[t], flags_done[t] );
+		if (trash_any[t])
+			printf( (DFlags & DRYRUN) ?
+			            ",\nwould move %d %s message(s) to trash" :
+			            ",\nmoved %d %s message(s) to trash",
+			        trash_done[t], str_fn[t] );
+		if (expunge_any[t])
+			printf( (DFlags & DRYRUN) ?
+			            ",\nwould expunge %d message(s) from %s" :
+			            ",\nexpunged %d message(s) from %s",
+			        expunge_done[t], str_fn[t] );
+	}
+	puts( "." );
+}
+
+static int
+matches( const char *t, const char *p )
+{
+	for (;;) {
+		if (!*p)
+			return !*t;
+		if (*p == '*') {
+			p++;
+			do {
+				if (matches( t, p ))
+					return 1;
+			} while (*t++);
+			return 0;
+		} else if (*p == '%') {
+			p++;
+			do {
+				if (*t == '/')
+					return 0;
+				if (matches( t, p ))
+					return 1;
+			} while (*t++);
+			return 0;
+		} else {
+			if (*p != *t)
+				return 0;
+			p++, t++;
+		}
+	}
+}
+
+
+static int
+is_inbox( const char *name )
+{
+	return starts_with( name, -1, "INBOX", 5 ) && (!name[5] || name[5] == '/');
+}
+
+static int
+cmp_box_names( const void *a, const void *b )
+{
+	const char *as = *(const char * const *)a;
+	const char *bs = *(const char * const *)b;
+	int ai = is_inbox( as );
+	int bi = is_inbox( bs );
+	int di = bi - ai;
+	if (di)
+		return di;
+	return strcmp( as, bs );
+}
+
+static char **
+filter_boxes( string_list_t *boxes, const char *prefix, string_list_t *patterns )
+{
+	char **boxarr = NULL;
+	uint num = 0, rnum = 0;
+
+	uint pfxl = prefix ? strlen( prefix ) : 0;
+	for (; boxes; boxes = boxes->next) {
+		if (!starts_with( boxes->string, -1, prefix, pfxl ))
+			continue;
+		uint fnot = 1, not;
+		for (string_list_t *cpat = patterns; cpat; cpat = cpat->next) {
+			const char *ps = cpat->string;
+			if (*ps == '!') {
+				ps++;
+				not = 1;
+			} else {
+				not = 0;
+			}
+			if (matches( boxes->string + pfxl, ps )) {
+				fnot = not;
+				break;
+			}
+		}
+		if (!fnot) {
+			if (num + 1 >= rnum)
+				boxarr = nfrealloc( boxarr, (rnum = (rnum + 10) * 2) * sizeof(*boxarr) );
+			boxarr[num++] = nfstrdup( boxes->string + pfxl );
+			boxarr[num] = NULL;
+		}
+	}
+	qsort( boxarr, num, sizeof(*boxarr), cmp_box_names );
+	return boxarr;
+}
+
+static void
+merge_actions( channel_conf_t *chan, int ops[], int have, int mask, int def )
+{
+	if (ops[F] & have) {
+		chan->ops[F] &= ~mask;
+		chan->ops[F] |= ops[F] & mask;
+		chan->ops[N] &= ~mask;
+		chan->ops[N] |= ops[N] & mask;
+	} else if (!(chan->ops[F] & have)) {
+		if (global_conf.ops[F] & have) {
+			chan->ops[F] |= global_conf.ops[F] & mask;
+			chan->ops[N] |= global_conf.ops[N] & mask;
+		} else {
+			chan->ops[F] |= def;
+			chan->ops[N] |= def;
+		}
+	}
+}
+
+typedef struct box_ent {
+	struct box_ent *next;
+	char *name;
+	int present[2];
+} box_ent_t;
+
+typedef struct chan_ent {
+	struct chan_ent *next;
+	channel_conf_t *conf;
+	box_ent_t *boxes;
+	int boxlist;
+} chan_ent_t;
+
+static chan_ent_t *
+add_channel( chan_ent_t ***chanapp, channel_conf_t *chan, int ops[] )
+{
+	chan_ent_t *ce = nfzalloc( sizeof(*ce) );
+	ce->conf = chan;
+
+	merge_actions( chan, ops, XOP_HAVE_TYPE, OP_MASK_TYPE, OP_DFLT_TYPE );
+	merge_actions( chan, ops, XOP_HAVE_CREATE, OP_CREATE, 0 );
+	merge_actions( chan, ops, XOP_HAVE_REMOVE, OP_REMOVE, 0 );
+	merge_actions( chan, ops, XOP_HAVE_EXPUNGE, OP_EXPUNGE, 0 );
+	merge_actions( chan, ops, XOP_HAVE_EXPUNGE_SOLO, OP_EXPUNGE_SOLO, 0 );
+	debug( "channel ops (%s):\n  far: %s\n  near: %s\n",
+	       chan->name, fmt_ops( ops[F] ).str, fmt_ops( ops[N] ).str );
+
+	for (int t = 0; t < 2; t++) {
+		if (!(~ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO))) {
+			error( "Specified both Expunge and ExpungeSolo for %s of Channel '%s'.\n",
+			       str_fn[t], chan->stores[t]->name );
+			free( ce );
+			return NULL;
+		}
+		if (chan->ops[t] & OP_MASK_TYPE)
+			ops_any[t] = 1;
+		if (chan->ops[t] & (OP_EXPUNGE | OP_EXPUNGE_SOLO)) {
+			expunge_any[t] = 1;
+			if (chan->stores[t]->trash ||
+			    (chan->stores[t^1]->trash && chan->stores[t^1]->trash_remote_new))
+				trash_any[t] = 1;
+		}
+	}
+
+	**chanapp = ce;
+	*chanapp = &ce->next;
+	chans_total++;
+	return ce;
+}
+
+static chan_ent_t *
+add_named_channel( chan_ent_t ***chanapp, char *channame, int ops[] )
+{
+	box_ent_t *boxes = NULL, **mboxapp = &boxes, *mbox;
+	int boxlist = 0;
+
+	char *boxp;
+	if ((boxp = strchr( channame, ':' )))
+		*boxp++ = 0;
+	channel_conf_t *chan;
+	for (chan = channels; chan; chan = chan->next)
+		if (!strcmp( chan->name, channame ))
+			goto gotchan;
+	error( "No channel or group named '%s' defined.\n", channame );
+	return NULL;
+  gotchan:
+	if (boxp) {
+		if (!chan->patterns) {
+			error( "Cannot override mailbox in channel '%s' - no Patterns.\n", channame );
+			return NULL;
+		}
+		boxlist = 1;
+		do {
+			char *nboxp = strpbrk( boxp, ",\n" );
+			size_t boxl;
+			if (nboxp) {
+				boxl = (size_t)(nboxp - boxp);
+				*nboxp++ = 0;
+			} else {
+				boxl = strlen( boxp );
+			}
+			mbox = nfmalloc( sizeof(*mbox) );
+			if (boxl)
+				mbox->name = nfstrndup( boxp, boxl );
+			else
+				mbox->name = nfstrndup( "INBOX", 5 );
+			mbox->present[F] = mbox->present[N] = BOX_POSSIBLE;
+			mbox->next = NULL;
+			*mboxapp = mbox;
+			mboxapp = &mbox->next;
+			boxes_total++;
+			boxp = nboxp;
+		} while (boxp);
+	} else {
+		if (!chan->patterns)
+			boxes_total++;
+	}
+
+	chan_ent_t *ce = add_channel( chanapp, chan, ops );
+	if (!ce)
+		return NULL;
+	ce->boxes = boxes;
+	ce->boxlist = boxlist;
+	return ce;
+}
+
+typedef struct {
+	int t[2];
+	core_vars_t *cvars;
+	channel_conf_t *chan;
+	driver_t *drv[2];
+	store_t *ctx[2];
+	chan_ent_t *chanptr;
+	box_ent_t *boxptr;
+	string_list_t *boxes[2];
+	char *names[2];
+	int state[2];
+	int chan_cben, fnlz_cben, box_cben, box_done;
+} main_vars_t;
+
+#define AUX &mvars->t[t]
+#define MVARS(aux) \
+	int t = *(int *)aux; \
+	main_vars_t *mvars = (main_vars_t *)(((char *)(&((int *)aux)[-t])) - offsetof(main_vars_t, t));
+
+static void do_sync_chans( main_vars_t *lvars );
+
+void
+sync_chans( core_vars_t *cvars, char **argv )
+{
+	main_vars_t mvars[1];
+	chan_ent_t *chans = NULL, **chanapp = &chans;
+
+	memset( mvars, 0, sizeof(*mvars) );
+	mvars->t[1] = 1;
+	mvars->cvars = cvars;
+
+	if (!channels) {
+		fputs( "No channels defined. Try 'man " EXE "'\n", stderr );
+		cvars->ret = 1;
+		return;
+	}
+
+	if (cvars->all) {
+		for (channel_conf_t *chan = channels; chan; chan = chan->next) {
+			if (!add_channel( &chanapp, chan, cvars->ops ))
+				cvars->ret = 1;
+			if (!chan->patterns)
+				boxes_total++;
+		}
+	} else {
+		for (; *argv; argv++) {
+			for (group_conf_t *group = groups; group; group = group->next) {
+				if (!strcmp( group->name, *argv )) {
+					for (string_list_t *channame = group->channels; channame; channame = channame->next)
+						if (!add_named_channel( &chanapp, channame->string, cvars->ops ))
+							cvars->ret = 1;
+					goto gotgrp;
+				}
+			}
+			if (!add_named_channel( &chanapp, *argv, cvars->ops ))
+				cvars->ret = 1;
+		  gotgrp: ;
+		}
+	}
+	if (cvars->ret)
+		return;
+	if (!chans) {
+		fputs( "No channel specified. Try '" EXE " -h'\n", stderr );
+		cvars->ret = 1;
+		return;
+	}
+	mvars->chanptr = chans;
+
+	if (!cvars->list && (DFlags & PROGRESS)) {
+		init_wakeup( &stats_wakeup, stats_timeout, NULL );
+		stats_timeout( NULL );
+	}
+	do_sync_chans( mvars );
+	main_loop();
+	if (!cvars->list) {
+		flushn();
+		summary();
+		if (DFlags & EXT_EXIT) {
+			for (int t = 0; t < 2; t++)
+				if (new_done[t] || flags_done[t] || trash_done[t] || expunge_done[t])
+					cvars->ret |= 32 << t;
+		}
+	}
+}
+
+enum {
+	ST_FRESH,
+	ST_CONNECTED,
+	ST_OPEN,
+	ST_CANCELING,
+	ST_CLOSED,
+};
+
+static int
+check_cancel( main_vars_t *mvars )
+{
+	return mvars->state[F] >= ST_CANCELING || mvars->state[N] >= ST_CANCELING;
+}
+
+static void store_connected( int sts, void *aux );
+static void store_listed( int sts, string_list_t *boxes, void *aux );
+static void sync_opened( main_vars_t *mvars, int t );
+static void do_sync_boxes( main_vars_t *lvars );
+static void done_sync_dyn( int sts, void *aux );
+static void done_sync( int sts, void *aux );
+static void finalize_sync( main_vars_t *mvars );
+
+static void
+store_bad( void *aux )
+{
+	MVARS(aux)
+
+	mvars->drv[t]->cancel_store( mvars->ctx[t] );
+	mvars->state[t] = ST_CLOSED;
+	mvars->cvars->ret = 1;
+	finalize_sync( mvars );
+}
+
+static void
+advance_chan( main_vars_t *mvars )
+{
+	if (!mvars->cvars->list) {
+		chans_done++;
+		stats();
+	}
+	chan_ent_t *nchan = mvars->chanptr->next;
+	free( mvars->chanptr );
+	mvars->chanptr = nchan;
+}
+
+static void
+do_sync_chans( main_vars_t *mvars )
+{
+	while (mvars->chanptr) {
+		stats_steps = 0;  // Determine main loop use afresh
+		mvars->chan = mvars->chanptr->conf;
+		info( "Channel %s\n", mvars->chan->name );
+		for (int t = 0; t < 2; t++) {
+			int st = mvars->chan->stores[t]->driver->get_fail_state( mvars->chan->stores[t] );
+			if (st != FAIL_TEMP) {
+				info( "Skipping due to %sfailed %s store %s.\n",
+				      (st == FAIL_WAIT) ? "temporarily " : "", str_fn[t], mvars->chan->stores[t]->name );
+				goto next;
+			}
+		}
+
+		uint dcaps[2];
+		for (int t = 0; t < 2; t++) {
+			mvars->drv[t] = mvars->chan->stores[t]->driver;
+			dcaps[t] = mvars->drv[t]->get_caps( NULL );
+		}
+		const char *labels[2];
+		if ((DFlags & DEBUG_DRV) || (dcaps[F] & dcaps[N] & DRV_VERBOSE))
+			labels[F] = "F: ", labels[N] = "N: ";
+		else
+			labels[F] = labels[N] = "";
+		for (int t = 0; t < 2; t++) {
+			store_t *ctx = mvars->drv[t]->alloc_store( mvars->chan->stores[t], labels[t] );
+			if ((DFlags & DEBUG_DRV) || ((DFlags & FORCEASYNC(t)) && !(dcaps[t] & DRV_ASYNC))) {
+				mvars->drv[t] = &proxy_driver;
+				ctx = proxy_alloc_store( ctx, labels[t], DFlags & FORCEASYNC(t) );
+			}
+			mvars->ctx[t] = ctx;
+			mvars->drv[t]->set_callbacks( ctx, NULL, store_bad, AUX );
+			mvars->state[t] = ST_FRESH;
+		}
+		mvars->chan_cben = 0;
+		for (int t = 0; ; t++) {
+			info( "Opening %s store %s...\n", str_fn[t], mvars->chan->stores[t]->name );
+			mvars->drv[t]->connect_store( mvars->ctx[t], store_connected, AUX );
+			if (t || check_cancel( mvars ))
+				break;
+		}
+		if (mvars->state[F] != ST_CLOSED || mvars->state[N] != ST_CLOSED) {
+			mvars->chan_cben = 1;
+			return;
+		}
+
+	  next:
+		advance_chan( mvars );
+	}
+	cleanup_mainloop();
+	if (!mvars->cvars->list && (DFlags & PROGRESS))
+		wipe_wakeup( &stats_wakeup );
+}
+
+static void
+sync_next_chan( main_vars_t *mvars )
+{
+	if (mvars->chan_cben) {
+		advance_chan( mvars );
+		do_sync_chans( mvars );
+	}
+}
+
+static void
+store_connected( int sts, void *aux )
+{
+	MVARS(aux)
+
+	switch (sts) {
+	case DRV_CANCELED:
+		return;
+	case DRV_OK:
+		mvars->state[t] = ST_CONNECTED;
+		if (check_cancel( mvars ))
+			break;
+		if (!mvars->chanptr->boxlist && mvars->chan->patterns) {
+			int cflags = 0;
+			for (string_list_t *cpat = mvars->chan->patterns; cpat; cpat = cpat->next) {
+				const char *pat = cpat->string;
+				if (*pat != '!') {
+					char buf[8];
+					int bufl = snprintf( buf, sizeof(buf), "%s%s", nz( mvars->chan->boxes[t], "" ), pat );
+					int flags = 0;
+					// Partial matches like "INB*" or even "*" are not considered,
+					// except implicity when the INBOX lives under Path.
+					if (starts_with( buf, bufl, "INBOX", 5 )) {
+						char c = buf[5];
+						if (!c) {
+							// User really wants the INBOX.
+							flags |= LIST_INBOX;
+						} else if (c == '/') {
+							// Flattened sub-folders of INBOX actually end up in Path.
+							if (mvars->ctx[t]->conf->flat_delim[0])
+								flags |= LIST_PATH;
+							else
+								flags |= LIST_INBOX;
+						} else if (c == '*' || c == '%') {
+							// It can be both INBOX and Path, but don't require Path to be configured.
+							flags |= LIST_INBOX | LIST_PATH_MAYBE;
+						} else {
+							// It's definitely not the INBOX.
+							flags |= LIST_PATH;
+						}
+					} else {
+						flags |= LIST_PATH;
+					}
+					debug( "pattern '%s' (effective '%s'): %sPath, %sINBOX\n",
+					       pat, buf, (flags & LIST_PATH) ? "" : "no ",  (flags & LIST_INBOX) ? "" : "no ");
+					cflags |= flags;
+				}
+			}
+			mvars->drv[t]->list_store( mvars->ctx[t], cflags, store_listed, AUX );
+			return;
+		}
+		sync_opened( mvars, t );
+		return;
+	default:
+		mvars->cvars->ret = 1;
+		break;
+	}
+	finalize_sync( mvars );
+}
+
+static void
+store_listed( int sts, string_list_t *boxes, void *aux )
+{
+	MVARS(aux)
+	int fail = 0;
+
+	switch (sts) {
+	case DRV_CANCELED:
+		return;
+	case DRV_OK:
+		if (check_cancel( mvars ))
+			break;
+		for (string_list_t *box = boxes; box; box = box->next) {
+			if (mvars->ctx[t]->conf->flat_delim[0]) {
+				string_list_t *nbox;
+				if (map_name( box->string, -1, (char **)&nbox, offsetof(string_list_t, string), mvars->ctx[t]->conf->flat_delim, "/" ) < 0) {
+					error( "Error: flattened mailbox name '%s' contains canonical hierarchy delimiter\n", box->string );
+					fail = 1;
+				} else {
+					nbox->next = mvars->boxes[t];
+					mvars->boxes[t] = nbox;
+				}
+			} else {
+				add_string_list( &mvars->boxes[t], box->string );
+			}
+		}
+		if (fail) {
+			mvars->cvars->ret = 1;
+			break;
+		}
+		if (mvars->ctx[t]->conf->map_inbox) {
+			debug( "adding mapped inbox to %s store: %s\n", str_fn[t], mvars->ctx[t]->conf->map_inbox );
+			add_string_list( &mvars->boxes[t], mvars->ctx[t]->conf->map_inbox );
+		}
+		sync_opened( mvars, t );
+		return;
+	default:
+		mvars->cvars->ret = 1;
+		break;
+	}
+	finalize_sync( mvars );
+}
+
+static void
+sync_opened( main_vars_t *mvars, int t )
+{
+	mvars->state[t] = ST_OPEN;
+	if (mvars->state[t^1] != ST_OPEN)
+		return;
+
+	if (!mvars->chanptr->boxlist && mvars->chan->patterns) {
+		mvars->chanptr->boxlist = 2;
+		char **boxes[2];
+		boxes[F] = filter_boxes( mvars->boxes[F], mvars->chan->boxes[F], mvars->chan->patterns );
+		boxes[N] = filter_boxes( mvars->boxes[N], mvars->chan->boxes[N], mvars->chan->patterns );
+		box_ent_t **mboxapp = &mvars->chanptr->boxes;
+		for (int mb = 0, sb = 0; ; ) {
+			char *fname = boxes[F] ? boxes[F][mb] : NULL;
+			char *nname = boxes[N] ? boxes[N][sb] : NULL;
+			if (!fname && !nname)
+				break;
+			box_ent_t *mbox = nfmalloc( sizeof(*mbox) );
+			int cmp;
+			if (!(cmp = !fname - !nname) && !(cmp = cmp_box_names( &fname, &nname ))) {
+				mbox->name = fname;
+				free( nname );
+				mbox->present[F] = mbox->present[N] = BOX_PRESENT;
+				mb++;
+				sb++;
+			} else if (cmp < 0) {
+				mbox->name = fname;
+				mbox->present[F] = BOX_PRESENT;
+				mbox->present[N] = (!mb && !strcmp( mbox->name, "INBOX" )) ? BOX_PRESENT : BOX_ABSENT;
+				mb++;
+			} else {
+				mbox->name = nname;
+				mbox->present[F] = (!sb && !strcmp( mbox->name, "INBOX" )) ? BOX_PRESENT : BOX_ABSENT;
+				mbox->present[N] = BOX_PRESENT;
+				sb++;
+			}
+			mbox->next = NULL;
+			*mboxapp = mbox;
+			mboxapp = &mbox->next;
+			boxes_total++;
+		}
+		free( boxes[F] );
+		free( boxes[N] );
+		if (!mvars->cvars->list)
+			stats();
+	}
+	mvars->boxptr = mvars->chanptr->boxes;
+
+	if (mvars->cvars->list && chans_total > 1)
+		printf( "%s:\n", mvars->chan->name );
+	mvars->box_done = 0;
+	do_sync_boxes( mvars );
+}
+
+static void
+do_sync_boxes( main_vars_t *mvars )
+{
+	mvars->box_cben = 0;
+	while (mvars->state[F] == ST_OPEN && mvars->state[N] == ST_OPEN) {
+		if (mvars->chanptr->boxlist) {
+			box_ent_t *mbox = mvars->boxptr;
+			if (!mbox)
+				break;
+			mvars->boxptr = mbox->next;
+			mvars->box_done = 0;
+			if (mvars->chan->boxes[F] || mvars->chan->boxes[N]) {
+				const char *fpfx = nz( mvars->chan->boxes[F], "" );
+				const char *npfx = nz( mvars->chan->boxes[N], "" );
+				if (mvars->cvars->list) {
+					printf( "%s%s <=> %s%s\n", fpfx, mbox->name, npfx, mbox->name );
+					continue;
+				}
+				nfasprintf( &mvars->names[F], "%s%s", fpfx, mbox->name );
+				nfasprintf( &mvars->names[N], "%s%s", npfx, mbox->name );
+				sync_boxes( mvars->ctx, (const char * const *)mvars->names, mbox->present, mvars->chan, done_sync_dyn, mvars );
+			} else {
+				if (mvars->cvars->list) {
+					puts( mbox->name );
+					continue;
+				}
+				mvars->names[F] = mvars->names[N] = mbox->name;
+				sync_boxes( mvars->ctx, (const char * const *)mvars->names, mbox->present, mvars->chan, done_sync, mvars );
+			}
+		} else {
+			if (mvars->cvars->list) {
+				printf( "%s <=> %s\n", nz( mvars->chan->boxes[F], "INBOX" ), nz( mvars->chan->boxes[N], "INBOX" ) );
+				break;
+			}
+			if (mvars->box_done)
+				break;
+			int present[] = { BOX_POSSIBLE, BOX_POSSIBLE };
+			sync_boxes( mvars->ctx, mvars->chan->boxes, present, mvars->chan, done_sync, mvars );
+		}
+		if (!mvars->box_done) {
+			mvars->box_cben = 1;
+			return;
+		}
+	}
+	finalize_sync( mvars );
+}
+
+static void
+done_sync_dyn( int sts, void *aux )
+{
+	main_vars_t *mvars = (main_vars_t *)aux;
+
+	free( mvars->names[F] );
+	free( mvars->names[N] );
+	done_sync( sts, aux );
+}
+
+static void
+done_sync( int sts, void *aux )
+{
+	main_vars_t *mvars = (main_vars_t *)aux;
+
+	boxes_done++;
+	stats();
+	if (sts) {
+		mvars->cvars->ret = 1;
+		if (sts & SYNC_BAD(F))
+			mvars->state[F] = ST_CLOSED;
+		if (sts & SYNC_BAD(N))
+			mvars->state[N] = ST_CLOSED;
+	}
+	mvars->box_done = 1;
+	if (mvars->box_cben)
+		do_sync_boxes( mvars );
+}
+
+static void sync_finalized( void *aux );
+
+static void
+finalize_sync( main_vars_t *mvars )
+{
+	if (mvars->chanptr->boxlist) {
+		box_ent_t *mbox, *nmbox;
+		for (nmbox = mvars->chanptr->boxes; (mbox = nmbox); ) {
+			nmbox = mbox->next;
+			free( mbox->name );
+			free( mbox );
+		}
+		mvars->chanptr->boxes = NULL;
+		mvars->chanptr->boxlist = 0;
+	}
+
+	mvars->fnlz_cben = 0;
+	for (int t = 0; t < 2; t++) {
+		free_string_list( mvars->boxes[t] );
+		mvars->boxes[t] = NULL;
+
+		if (mvars->state[t] == ST_FRESH || mvars->state[t] == ST_OPEN) {
+			mvars->drv[t]->free_store( mvars->ctx[t] );
+			mvars->state[t] = ST_CLOSED;
+		} else if (mvars->state[t] == ST_CONNECTED) {
+			mvars->state[t] = ST_CANCELING;
+			mvars->drv[t]->cancel_cmds( mvars->ctx[t], sync_finalized, AUX );
+		}
+	}
+	if (mvars->state[F] != ST_CLOSED || mvars->state[N] != ST_CLOSED) {
+		mvars->fnlz_cben = 1;
+		return;
+	}
+	sync_next_chan( mvars );
+}
+
+static void
+sync_finalized( void *aux )
+{
+	MVARS(aux)
+
+	mvars->drv[t]->free_store( mvars->ctx[t] );
+	mvars->state[t] = ST_CLOSED;
+	if (mvars->state[t^1] != ST_CLOSED)
+		return;
+
+	if (mvars->fnlz_cben)
+		sync_next_chan( mvars );
+}

src/mbsync.1

@@ -1,31 +1,18 @@
+.\" SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+.\" SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+.\" SPDX-License-Identifier: GPL-2.0-or-later
+.\"
 .\" mbsync - mailbox synchronizer
-.\" Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
-.\" Copyright (C) 2002-2004,2011-2015 Oswald Buddenhagen <ossi@users.sf.net>
-.\" Copyright (C) 2004 Theodore Y. Ts'o <tytso@mit.edu>
-.\"
-.\"  This program is free software; you can redistribute it and/or modify
-.\"  it under the terms of the GNU General Public License as published by
-.\"  the Free Software Foundation; either version 2 of the License, or
-.\"  (at your option) any later version.
-.\"
-.\"  This program is distributed in the hope that it will be useful,
-.\"  but WITHOUT ANY WARRANTY; without even the implied warranty of
-.\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-.\"  GNU General Public License for more details.
-.\"
-.\"  You should have received a copy of the GNU General Public License
-.\"  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-.\"
-.\" As a special exception, mbsync may be linked with the OpenSSL library,
-.\" despite that library's more restrictive license.
 .
-.TH mbsync 1 "2015 Mar 22"
+.TH mbsync 1 "2022 Jun 16"
 .
 .SH NAME
 mbsync - synchronize IMAP4 and Maildir mailboxes
 .
 .SH SYNOPSIS
 \fBmbsync\fR [\fIoptions\fR ...] {{\fIchannel\fR[\fB:\fIbox\fR[{\fB,\fR|\fB\\n\fR}...]]|\fIgroup\fR} ...|\fB-a\fR}
+.br
+\fBmbsync\fR --list-stores [\fIoptions\fR ...] [\fIstore\fR} ...]
 .
 .SH DESCRIPTION
 \fBmbsync\fR is a command line application which synchronizes mailboxes;
@@ -47,16 +34,25 @@ Multiple replicas of each mailbox can be maintained.
 .TP
 \fB-c\fR, \fB--config\fR \fIfile\fR
 Read configuration from \fIfile\fR.
-By default, the configuration is read from ~/.mbsyncrc.
+By default, the configuration is read from $XDG_CONFIG_HOME/isyncrc, and
+if that does not exist, ~/.mbsyncrc is tried in turn.
+$XDG_CONFIG_HOME defaults to ~/.config if not set.
 .TP
 \fB-a\fR, \fB--all\fR
-Select all configured channels. Any channel/group specifications on the command
-line are ignored.
+Select all configured Channels. Any Channel/Group specifications on the
+command line are ignored.
 .TP
 \fB-l\fR, \fB--list\fR
-Don't synchronize anything, but list all mailboxes in the selected channels
+Don't synchronize anything, but list all mailboxes in the selected Channels
 and exit.
 .TP
+\fB-ls\fR, \fB--list-stores\fR
+Don't synchronize anything, but list all mailboxes in the selected Stores
+and exit.
+If no Stores are specified, all configured ones are listed.
+These are raw Store contents, not filtered by any Channel's \fBPatterns\fR.
+This option may be used to verify each Store's configuration.
+.TP
 \fB-C\fR[\fBf\fR][\fBn\fR], \fB--create\fR[\fB-far\fR|\fB-near\fR]
 Override any \fBCreate\fR options from the config file. See below.
 .TP
@@ -66,11 +62,11 @@ Override any \fBRemove\fR options from the config file. See below.
 \fB-X\fR[\fBf\fR][\fBn\fR], \fB--expunge\fR[\fB-far\fR|\fB-near\fR]
 Override any \fBExpunge\fR options from the config file. See below.
 .TP
-{\fB-n\fR|\fB-N\fR|\fB-d\fR|\fB-f\fR|\fB-0\fR|\fB-F\fR},\
- {\fB--new\fR|\fB--renew\fR|\fB--delete\fR|\fB--flags\fR|\fB--noop\fR|\fB--full\fR}
+{\fB-n\fR|\fB-o\fR|\fB-u\fR|\fB-g\fR|\fB-f\fR|\fB-0\fR|\fB-F\fR},\
+ {\fB--new\fR|\fB--old\fR|\fB--upgrade\fR|\fB--gone\fR|\fB--flags\fR|\fB--noop\fR|\fB--full\fR}
 .TP
-\r{\fB-L\fR|\fB-H\fR}[\fBn\fR][\fBN\fR][\fBd\fR][\fBf\fR],\
- {\fB--pull\fR|\fB--push\fR}[\fB-new\fR|\fB-renew\fR|\fB-delete\fR|\fB-flags\fR]
+\r{\fB-L\fR|\fB-H\fR}[\fBn\fR][\fBo\fR][\fBu\fR][\fBg\fR][\fBf\fR],\
+ {\fB--pull\fR|\fB--push\fR}[\fB-new\fR|\fB-old\fR|\fB-upgrade\fR|\fB-gone\fR|\fB-flags\fR]
 Override any \fBSync\fR options from the config file. See below.
 .TP
 \fB-h\fR, \fB--help\fR
@@ -79,6 +75,19 @@ Display a summary of command line options.
 \fB-v\fR, \fB--version\fR
 Display version information.
 .TP
+\fB-y\fR, \fB--dry-run\fR
+Enter simulation mode: the Channel status is queried and all required
+operations are determined, but no modifications are actually made
+to either the mailboxes or the state files.
+.TP
+\fB-e\fR, \fB--ext-exit\fR
+Return an extended exit code: Add 32 or 64 to the code if any
+modifications were made on the far or near side, respectively; these
+are not mutually exclusive, so the code may be 96 if changes were both
+pushed and pulled.
+An error may be reported at the same time, so the code may be for example
+65 if some changes were successfully pulled, while others failed.
+.TP
 \fB-V\fR, \fB--verbose\fR
 Enable \fIverbose\fR mode, which displays what is currently happening.
 .TP
@@ -107,7 +116,7 @@ Without category specification, all categories except net-all are enabled.
 .TP
 \fB-q\fR, \fB--quiet\fR
 Suppress progress counters (this is implicit if stdout is no TTY,
-or any debugging categories are enabled) and notices.
+or any debugging categories are enabled), notices, and the summary.
 If specified twice, suppress warning messages as well.
 .
 .SH CONFIGURATION
@@ -119,6 +128,8 @@ and literal double quotes and backslashes (\fB\\\fR) must be backslash-escaped.
 All keywords (including those used as arguments) are case-insensitive.
 Bash-like home directory expansion using the tilde (\fB~\fR) is supported
 in all options which represent local paths.
+The reference point for relative local paths is the configuration file's
+containing directory.
 There are a few global options, the others apply to particular sections.
 Sections begin with a section-starting keyword and are terminated by an empty
 line or end of file.
@@ -144,8 +155,7 @@ Unix-like forward slashes.
 .SS All Stores
 These options can be used in all supported Store types.
 .br
-In this context, the term "remote" describes the second Store within a Channel,
-and not necessarily a remote server.
+The term "opposite Store" refers to the other Store within a Channel.
 .br
 The special mailbox \fBINBOX\fR exists in every Store; its physical location
 in the file system is Store type specific.
@@ -164,14 +174,17 @@ directory.
 .TP
 \fBMaxSize\fR \fIsize\fR[\fBk\fR|\fBm\fR][\fBb\fR]
 Messages larger than \fIsize\fR will have only a small placeholder message
-propagated into this Store. To propagate the full message, it must be
-flagged in either Store; that can be done retroactively, in which case
-the \fBReNew\fR operation needs to be executed instead of \fBNew\fR.
+propagated into this Store.
 This is useful for avoiding downloading messages with large attachments
 unless they are actually needed.
-Caveat: Setting a size limit on a Store you never read directly (which is
+To upgrade the placeholder to the full message, it must be flagged, and
+the \fBUpgrade\fR operation executed.
+Caveats: Setting a size limit on a Store you never read directly (which is
 typically the case for servers) is not recommended, as you may never
 notice that affected messages were not propagated to it.
+Also, as flagging is (ab-)used to request an upgrade, changes to the
+message's flagging state will not be propagated in either direction until
+after the placeholder is upgraded.
 .br
 \fBK\fR and \fBM\fR can be appended to the size to specify KiBytes resp.
 MeBytes instead of bytes. \fBB\fR is accepted but superfluous.
@@ -208,18 +221,20 @@ See \fBRECOMMENDATIONS\fR and \fBINHERENT PROBLEMS\fR below.
 .TP
 \fBTrashNewOnly\fR \fByes\fR|\fBno\fR
 When trashing, copy only not yet propagated messages. This makes sense if the
-remote Store has a \fBTrash\fR as well (with \fBTrashNewOnly\fR \fBno\fR).
+opposite Store has a \fBTrash\fR as well (with \fBTrashNewOnly\fR \fBno\fR).
 (Default: \fBno\fR)
 .
 .TP
 \fBTrashRemoteNew\fR \fByes\fR|\fBno\fR
-When expunging the remote Store, copy not yet propagated messages to this
-Store's \fBTrash\fR. When using this, the remote Store does not need an own
-\fBTrash\fR at all, yet all messages are archived.
+When expunging the opposite Store, copy not yet propagated messages to this
+Store's \fBTrash\fR.
+When using this, the opposite Store does not need an own \fBTrash\fR at all,
+yet all messages are archived.
 (Default: \fBno\fR)
 .
 .SS Maildir Stores
-The reference point for relative \fBPath\fRs is the current working directory.
+The reference point for relative \fBPath\fRs is the configuration file's
+containing directory.
 .P
 As \fBmbsync\fR needs UIDs, but no standardized UID storage scheme exists for
 Maildir, \fBmbsync\fR supports two schemes, each with its pros and cons.
@@ -301,7 +316,7 @@ Define the IMAP4 Account \fIname\fR, opening a section for its parameters.
 \fBHost\fR \fIhost\fR
 Specify the DNS name or IP address of the IMAP server.
 .br
-If \fBTunnel\fR is used, this setting is needed only if \fBSSLType\fR is
+If \fBTunnel\fR is used, this setting is needed only if \fBTLSType\fR is
 not \fBNone\fR and \fBCertificateFile\fR is not used,
 in which case the host name is used for certificate subject verification.
 .
@@ -382,13 +397,13 @@ The list of acceptable authentication mechanisms.
 In addition to the mechanisms listed in the SASL registry (link below),
 the legacy IMAP \fBLOGIN\fR mechanism is known.
 The wildcard \fB*\fR represents all mechanisms that are deemed secure
-enough for the current \fBSSLType\fR setting.
+enough for the current \fBTLSType\fR setting.
 The actually used mechanism is the most secure choice from the intersection
 of this list, the list supplied by the server, and the installed SASL modules.
 (Default: \fB*\fR)
 .
 .TP
-\fBSSLType\fR {\fBNone\fR|\fBSTARTTLS\fR|\fBIMAPS\fR}
+\fBTLSType\fR {\fBNone\fR|\fBSTARTTLS\fR|\fBIMAPS\fR}
 Select the connection security/encryption method:
 .br
 \fBNone\fR - no security.
@@ -398,14 +413,16 @@ This is the default when \fBTunnel\fR is set, as tunnels are usually secure.
 after connecting the regular IMAP port 143. Most servers support this,
 so it is the default (unless a tunnel is used).
 .br
-\fBIMAPS\fR - security is established by starting SSL/TLS negotiation
+\fBIMAPS\fR - security is established by starting TLS negotiation
 right after connecting the secure IMAP port 993.
 .
 .TP
-\fBSSLVersions\fR [\fBSSLv3\fR] [\fBTLSv1\fR] [\fBTLSv1.1\fR] [\fBTLSv1.2\fR] [\fBTLSv1.3\fR]
-Select the acceptable SSL/TLS versions.
+\fBTLSVersions\fR {\fB+\fR|\fB-\fR}{\fB1.0\fR|\fB1.1\fR|\fB1.2\fR|\fB1.3\fR} ...
+Add/remove the specified TLS versions to/from the set of acceptable choices.
 Use old versions only when the server has problems with newer ones.
-(Default: [\fBTLSv1\fR] [\fBTLSv1.1\fR] [\fBTLSv1.2\fR] [\fBTLSv1.3\fR]).
+Note that new versions are automatically enabled as soon as OpenSSL supports
+them, even if \fBmbsync\fR does not recognize them yet.
+(Default: All starting with 1.2).
 .
 .TP
 \fBSystemCertificates\fR \fByes\fR|\fBno\fR
@@ -540,7 +557,7 @@ Note that \fBINBOX\fR is not matched by wildcards, unless it lives under
 \fBPath\fR.
 .br
 The mailbox list selected by \fBPatterns\fR can be overridden by a mailbox
-list in a channel reference (a \fBGroup\fR specification or the command line).
+list in a Channel reference (a \fBGroup\fR specification or the command line).
 .br
 Example: "\fBPatterns\fR\ \fI%\ !Trash\fR"
 .
@@ -560,7 +577,7 @@ the actual date of the message) will be deleted first.
 Messages that are flagged (marked as important) and (by default) unread
 messages will not be automatically deleted.
 If \fIcount\fR is 0, the maximum number of messages is \fBunlimited\fR
-(Default: \fI0\fR).
+(Global default: \fI0\fR).
 .
 .TP
 \fBExpireUnread\fR \fByes\fR|\fBno\fR
@@ -570,10 +587,16 @@ This ensures that you never miss new messages even after an extended absence.
 However, if your archive contains large amounts of unread messages by design,
 treating them as important would practically defeat \fBMaxMessages\fR. In this
 case you need to enable this option.
-(Default: \fBno\fR).
+(Global default: \fBno\fR).
 .
 .TP
-\fBSync\fR {\fBNone\fR|[\fBPull\fR] [\fBPush\fR] [\fBNew\fR] [\fBReNew\fR] [\fBDelete\fR] [\fBFlags\fR]|\fBAll\fR}
+\fBExpireSide\fR \fBFar\fR|\fBNear\fR
+Selects on which side messages should be expired when \fBMaxMessages\fR is
+configured.
+(Global default: \fBNear\fR).
+.
+.TP
+\fBSync\fR {\fBNone\fR|[\fBPull\fR] [\fBPush\fR] [\fBNew\fR] [\fBOld\fR] [\fBUpgrade\fR] [\fBGone\fR] [\fBFlags\fR] [\fBFull\fR]}
 Select the synchronization operation(s) to perform:
 .br
 \fBPull\fR - propagate changes from far to near side.
@@ -582,11 +605,15 @@ Select the synchronization operation(s) to perform:
 .br
 \fBNew\fR - propagate newly appeared messages.
 .br
-\fBReNew\fR - upgrade placeholders to full messages. Useful only with
+\fBOld\fR - propagate previously skipped, failed, and expired messages.
+This has a (relatively) high cost and may repeatedly produce error messages,
+so it always must be specified explicitly.
+.br
+\fBUpgrade\fR - upgrade placeholders to full messages. Useful only with
 a configured \fBMaxSize\fR.
 .br
-\fBDelete\fR - propagate message deletions. This applies only to messages that
-are actually gone, i.e., were expunged. The affected messages in the remote
+\fBGone\fR - propagate message disappearances. This applies only to messages that
+are actually gone, i.e., were expunged. The affected messages in the opposite
 Store are marked as deleted only, i.e., they won't be really deleted until
 that Store is expunged.
 .br
@@ -594,23 +621,24 @@ that Store is expunged.
 well; this is particularly interesting if you use \fBmutt\fR with the
 maildir_trash option.
 .br
-\fBAll\fR (\fB--full\fR on the command line) - all of the above.
+\fBFull\fR - alias for "\fBNew\fR\ \fBUpgrade\fR\ \fBGone\fR\ \fBFlags\fR".
 This is the global default.
 .br
 \fBNone\fR (\fB--noop\fR on the command line) - don't propagate anything.
 Useful if you want to expunge only.
 .IP
-\fBPull\fR and \fBPush\fR are direction flags, while \fBNew\fR, \fBReNew\fR,
-\fBDelete\fR and \fBFlags\fR are type flags. The two flag classes make up a
-two-dimensional matrix (a table). Its cells are the individual actions to
-perform. There are two styles of asserting the cells:
+\fBPull\fR and \fBPush\fR are direction flags, while \fBNew\fR, \fBOld\fR,
+\fBUpgrade\fR, \fBGone\fR, and \fBFlags\fR are type flags.
+The two flag classes make up a two-dimensional matrix (a table). Its cells are
+the individual actions to perform. There are two styles of asserting the cells:
 .br
 In the first style, the flags select entire rows/colums in the matrix. Only
 the cells which are selected both horizontally and vertically are asserted.
-Specifying no flags from a class is like specifying all flags from this class.
+Specifying no direction is like specifying both directions, and specifying
+no type is like specifying \fBFull\fR.
 For example, "\fBSync\fR\ \fBPull\fR\ \fBNew\fR\ \fBFlags\fR" will propagate
 new messages and flag changes from the far side to the near side,
-"\fBSync\fR\ \fBNew\fR\ \fBDelete\fR" will propagate message arrivals and
+"\fBSync\fR\ \fBNew\fR\ \fBGone\fR" will propagate message arrivals and
 deletions both ways, and "\fBSync\fR\ \fBPush\fR" will propagate all changes
 from the near side to the far side.
 .br
@@ -618,13 +646,16 @@ In the second style, direction flags are concatenated with type flags; every
 compound flag immediately asserts a cell in the matrix. In addition to at least
 one compound flag, the individual flags can be used as well, but as opposed to
 the first style, they immediately assert all cells in their respective
-row/column. For example,
-"\fBSync\fR\ \fBPullNew\fR\ \fBPullDelete\fR\ \fBPush\fR" will propagate
+row/column (with the exception of \fBOld\fR). For example,
+"\fBSync\fR\ \fBPullNew\fR\ \fBPullGone\fR\ \fBPush\fR" will propagate
 message arrivals and deletions from the far side to the near side and any
-changes from the near side to the far side.
+changes (except old messages) from the near side to the far side.
+.br
 Note that it is not allowed to assert a cell in two ways, e.g.
 "\fBSync\fR\ \fBPullNew\fR\ \fBPull\fR" and
-"\fBSync\fR\ \fBPullNew\fR\ \fBDelete\fR\ \fBPush\fR" induce error messages.
+"\fBSync\fR\ \fBPullNew\fR\ \fBGone\fR\ \fBPush\fR" induce error messages.
+.br
+\fBNone\fR may not be combined with any other operation.
 .
 .TP
 \fBCreate\fR {\fBNone\fR|\fBFar\fR|\fBNear\fR|\fBBoth\fR}
@@ -648,11 +679,26 @@ Note that for safety, non-empty mailboxes are never deleted.
 .
 .TP
 \fBExpunge\fR {\fBNone\fR|\fBFar\fR|\fBNear\fR|\fBBoth\fR}
-Permanently remove all messages [on the far/near side] marked for deletion.
+Permanently remove all messages [on the far/near side] which are marked
+for deletion.
+Mutually exclusive with \fBExpungeSolo\fR for the same side.
 See \fBRECOMMENDATIONS\fR below.
 (Global default: \fBNone\fR)
 .
 .TP
+\fBExpungeSolo\fR {\fBNone\fR|\fBFar\fR|\fBNear\fR|\fBBoth\fR}
+Permanently remove all messages [on the far/near side] which are both
+marked for deletion and have no corresponding message in the opposite
+Store.
+Together with \fBSync Gone\fR, this allows actual mirroring of
+expunges. Note, however, that this makes sense only if nothing else
+expunges the other messages which are marked for deletion.
+Also note that this does not work for IMAP Stores which do not support
+the UIDPLUS extension.
+Mutually exclusive with \fBExpunge\fR for the same side.
+(Global default: \fBNone\fR)
+.
+.TP
 \fBCopyArrivalDate\fR {\fByes\fR|\fBno\fR}
 Selects whether their arrival time should be propagated together with
 the messages.
@@ -660,11 +706,12 @@ Enabling this makes sense in order to keep the time stamp based message
 sorting intact.
 Note that IMAP does not guarantee that the time stamp (termed \fBinternal
 date\fR) is actually the arrival time, but it is usually close enough.
-(Default: \fBno\fR)
+(Global default: \fBno\fR)
 .
 .P
-\fBSync\fR, \fBCreate\fR, \fBRemove\fR, \fBExpunge\fR,
-\fBMaxMessages\fR, and \fBCopyArrivalDate\fR
+\fBSync\fR, \fBCreate\fR, \fBRemove\fR, \fBExpunge\fR, \fBExpungeSolo\fR,
+\fBMaxMessages\fR, \fBExpireUnread\fR, \fBExpireSide\fR,
+and \fBCopyArrivalDate\fR
 can be used before any section for a global effect.
 The global settings are overridden by Channel-specific options,
 which in turn are overridden by command line switches.
@@ -677,14 +724,17 @@ in the near side mailbox itself; this has the advantage that you do not need
 to handle the state file separately if you delete the mailbox, but it works
 only with Maildir mailboxes, obviously.
 Otherwise this is interpreted as a string to prepend to the near side mailbox
-name to make up a complete path.
+name to make up a complete path. Note that you \fBmust\fR append a slash if
+you want to specify a directory.
 .br
 This option can be used outside any section for a global effect. In this case
 the appended string is made up according to the pattern
 \fB:\fIfar-store\fB:\fIfar-box\fB_:\fInear-store\fB:\fInear-box\fR
 (see also \fBFieldDelimiter\fR below).
 .br
-(Global default: \fI~/.mbsync/\fR).
+(Global default: \fI$XDG_STATE_HOME/isync/\fR, with a fallback to
+\fI~/.mbsync/\fR if only that exists.
+$XDG_STATE_HOME defaults to ~/.local/state if not set.)
 .
 .SS Groups
 .TP
@@ -702,7 +752,7 @@ used as mailbox name separators as well.
 .
 .TP
 \fBChannel\fR[\fBs\fR] \fIchannel\fR[\fB:\fIbox\fR[\fB,\fR...]] ...
-Add the specified channels to the group. This option can be specified multiple
+Add the specified Channels to the Group. This option can be specified multiple
 times within a Group.
 .
 .SS Global Options
@@ -741,30 +791,33 @@ If \fBmbsync\fR's output is connected to a console, it will print progress
 counters by default. The output will look like this:
 .P
 .in +4
-C: 1/2  B: 3/4  F: +13/13 *23/42 #0/0  N: +0/7 *0/0 #0/0
+C: 1/2  B: 3/4  F: +13/13 *23/42 #0/0 -0/0  N: +0/7 *0/0 #0/0 -0/0
 .in -4
 .P
-This represents the cumulative progress over channels, boxes, and messages
+This represents the cumulative progress over Channels, boxes, and messages
 affected on the far and near side, respectively.
 The message counts represent added messages, messages with updated flags,
-and trashed messages, respectively.
+trashed messages, and expunged messages, respectively.
 No attempt is made to calculate the totals in advance, so they grow over
 time as more information is gathered.
+.P
+Irrespective of output redirection, \fBmbsync\fR will print a summary
+of the above in plain language upon completion, except in quiet mode.
 .
 .SH RECOMMENDATIONS
 Make sure your IMAP server does not auto-expunge deleted messages - it is
 slow, and semantically somewhat questionable. Specifically, Gmail needs to
 be configured not to do it.
 .P
-By default, \fBmbsync\fR will not delete any messages - deletions are
-propagated by marking the messages as deleted on the remote store.
+By default, \fBmbsync\fR will not delete any messages - expunges are
+propagated by marking the messages as deleted in the opposite Store.
 Once you have verified that your setup works, you will typically want to
 set \fBExpunge\fR to \fBBoth\fR, so that deletions become effective.
 .P
 \fBmbsync\fR's built-in trash functionality relies on \fBmbsync\fR doing
 the expunging of deleted messages. This is the case when it propagates
 deletions of previously propagated messages, and the trash is on the target
-store (typically your IMAP server).
+Store (typically your IMAP server).
 .br
 However, when you intend \fBmbsync\fR to trash messages which were not
 propagated yet, the MUA must mark the messages as deleted without expunging
@@ -789,6 +842,10 @@ Mutt always does that, while mu4e needs to be configured to do it:
 .in +4
 (setq mu4e-change-filenames-when-moving t)
 .in -4
+.br
+The general expectation is that a completely new filename is generated
+as if the message was new, but stripping the \fB,U=\fIxxx\fR infix is
+sufficient as well.
 .
 .SH INHERENT PROBLEMS
 Changes done after \fBmbsync\fR has retrieved the message list will not be
@@ -803,11 +860,18 @@ There is no risk as long as the IMAP mailbox is accessed by only one client
 .
 .SH FILES
 .TP
+\fB$XDG_CONFIG_HOME/isyncrc\fR (usually \fB~/.config/isyncrc\fR)
+Default configuration file.
+See also the example file in the documentation directory.
+.TP
+\fB$XDG_STATE_HOME/isync/\fR (usually \fB~/.local/state/isync/\fR)
+Directory containing synchronization state files.
+.TP
 .B ~/.mbsyncrc
-Default configuration file
+Legacy configuration file.
 .TP
 .B ~/.mbsync/
-Directory containing synchronization state files
+Legacy directory containing synchronization state files.
 .
 .SH SEE ALSO
 mdconvert(1), mutt(1), maildir(5)

src/mbsyncrc.sample

@@ -4,6 +4,10 @@
 Expunge None
 Create Both
 
+# More sections follow
+#
+# !!!! Note that empty lines delimit sections !!!!
+
 MaildirStore local
 Path ~/Mail/
 Trash Trash
@@ -32,7 +36,7 @@ Sync PullNew Push
 IMAPStore personal
 Host host.play.com
 Port 6789
-RequireSSL no
+TLSType None
 
 Channel personal
 Far :personal:
@@ -53,16 +57,42 @@ Group boxes
 Channels work personal remote
 
 
+# Due to the divergent Path suffixes, it's possible to have
+# multiple Stores homed in the same directory.
+# You could even put them all directly into $HOME.
+
+MaildirStore local-personal
+Path ~/Mail/personal-
+Inbox ~/Mail/personal-INBOX
+
+MaildirStore local-work
+Path ~/Mail/work-
+# Just because.
+Inbox ~/Mail/w0rk_InBoX
+
+Channel personal-joined
+Far :personal:
+Near :local-personal:
+Paterns *
+
+Channel work-joined
+Far :work:
+Near :local-work:
+Paterns *
+
+Group joined personal-joined work-joined
+
+
 IMAPStore st1
 Host st1.domain.com
-RequireCRAM yes
+AuthMech CRAM-MD5
+# Omit if you want to use the system certificate store.
 CertificateFile ~/.st1-certificate.crt
 
 IMAPStore st2
 Host imap.another-domain.com
 Path non-standard/
-RequireSSL no
-UseTLSv1 no
+TLSVersions -1.2
 
 Channel rst
 Far :st1:somebox
@@ -71,6 +101,7 @@ Near :st2:
 
 IMAPAccount server
 Host imaps:foo.bar.com
+# Omit if you want to use the system certificate store.
 CertificateFile ~/.server-certificate.crt
 
 IMAPStore server

src/mdconvert.1

@@ -1,33 +1,21 @@
-.ig
-\" mdconvert - Maildir mailbox UID storage scheme converter
-\" Copyright (C) 2004 Oswald Buddenhagen <ossi@users.sf.net>
-\"
-\"  This program is free software; you can redistribute it and/or modify
-\"  it under the terms of the GNU General Public License as published by
-\"  the Free Software Foundation; either version 2 of the License, or
-\"  (at your option) any later version.
-\"
-\"  This program is distributed in the hope that it will be useful,
-\"  but WITHOUT ANY WARRANTY; without even the implied warranty of
-\"  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-\"  GNU General Public License for more details.
-\"
-\"  You should have received a copy of the GNU General Public License
-\"  along with this program.  If not, see <http://www.gnu.org/licenses/>.
-..
+.\" SPDX-FileCopyrightText: 2004-2022 Oswald Buddenhagen <ossi@users.sf.net>
+.\" SPDX-License-Identifier: GPL-2.0-or-later
+.\"
+.\" mdconvert - Maildir mailbox UID storage scheme converter
+.
 .TH mdconvert 1 "2004 Mar 27"
-..
+.
 .SH NAME
 mdconvert - Maildir mailbox UID storage scheme converter
-..
+.
 .SH SYNOPSIS
 \fBmdconvert\fR [\fIoptions\fR ...] \fImailbox\fR ...
-..
+.
 .SH DESCRIPTION
 \fBmdconvert\fR converts Maildir mailboxes between the two UID storage schemes
 supported by \fBmbsync\fR. See \fBmbsync\fR's manual page for details on these
 schemes.
-..
+.
 .SH OPTIONS
 .TP
 \fB-a\fR, \fB--alt\fR
@@ -42,9 +30,9 @@ Displays a summary of command line options.
 .TP
 \fB-v\fR, \fB--version\fR
 Displays version information.
-..
+.
 .SH SEE ALSO
 mbsync(1)
-..
+.
 .SH AUTHOR
 Written and maintained by Oswald Buddenhagen <ossi@users.sf.net>.

src/mdconvert.c

@@ -1,19 +1,7 @@
+// SPDX-FileCopyrightText: 2004-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * mdconvert - Maildir UID scheme converter
- * Copyright (C) 2004 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <autodefs.h>
@@ -263,8 +251,9 @@ main( int argc, char **argv )
 		} else if (argv[oint][0] == '-') {
 			fprintf( stderr, "Unrecognized option '%s'. Try " EXE " -h\n", argv[oint] );
 			return 1;
-		} else
+		} else {
 			break;
+		}
 	}
 	if (oint == argc) {
 		fprintf( stderr, "Mailbox specification missing. Try " EXE " -h\n" );

src/socket.c

@@ -1,34 +1,14 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-FileCopyrightText: 2004 Theodore Y. Ts'o <tytso@mit.edu>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2008,2010,2011, 2013 Oswald Buddenhagen <ossi@users.sf.net>
- * Copyright (C) 2004 Theodore Y. Ts'o <tytso@mit.edu>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #include "socket.h"
 
-#include <assert.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stddef.h>
 #include <errno.h>
-#include <string.h>
 #include <fcntl.h>
 #include <sys/socket.h>
 #include <sys/ioctl.h>
@@ -48,6 +28,7 @@
 #endif
 
 enum {
+	SCK_RESOLVING,
 	SCK_CONNECTING,
 #ifdef HAVE_LIBSSL
 	SCK_STARTTLS,
@@ -402,7 +383,7 @@ socket_start_deflate( conn_t *conn )
 {
 	int result;
 
-	conn->in_z = nfcalloc( sizeof(*conn->in_z) );
+	conn->in_z = nfzalloc( sizeof(*conn->in_z) );
 	result = inflateInit2(
 			conn->in_z,
 			-15 /* Use raw deflate */
@@ -412,7 +393,7 @@ socket_start_deflate( conn_t *conn )
 		abort();
 	}
 
-	conn->out_z = nfcalloc( sizeof(*conn->out_z) );
+	conn->out_z = nfzalloc( sizeof(*conn->out_z) );
 	result = deflateInit2(
 			conn->out_z,
 			Z_DEFAULT_COMPRESSION, /* Compression level */
@@ -427,6 +408,7 @@ socket_start_deflate( conn_t *conn )
 	}
 
 	init_wakeup( &conn->z_fake, z_fake_cb, conn );
+	conn->readsz = 0;  // This optimization makes no sense past this point
 }
 #endif /* HAVE_LIBZ */
 
@@ -434,6 +416,7 @@ static void socket_fd_cb( int, void * );
 static void socket_fake_cb( void * );
 static void socket_timeout_cb( void * );
 
+static void socket_resolve( conn_t * );
 static void socket_connect_one( conn_t * );
 static void socket_connect_next( conn_t * );
 static void socket_connect_failed( conn_t * );
@@ -441,15 +424,21 @@ static void socket_connected( conn_t * );
 static void socket_connect_bail( conn_t * );
 
 static void
-socket_open_internal( conn_t *sock, int fd )
+socket_register_internal( conn_t *sock, int fd )
 {
 	sock->fd = fd;
-	fcntl( fd, F_SETFL, O_NONBLOCK );
 	init_notifier( &sock->notify, fd, socket_fd_cb, sock );
 	init_wakeup( &sock->fd_fake, socket_fake_cb, sock );
 	init_wakeup( &sock->fd_timeout, socket_timeout_cb, sock );
 }
 
+static void
+socket_open_internal( conn_t *sock, int fd )
+{
+	fcntl( fd, F_SETFL, O_NONBLOCK );
+	socket_register_internal( sock, fd );
+}
+
 static void
 socket_close_internal( conn_t *sock )
 {
@@ -460,32 +449,6 @@ socket_close_internal( conn_t *sock )
 	sock->fd = -1;
 }
 
-#ifndef HAVE_IPV6
-struct addr_info {
-	struct addr_info *ai_next;
-	struct sockaddr_in ai_addr[1];
-};
-
-#define freeaddrinfo(ai) free( ai )
-
-static struct addr_info *
-init_addrinfo( struct hostent *he )
-{
-	uint naddr = 0;
-	for (char **addr = he->h_addr_list; *addr; addr++)
-		naddr++;
-	struct addr_info *caddr = nfcalloc( naddr * sizeof(struct addrinfo) );
-	struct addr_info *ret, **caddrp = &ret;
-	for (char **addr = he->h_addr_list; *addr; addr++, caddr++) {
-		caddr->ai_addr->sin_family = AF_INET;
-		memcpy( &caddr->ai_addr->sin_addr.s_addr, *addr, sizeof(struct in_addr) );
-		*caddrp = caddr;
-		caddrp = &caddr->ai_next;
-	}
-	return ret;
-}
-#endif
-
 void
 socket_connect( conn_t *sock, void (*cb)( int ok, void *aux ) )
 {
@@ -520,77 +483,202 @@ socket_connect( conn_t *sock, void (*cb)( int ok, void *aux ) )
 		info( "\vok\n" );
 		socket_connected( sock );
 	} else {
-#ifdef HAVE_IPV6
-		int gaierr;
-		struct addrinfo hints;
-
-		memset( &hints, 0, sizeof(hints) );
-		hints.ai_family = AF_UNSPEC;
-		hints.ai_socktype = SOCK_STREAM;
-		hints.ai_flags = AI_ADDRCONFIG;
-		infon( "Resolving %s... ", conf->host );
-		if ((gaierr = getaddrinfo( conf->host, NULL, &hints, &sock->addrs ))) {
-			error( "Error: Cannot resolve server '%s': %s\n", conf->host, gai_strerror( gaierr ) );
-			socket_connect_bail( sock );
-			return;
-		}
-		info( "\vok\n" );
-#else
-		struct hostent *he;
-
-		infon( "Resolving %s... ", conf->host );
-		he = gethostbyname( conf->host );
-		if (!he) {
-			error( "Error: Cannot resolve server '%s': %s\n", conf->host, hstrerror( h_errno ) );
-			socket_connect_bail( sock );
-			return;
-		}
-		info( "\vok\n" );
-
-		sock->addrs = init_addrinfo( he );
-#endif
-		sock->curr_addr = sock->addrs;
-		socket_connect_one( sock );
+		socket_resolve( sock );
 	}
 }
 
+static void
+pipe_write( int fd, void *buf, int len )
+{
+	do {
+		int wrote = write( fd, buf, len );
+		if (wrote < 0) {
+			perror( "write" );
+			_exit( 1 );
+		}
+		buf = ((char *)buf) + wrote;
+		len -= wrote;
+	} while (len);
+}
+
+static void
+socket_resolve( conn_t *sock )
+{
+	info( "Resolving %s...\n", sock->conf->host );
+
+	int pfd[2];
+	if (pipe( pfd )) {
+		perror( "pipe" );
+		exit( 1 );
+	}
+	switch (fork()) {
+	case -1:
+		perror( "fork" );
+		exit( 1 );
+	case 0:
+		break;
+	default:
+		close( pfd[1] );
+		socket_register_internal( sock, pfd[0] );
+		sock->state = SCK_RESOLVING;
+		conf_notifier( &sock->notify, 0, POLLIN );
+		socket_expect_activity( sock, 1 );
+		return;
+	}
+
+#ifdef HAVE_IPV6
+	struct addrinfo *res, hints = { 0 };
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	hints.ai_flags = AI_ADDRCONFIG;
+	int gaierr = getaddrinfo( sock->conf->host, NULL, &hints, &res );
+	pipe_write( pfd[1], &gaierr, sizeof(gaierr) );
+	if (gaierr)
+		_exit( 1 );
+	static_assert( sizeof(((struct addrinfo){ 0 }).ai_family) == sizeof(int), "unexpected size of ai_family" );
+	static_assert( sizeof(struct in_addr) % sizeof(int) == 0, "unexpected size of struct in_addr" );
+	static_assert( sizeof(struct in6_addr) % sizeof(int) == 0, "unexpected size of struct in6_addr" );
+	int nbytes = 0;
+	for (struct addrinfo *cres = res; cres; cres = cres->ai_next) {
+		if (cres->ai_family == AF_INET) {
+			nbytes += sizeof(int) + sizeof(struct in_addr);
+		} else {
+			assert( cres->ai_family == AF_INET6 );
+			nbytes += sizeof(int) + sizeof(struct in6_addr);
+		}
+	}
+	pipe_write( pfd[1], &nbytes, sizeof(nbytes) );
+	for (struct addrinfo *cres = res; cres; cres = cres->ai_next) {
+		pipe_write( pfd[1], &cres->ai_family, sizeof(int) );
+		if (cres->ai_family == AF_INET)
+			pipe_write( pfd[1], &((struct sockaddr_in *)cres->ai_addr)->sin_addr, sizeof(struct in_addr) );
+		else
+			pipe_write( pfd[1], &((struct sockaddr_in6 *)cres->ai_addr)->sin6_addr, sizeof(struct in6_addr) );
+	}
+#else
+	struct hostent *he = gethostbyname( sock->conf->host );
+	int herrno = he ? 0 : h_errno;
+	pipe_write( pfd[1], &herrno, sizeof(herrno) );
+	if (!he)
+		_exit( 1 );
+	static_assert( sizeof(struct in_addr) % sizeof(int) == 0, "unexpected size of struct in_addr" );
+	int nbytes = 0;
+	for (char **addr = he->h_addr_list; *addr; addr++)
+		nbytes += sizeof(struct in_addr);
+	pipe_write( pfd[1], &nbytes, sizeof(nbytes) );
+	for (char **addr = he->h_addr_list; *addr; addr++)
+		pipe_write( pfd[1], *addr, sizeof(struct in_addr) );
+#endif
+	_exit( 0 );
+}
+
+static void
+pipe_read( int fd, void *buf, int len )
+{
+	do {
+		int didrd = read( fd, buf, len );
+		if (didrd < 0) {
+			sys_error( "read" );
+			exit( 1 );
+		}
+		if (!didrd) {
+			error( "read: unexpected EOF\n" );
+			exit( 1 );
+		}
+		buf = ((char *)buf) + didrd;
+		len -= didrd;
+	} while (len);
+}
+
+static void
+socket_resolve_finalize( conn_t *sock )
+{
+	int errcode;
+	pipe_read( sock->fd, &errcode, sizeof(errcode) );
+	if (errcode) {
+#ifdef HAVE_IPV6
+		const char *err = gai_strerror( errcode );
+#else
+		const char *err = hstrerror( errcode );
+#endif
+		error( "Error: Cannot resolve server '%s': %s\n", sock->conf->host, err );
+		socket_close_internal( sock );
+		socket_connect_bail( sock );
+		return;
+	}
+
+	int nbytes;
+	pipe_read( sock->fd, &nbytes, sizeof(nbytes) );
+	char *addrs = nfmalloc( nbytes );
+	pipe_read( sock->fd, addrs, nbytes );
+	sock->curr_addr = sock->addrs = addrs;
+	sock->addrs_end = addrs + nbytes;
+	socket_close_internal( sock );  // Get rid of the pipe
+	socket_connect_one( sock );
+}
+
+static void
+socket_resolve_timeout( conn_t *sock )
+{
+	error( "Error: Cannot resolve server '%s': timeout.\n", sock->conf->host );
+	socket_close_internal( sock );
+	socket_connect_bail( sock );
+}
+
 static void
 socket_connect_one( conn_t *sock )
 {
-	int s;
-#ifdef HAVE_IPV6
-	struct addrinfo *ai;
-#else
-	struct addr_info *ai;
-#endif
-
-	if (!(ai = sock->curr_addr)) {
+	char *ai = sock->curr_addr;
+	if (ai == sock->addrs_end) {
 		error( "No working address found for %s\n", sock->conf->host );
 		socket_connect_bail( sock );
 		return;
 	}
 
+	union {
+		struct sockaddr any;
+		struct sockaddr_in ip4;
 #ifdef HAVE_IPV6
-	if (ai->ai_family == AF_INET6) {
-		struct sockaddr_in6 *in6 = ((struct sockaddr_in6 *)ai->ai_addr);
+		struct sockaddr_in6 ip6;
+#endif
+	} addr;
+
+#ifdef HAVE_IPV6
+	int fam = *(int *)ai;
+	ai += sizeof(int);
+	int addr_len;
+	if (fam == AF_INET6) {
+		addr_len = sizeof(addr.ip6);
+		addr.ip6.sin6_addr = *(struct in6_addr *)ai;
+		addr.ip6.sin6_flowinfo = 0;
+		addr.ip6.sin6_scope_id = 0;
+		ai += sizeof(struct in6_addr);
+	} else {
+		addr_len = sizeof(addr.ip4);
+#else
+	const int fam = AF_INET;
+	const int addr_len = sizeof(addr.ip4);
+	{
+#endif
+		addr.ip4.sin_addr = *(struct in_addr *)ai;
+		ai += sizeof(struct in_addr);
+	}
+	sock->curr_addr = ai;
+
+#ifdef HAVE_IPV6
+	if (fam == AF_INET6) {
 		char sockname[64];
-		in6->sin6_port = htons( sock->conf->port );
+		inet_ntop( fam, &addr.ip6.sin6_addr, sockname, sizeof(sockname) );
 		nfasprintf( &sock->name, "%s ([%s]:%hu)",
-		            sock->conf->host, inet_ntop( AF_INET6, &in6->sin6_addr, sockname, sizeof(sockname) ), sock->conf->port );
+		            sock->conf->host, sockname, sock->conf->port );
 	} else
 #endif
 	{
-		struct sockaddr_in *in = ((struct sockaddr_in *)ai->ai_addr);
-		in->sin_port = htons( sock->conf->port );
 		nfasprintf( &sock->name, "%s (%s:%hu)",
-		            sock->conf->host, inet_ntoa( in->sin_addr ), sock->conf->port );
+		            sock->conf->host, inet_ntoa( addr.ip4.sin_addr ), sock->conf->port );
 	}
 
-#ifdef HAVE_IPV6
-	s = socket( ai->ai_family, SOCK_STREAM, 0 );
-#else
-	s = socket( PF_INET, SOCK_STREAM, 0 );
-#endif
+	int s = socket( fam, SOCK_STREAM, 0 );
 	if (s < 0) {
 		socket_connect_next( sock );
 		return;
@@ -598,11 +686,9 @@ socket_connect_one( conn_t *sock )
 	socket_open_internal( sock, s );
 
 	infon( "Connecting to %s... ", sock->name );
-#ifdef HAVE_IPV6
-	if (connect( s, ai->ai_addr, ai->ai_addrlen )) {
-#else
-	if (connect( s, ai->ai_addr, sizeof(*ai->ai_addr) )) {
-#endif
+	addr.any.sa_family = fam;
+	addr.ip4.sin_port = htons( sock->conf->port );  // Aliased for ip6
+	if (connect( s, &addr.any, addr_len )) {
 		if (errno != EINPROGRESS) {
 			socket_connect_failed( sock );
 			return;
@@ -623,7 +709,6 @@ socket_connect_next( conn_t *conn )
 	sys_error( "Cannot connect to %s", conn->name );
 	free( conn->name );
 	conn->name = NULL;
-	conn->curr_addr = conn->curr_addr->ai_next;
 	socket_connect_one( conn );
 }
 
@@ -637,10 +722,8 @@ socket_connect_failed( conn_t *conn )
 static void
 socket_connected( conn_t *conn )
 {
-	if (conn->addrs) {
-		freeaddrinfo( conn->addrs );
-		conn->addrs = NULL;
-	}
+	free( conn->addrs );
+	conn->addrs = NULL;
 	conf_notifier( &conn->notify, 0, POLLIN );
 	socket_expect_activity( conn, 0 );
 	conn->state = SCK_READY;
@@ -650,10 +733,8 @@ socket_connected( conn_t *conn )
 static void
 socket_cleanup_names( conn_t *conn )
 {
-	if (conn->addrs) {
-		freeaddrinfo( conn->addrs );
-		conn->addrs = NULL;
-	}
+	free( conn->addrs );
+	conn->addrs = NULL;
 	free( conn->name );
 	conn->name = NULL;
 }
@@ -739,6 +820,34 @@ do_read( conn_t *sock, char *buf, uint len )
 	return n;
 }
 
+static void
+socket_filled( conn_t *conn, uint len )
+{
+	uint off = conn->offset;
+	uint cnt = conn->bytes + len;
+	conn->bytes = cnt;
+	if (conn->wanted) {
+		// Fulfill as much of the request as still fits into the buffer,
+		// but avoid chopping up the actual socket reads
+		if (cnt < conn->wanted && off + cnt < sizeof(conn->buf) - conn->readsz)
+			return;
+	} else {
+		// Need a full line
+		char *s = conn->buf + off;
+		char *p = memchr( s + conn->scanoff, '\n', cnt - conn->scanoff );
+		if (!p) {
+			conn->scanoff = cnt;
+			if (off && off + cnt >= sizeof(conn->buf) - conn->readsz) {
+				memmove( conn->buf, conn->buf + off, cnt );
+				conn->offset = 0;
+			}
+			return;
+		}
+		conn->scanoff = (uint)(p - s);
+	}
+	conn->read_callback( conn->callback_aux );
+}
+
 #ifdef HAVE_LIBZ
 static void
 socket_fill_z( conn_t *sock )
@@ -765,10 +874,8 @@ socket_fill_z( conn_t *sock )
 	if (!sock->in_z->avail_out)
 		conf_wakeup( &sock->z_fake, 0 );
 
-	if ((len = (uint)((char *)sock->in_z->next_out - buf))) {
-		sock->bytes += len;
-		sock->read_callback( sock->callback_aux );
-	}
+	if ((len = (uint)((char *)sock->in_z->next_out - buf)))
+		socket_filled( sock, len );
 }
 #endif
 
@@ -798,8 +905,13 @@ socket_fill( conn_t *sock )
 		if ((n = do_read( sock, buf, len )) <= 0)
 			return;
 
-		sock->bytes += (uint)n;
-		sock->read_callback( sock->callback_aux );
+		// IIR filter for tracking average size of bulk reads.
+		// We use this to optimize the free space at the end of the
+		// buffer, hence the factor of 1.5.
+		if (n >= MIN_BULK_READ)
+			sock->readsz = (sock->readsz * 3 + n * 3 / 2) / 4;
+
+		socket_filled( sock, (uint)n );
 	}
 }
 
@@ -810,44 +922,70 @@ socket_expect_activity( conn_t *conn, int expect )
 		conf_wakeup( &conn->fd_timeout, expect ? conn->conf->timeout : -1 );
 }
 
-int
-socket_read( conn_t *conn, char *buf, uint len )
+void
+socket_expect_eof( conn_t *sock )
 {
-	uint n = conn->bytes;
-	if (!n && conn->state == SCK_EOF)
-		return -1;
-	if (n > len)
-		n = len;
-	memcpy( buf, conn->buf + conn->offset, n );
-	if (!(conn->bytes -= n))
-		conn->offset = 0;
-	else
-		conn->offset += n;
-	return (int)n;
+#ifdef SSL_OP_IGNORE_UNEXPECTED_EOF  // implies HAVE_LIBSSL
+	if (sock->ssl)
+		SSL_set_options( sock->ssl, SSL_OP_IGNORE_UNEXPECTED_EOF );
+#endif
+}
+
+void
+socket_expect_bytes( conn_t *conn, uint len )
+{
+	conn->wanted = len;
+	uint off = conn->offset;
+	if (off) {
+		uint cnt = conn->bytes;
+		if (off + len > sizeof(conn->buf) ||
+		    off + cnt >= sizeof(conn->buf) - conn->readsz) {
+			memmove( conn->buf, conn->buf + off, cnt );
+			conn->offset = 0;
+		}
+	}
 }
 
 char *
-socket_read_line( conn_t *b )
+socket_read( conn_t *conn, uint min_len, uint max_len, uint *out_len )
 {
-	char *p, *s;
-	uint n;
+	assert( min_len > 0 );
+	assert( min_len <= sizeof(conn->buf) );
+	assert( min_len <= max_len );
 
-	s = b->buf + b->offset;
-	p = memchr( s + b->scanoff, '\n', b->bytes - b->scanoff );
-	if (!p) {
-		b->scanoff = b->bytes;
-		if (b->offset + b->bytes == sizeof(b->buf)) {
-			memmove( b->buf, b->buf + b->offset, b->bytes );
-			b->offset = 0;
-		}
-		if (b->state == SCK_EOF)
+	uint off = conn->offset;
+	uint cnt = conn->bytes;
+	if (cnt < min_len) {
+		if (conn->state == SCK_EOF)
 			return (void *)~0;
 		return NULL;
 	}
-	n = (uint)(p + 1 - s);
-	b->offset += n;
-	b->bytes -= n;
-	b->scanoff = 0;
+	uint n = (cnt < max_len) ? cnt : max_len;
+	cnt -= n;
+	conn->offset = cnt ? off + n : 0;
+	conn->bytes = cnt;
+	*out_len = n;
+	return conn->buf + off;
+}
+
+char *
+socket_read_line( conn_t *conn )
+{
+	uint off = conn->offset;
+	uint cnt = conn->bytes;
+	char *s = conn->buf + off;
+	char *p = memchr( s + conn->scanoff, '\n', cnt - conn->scanoff );
+	if (!p) {
+		if (conn->state == SCK_EOF)
+			return (void *)~0;
+		conn->scanoff = cnt;
+		return NULL;
+	}
+	uint n = (uint)(p + 1 - s);
+	cnt -= n;
+	conn->offset = cnt ? off + n : 0;
+	conn->bytes = cnt;
+	conn->scanoff = 0;
 	if (p != s && p[-1] == '\r')
 		p--;
 	*p = 0;
@@ -1068,6 +1206,11 @@ socket_fd_cb( int events, void *aux )
 {
 	conn_t *conn = (conn_t *)aux;
 
+	if (conn->state == SCK_RESOLVING) {
+		socket_resolve_finalize( conn );
+		return;
+	}
+
 	if ((events & POLLERR) || conn->state == SCK_CONNECTING) {
 		int soerr;
 		socklen_t selen = sizeof(soerr);
@@ -1130,7 +1273,9 @@ socket_timeout_cb( void *aux )
 {
 	conn_t *conn = (conn_t *)aux;
 
-	if (conn->state == SCK_CONNECTING) {
+	if (conn->state == SCK_RESOLVING) {
+		socket_resolve_timeout( conn );
+	} else if (conn->state == SCK_CONNECTING) {
 		errno = ETIMEDOUT;
 		socket_connect_failed( conn );
 	} else {

src/socket.h

@@ -1,23 +1,8 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #ifndef SOCKET_H
@@ -72,11 +57,7 @@ typedef struct {
 	int fd;
 	int state;
 	const server_conf_t *conf; /* needed during connect */
-#ifdef HAVE_IPV6
-	struct addrinfo *addrs, *curr_addr; /* needed during connect */
-#else
-	struct addr_info *addrs, *curr_addr; /* needed during connect */
-#endif
+	char *addrs, *addrs_end, *curr_addr;  // needed during connect; assumed to be int-aligned
 	char *name;
 #ifdef HAVE_LIBSSL
 	SSL *ssl;
@@ -114,12 +95,17 @@ typedef struct {
 	uint offset; /* start of filled bytes in buffer */
 	uint bytes; /* number of filled bytes in buffer */
 	uint scanoff; /* offset to continue scanning for newline at, relative to 'offset' */
+	uint wanted;  // try to accumulate that many bytes before calling back; 0 => full line
+	uint readsz;  // average size of bulk reads from the underlying socket, times 1.5
 	char buf[100000];
 #ifdef HAVE_LIBZ
 	char z_buf[100000];
 #endif
 } conn_t;
 
+// Shorter reads are assumed to be limited by round-trips.
+#define MIN_BULK_READ 1000
+
 /* call this before doing anything with the socket */
 static INLINE void socket_init( conn_t *conn,
                                 const server_conf_t *conf,
@@ -136,14 +122,19 @@ static INLINE void socket_init( conn_t *conn,
 	conn->fd = -1;
 	conn->name = NULL;
 	conn->write_buf_append = &conn->write_buf;
+	conn->wanted = 1;
+	conn->readsz = MIN_BULK_READ * 3 / 2;
 }
 void socket_connect( conn_t *conn, void (*cb)( int ok, void *aux ) );
 void socket_start_tls(conn_t *conn, void (*cb)( int ok, void *aux ) );
 void socket_start_deflate( conn_t *conn );
 void socket_close( conn_t *sock );
 void socket_expect_activity( conn_t *sock, int expect );
-int socket_read( conn_t *sock, char *buf, uint len ); /* never waits */
-char *socket_read_line( conn_t *sock ); /* don't free return value; never waits */
+void socket_expect_eof( conn_t *sock );
+void socket_expect_bytes( conn_t *sock, uint len );
+// Don't free return values. These functions never wait.
+char *socket_read( conn_t *conn, uint min_len, uint max_len, uint *out_len );
+char *socket_read_line( conn_t *conn );
 typedef enum { KeepOwn = 0, GiveOwn } ownership_t;
 typedef struct {
 	char *buf;

src/sync.h

@@ -1,59 +1,63 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2010-2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #ifndef SYNC_H
 #define SYNC_H
 
 #include "driver.h"
+#include "sync_enum.h"
 
 #define F 0  // far side
 #define N 1  // near side
 
-#define OP_NEW             (1<<0)
-#define OP_RENEW           (1<<1)
-#define OP_DELETE          (1<<2)
-#define OP_FLAGS           (1<<3)
-#define  OP_MASK_TYPE      (OP_NEW|OP_RENEW|OP_DELETE|OP_FLAGS) /* asserted in the target ops */
-#define OP_EXPUNGE         (1<<4)
-#define OP_CREATE          (1<<5)
-#define OP_REMOVE          (1<<6)
-#define XOP_PUSH           (1<<8)
-#define XOP_PULL           (1<<9)
-#define  XOP_MASK_DIR      (XOP_PUSH|XOP_PULL)
-#define XOP_HAVE_TYPE      (1<<10)
-// The following must all have the same bit shift from the corresponding OP_* flags.
-#define XOP_HAVE_EXPUNGE   (1<<11)
-#define XOP_HAVE_CREATE    (1<<12)
-#define XOP_HAVE_REMOVE    (1<<13)
+BIT_ENUM(
+	OP_NEW,
+	OP_OLD,
+	OP_UPGRADE,
+	OP_GONE,
+	OP_FLAGS,
+	OP_EXPUNGE,
+	OP_EXPUNGE_SOLO,
+	OP_CREATE,
+	OP_REMOVE,
+
+	XOP_PUSH,
+	XOP_PULL,
+	XOP_HAVE_TYPE,  // Aka mode; have at least one of dir and type (see below)
+	// The following must all have the same bit shift from the corresponding OP_* flags.
+	XOP_HAVE_EXPUNGE,
+	XOP_HAVE_EXPUNGE_SOLO,
+	XOP_HAVE_CREATE,
+	XOP_HAVE_REMOVE,
+	// ... until here.
+	XOP_TYPE_NOOP,
+	// ... and here again from scratch.
+	XOP_EXPUNGE_NOOP,
+	XOP_EXPUNGE_SOLO_NOOP,
+	XOP_CREATE_NOOP,
+	XOP_REMOVE_NOOP,
+)
+
+#define OP_DFLT_TYPE (OP_NEW | OP_UPGRADE | OP_GONE | OP_FLAGS)
+#define OP_MASK_TYPE (OP_DFLT_TYPE | OP_OLD)  // Asserted in the target side ops
+#define XOP_MASK_DIR (XOP_PUSH | XOP_PULL)
+
+DECL_BIT_FORMATTER_FUNCTION(ops, OP)
 
 typedef struct channel_conf {
 	struct channel_conf *next;
 	const char *name;
 	store_conf_t *stores[2];
 	const char *boxes[2];
-	char *sync_state;
+	const char *sync_state;
 	string_list_t *patterns;
 	int ops[2];
 	int max_messages;  // For near side only.
+	int expire_side;
 	signed char expire_unread;
 	char use_internal_date;
 } channel_conf_t;
@@ -68,13 +72,18 @@ extern channel_conf_t global_conf;
 extern channel_conf_t *channels;
 extern group_conf_t *groups;
 
+extern uint BufferLimit;
+
+extern int new_total[2], new_done[2];
+extern int flags_total[2], flags_done[2];
+extern int trash_total[2], trash_done[2];
+extern int expunge_total[2], expunge_done[2];
+
 extern const char *str_fn[2], *str_hl[2];
 
 #define SYNC_OK       0 /* assumed to be 0 */
 #define SYNC_FAIL     1
 #define SYNC_BAD(fn)  (4<<(fn))
-#define SYNC_NOGOOD   16 /* internal */
-#define SYNC_CANCELED 32 /* internal */
 
 #define BOX_POSSIBLE -1
 #define BOX_ABSENT    0

src/sync_msg_cvt.c

@@ -0,0 +1,226 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#include "sync_p.h"
+
+static void
+copy_msg_bytes( char **out_ptr, const char *in_buf, uint *in_idx, uint in_len, int in_cr, int out_cr )
+{
+	char *out = *out_ptr;
+	uint idx = *in_idx;
+	if (out_cr != in_cr) {
+		if (out_cr) {
+			for (char c, pc = 0; idx < in_len; idx++) {
+				if (((c = in_buf[idx]) == '\n') && (pc != '\r'))
+					*out++ = '\r';
+				*out++ = c;
+				pc = c;
+			}
+		} else {
+			for (char c, pc = 0; idx < in_len; idx++) {
+				if (((c = in_buf[idx]) == '\n') && (pc == '\r'))
+					out--;
+				*out++ = c;
+				pc = c;
+			}
+		}
+	} else {
+		memcpy( out, in_buf + idx, in_len - idx );
+		out += in_len - idx;
+		idx = in_len;
+	}
+	*out_ptr = out;
+	*in_idx = idx;
+}
+
+char *
+copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars )
+{
+	char *in_buf = vars->data.data;
+	uint in_len = vars->data.len;
+	uint idx = 0, sbreak = 0, ebreak = 0, break2 = UINT_MAX;
+	uint lines = 0, hdr_crs = 0, bdy_crs = 0, app_cr = 0, extra = 0;
+	uint add_subj = 0, fix_tuid = 0, fix_subj = 0, fix_hdr = 0, end_hdr = 0;
+
+	if (vars->srec) {
+		for (;;) {
+			uint start = idx;
+			uint line_cr = 0;
+			uint got_line = 0;
+			char pc = 0;
+			while (idx < in_len) {
+				char c = in_buf[idx++];
+				if (c == '\n') {
+					if (pc == '\r')
+						line_cr = 1;
+					got_line = 1;
+					break;
+				}
+				pc = c;
+			}
+			if (!ebreak && starts_with_upper( in_buf + start, (int)(in_len - start), "X-TUID: ", 8 )) {
+				extra = (sbreak = start) - (ebreak = idx);
+				if (!vars->minimal)
+					break;
+				continue;
+			}
+			if (break2 == UINT_MAX && vars->minimal &&
+			    starts_with_upper( in_buf + start, (int)(in_len - start), "SUBJECT:", 8 )) {
+				break2 = start + 8;
+				if (break2 < in_len && in_buf[break2] == ' ')
+					break2++;
+			}
+			hdr_crs += line_cr;
+			if (got_line) {
+				lines++;
+				if (idx - line_cr - 1 != start)
+					continue;
+				// Empty line => end of headers
+			} else {
+				// The line is incomplete.
+				if (pc == '\r')
+					idx--;  // For simplicity, move back before trailing CR
+				if (idx != start) {
+					// The line is non-empty, so schedule completing it
+					fix_hdr = 1;
+					// ... and put our headers after it. (It would seem easier
+					// to prepend them, as then we could avoid the fixing - but
+					// the line might be a continuation. We could also prepend
+					// it to _all_ pre-exiting headers, but then we would risk
+					// masking an (incorrectly present) leading 'From ' header.)
+					start = idx;
+				}
+				end_hdr = 1;
+			}
+			if (!ebreak) {
+				sbreak = ebreak = start;
+				fix_tuid = fix_hdr;
+				fix_hdr = 0;
+			}
+			if (vars->minimal) {
+				in_len = idx;
+				if (break2 == UINT_MAX) {
+					break2 = start;
+					add_subj = 1;
+					fix_subj = fix_hdr;
+					fix_hdr = 0;
+				}
+			} else {
+				fix_hdr = 0;
+				end_hdr = 0;
+			}
+			break;
+		}
+		app_cr = out_cr && (!in_cr || hdr_crs || !lines);
+		if (fix_tuid || fix_subj || fix_hdr)
+			extra += app_cr + 1;
+		if (end_hdr)
+			extra += app_cr + 1;
+		extra += 8 + TUIDL + app_cr + 1;
+	}
+	if (out_cr != in_cr) {
+		for (char pc = 0; idx < in_len; idx++) {
+			char c = in_buf[idx];
+			if (c == '\n') {
+				lines++;
+				if (pc == '\r')
+					bdy_crs++;
+			}
+			pc = c;
+		}
+		extra -= hdr_crs + bdy_crs;
+		if (out_cr)
+			extra += lines;
+	}
+
+	uint dummy_msg_len = 0;
+	char dummy_msg_buf[256];
+	static const char dummy_pfx[] = "[placeholder] ";
+	static const char dummy_subj[] = "Subject: [placeholder] (No Subject)";
+	static const char dummy_msg[] =
+		"Having a size of %s, this message is over the MaxSize limit.%s"
+		"Flag it and sync again (Sync mode Upgrade) to fetch its real contents.%s";
+	static const char dummy_flag[] =
+		"%s"
+		"The original message is flagged as important.%s";
+
+	if (vars->minimal) {
+		char sz[32];
+
+		if (vars->msg->size < 1024000)
+			sprintf( sz, "%dKiB", (int)(vars->msg->size >> 10) );
+		else
+			sprintf( sz, "%.1fMiB", vars->msg->size / 1048576. );
+		const char *nl = app_cr ? "\r\n" : "\n";
+		dummy_msg_len = (uint)sprintf( dummy_msg_buf, dummy_msg, sz, nl, nl );
+		if (vars->data.flags & F_FLAGGED) {
+			vars->data.flags &= ~F_FLAGGED;
+			dummy_msg_len += (uint)sprintf( dummy_msg_buf + dummy_msg_len, dummy_flag, nl, nl );
+		}
+		extra += dummy_msg_len;
+		extra += add_subj ? strlen(dummy_subj) + app_cr + 1 : strlen(dummy_pfx);
+	}
+
+#define ADD_NL() \
+		do { \
+			if (app_cr) \
+				*out_buf++ = '\r'; \
+			*out_buf++ = '\n'; \
+		} while (0)
+
+	vars->data.len = in_len + extra;
+	if (vars->data.len > INT_MAX) {
+		free( in_buf );
+		return "is too big after conversion";
+	}
+	char *out_buf = vars->data.data = nfmalloc( vars->data.len );
+	idx = 0;
+	if (vars->srec) {
+		if (break2 < sbreak) {
+			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
+			memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
+			out_buf += strlen(dummy_pfx);
+		}
+		copy_msg_bytes( &out_buf, in_buf, &idx, sbreak, in_cr, out_cr );
+
+		if (fix_tuid)
+			ADD_NL();
+		memcpy( out_buf, "X-TUID: ", 8 );
+		out_buf += 8;
+		memcpy( out_buf, vars->srec->tuid, TUIDL );
+		out_buf += TUIDL;
+		ADD_NL();
+		idx = ebreak;
+
+		if (break2 != UINT_MAX && break2 >= sbreak) {
+			copy_msg_bytes( &out_buf, in_buf, &idx, break2, in_cr, out_cr );
+			if (!add_subj) {
+				memcpy( out_buf, dummy_pfx, strlen(dummy_pfx) );
+				out_buf += strlen(dummy_pfx);
+			} else {
+				if (fix_subj)
+					ADD_NL();
+				memcpy( out_buf, dummy_subj, strlen(dummy_subj) );
+				out_buf += strlen(dummy_subj);
+				ADD_NL();
+			}
+		}
+	}
+	copy_msg_bytes( &out_buf, in_buf, &idx, in_len, in_cr, out_cr );
+
+	if (vars->minimal) {
+		if (end_hdr) {
+			if (fix_hdr)
+				ADD_NL();
+			ADD_NL();
+		}
+		memcpy( out_buf, dummy_msg_buf, dummy_msg_len );
+	}
+
+	free( in_buf );
+	return NULL;
+}

src/sync_p.h

@@ -0,0 +1,116 @@
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#define DEBUG_FLAG DEBUG_SYNC
+
+#include "sync.h"
+#include "sync_p_enum.h"
+
+BIT_ENUM(
+	S_DEAD,         // ephemeral: the entry was killed and should be ignored
+	S_EXPIRE,       // the entry is being expired (expire-side message removal scheduled)
+	S_EXPIRED,      // the entry is expired (expire-side message removal confirmed)
+	S_NEXPIRE,      // temporary: new expiration state
+	S_PENDING,      // the entry is new and awaits propagation (possibly a retry)
+	S_DUMMY(2),     // f/n message is only a placeholder
+	S_SKIPPED,      // pre-1.4 legacy: the entry was not propagated (message is too big)
+	S_GONE(2),      // ephemeral: f/n message has been expunged
+	S_DEL(2),       // ephemeral: f/n message would be subject to non-selective expunge
+	S_DELETE,       // ephemeral: flags propagation is a deletion
+	S_UPGRADE,      // ephemeral: upgrading placeholder, do not apply MaxSize
+	S_PURGE,        // ephemeral: placeholder is being nuked
+	S_PURGED,       // ephemeral: placeholder was nuked
+)
+
+// This is the persistent status of the sync record, with regard to the journal.
+#define S_LOGGED (S_EXPIRE | S_EXPIRED | S_PENDING | S_DUMMY(F) | S_DUMMY(N) | S_SKIPPED)
+
+typedef struct sync_rec {
+	struct sync_rec *next;
+	/* string_list_t *keywords; */
+	uint uid[2];
+	message_t *msg[2];
+	ushort status;
+	uchar flags, pflags, aflags[2], dflags[2];
+	char tuid[TUIDL];
+} sync_rec_t;
+
+static_assert_bits(F, sync_rec_t, flags);
+static_assert_bits(S, sync_rec_t, status);
+
+typedef struct {
+	int t[2];
+	void (*cb)( int sts, void *aux ), *aux;
+	char *dname, *jname, *nname, *lname, *box_name[2];
+	FILE *jfp, *nfp;
+	sync_rec_t *srecs, **srecadd;
+	channel_conf_t *chan;
+	store_t *ctx[2];
+	driver_t *drv[2];
+	const char *orig_name[2];
+	message_t *msgs[2], *new_msgs[2];
+	uint_array_alloc_t trashed_msgs[2];
+	int state[2], lfd, ret, existing, replayed, any_expiring;
+	uint ref_count, nsrecs, opts[2];
+	uint new_pending[2], flags_pending[2], trash_pending[2];
+	uint maxuid[2];     // highest UID that was already propagated
+	uint oldmaxuid[2];  // highest UID that was already propagated before this run
+	uint newmaxuid[2];  // highest UID that is currently being propagated
+	uint uidval[2];     // UID validity value
+	uint newuidval[2];  // UID validity obtained from driver
+	uint finduid[2];    // TUID lookup makes sense only for UIDs >= this
+	uint maxxfuid;      // highest expired UID on full side
+	uchar good_flags[2], bad_flags[2], can_crlf[2];
+} sync_vars_t;
+
+int prepare_state( sync_vars_t *svars );
+int lock_state( sync_vars_t *svars );
+int load_state( sync_vars_t *svars );
+void save_state( sync_vars_t *svars );
+void delete_state( sync_vars_t *svars );
+
+void ATTR_PRINTFLIKE(2, 3) jFprintf( sync_vars_t *svars, const char *msg, ... );
+
+#define JLOG_(pre_commit, log_fmt, log_args, dbg_fmt, ...) \
+	do { \
+		if (pre_commit && !(DFlags & FORCEJOURNAL)) { \
+			debug( "-> (log: " log_fmt ") (" dbg_fmt ")\n", __VA_ARGS__ ); \
+		} else { \
+			debug( "-> log: " log_fmt " (" dbg_fmt ")\n", __VA_ARGS__ ); \
+			jFprintf( svars, log_fmt "\n", deparen(log_args) ); \
+		} \
+	} while (0)
+#define JLOG3(pre_commit, log_fmt, log_args, dbg_fmt) \
+	JLOG_(pre_commit, log_fmt, log_args, dbg_fmt, deparen(log_args))
+#define JLOG4(pre_commit, log_fmt, log_args, dbg_fmt, dbg_args) \
+	JLOG_(pre_commit, log_fmt, log_args, dbg_fmt, deparen(log_args), deparen(dbg_args))
+#define JLOG_SEL(_1, _2, _3, _4, x, ...) x
+#define JLOG(...) JLOG_SEL(__VA_ARGS__, JLOG4, JLOG3, NO_JLOG2, NO_JLOG1)(0, __VA_ARGS__)
+#define PC_JLOG(...) JLOG_SEL(__VA_ARGS__, JLOG4, JLOG3, NO_JLOG2, NO_JLOG1)(1, __VA_ARGS__)
+
+void assign_uid( sync_vars_t *svars, sync_rec_t *srec, int t, uint uid );
+
+#define ASSIGN_UID(srec, t, nuid, ...) \
+	do { \
+		JLOG( "%c %u %u %u", ("<>"[t], srec->uid[F], srec->uid[N], nuid), __VA_ARGS__ ); \
+		assign_uid( svars, srec, t, nuid ); \
+	} while (0)
+
+void assign_tuid( sync_vars_t *svars, sync_rec_t *srec );
+int match_tuids( sync_vars_t *svars, int t, message_t *msgs );
+
+sync_rec_t *upgrade_srec( sync_vars_t *svars, sync_rec_t *srec, int t );
+
+typedef struct copy_vars {
+	void (*cb)( int sts, uint uid, struct copy_vars *vars );
+	void *aux;
+	sync_rec_t *srec; /* also ->tuid */
+	message_t *msg;
+	msg_data_t data;
+	int minimal;
+} copy_vars_t;
+
+char *copy_msg_convert( int in_cr, int out_cr, copy_vars_t *vars );

src/sync_state.c

@@ -0,0 +1,643 @@
+// SPDX-FileCopyrightText: 2004-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
+//
+// mbsync - mailbox synchronizer
+//
+
+#define DEBUG_FLAG DEBUG_SYNC
+
+#include "sync_p.h"
+
+#include <fcntl.h>
+#include <ctype.h>
+#include <errno.h>
+#include <sys/stat.h>
+
+#define JOURNAL_VERSION "5"
+
+const char *str_fn[] = { "far side", "near side" }, *str_hl[] = { "push", "pull" };
+
+BIT_FORMATTER_FUNCTION(sts, S)
+
+static char *
+clean_strdup( const char *s )
+{
+	char *cs = nfstrdup( s );
+	for (uint i = 0; cs[i]; i++)
+		if (cs[i] == '/')
+			cs[i] = '!';
+	return cs;
+}
+
+int
+prepare_state( sync_vars_t *svars )
+{
+	channel_conf_t *chan = svars->chan;
+	if (!strcmp( chan->sync_state ? chan->sync_state : global_conf.sync_state, "*" )) {
+		const char *path = svars->drv[N]->get_box_path( svars->ctx[N] );
+		if (!path) {
+			error( "Error: store '%s' does not support in-box sync state\n", chan->stores[N]->name );
+			return 0;
+		}
+		nfasprintf( &svars->dname, "%s/." EXE "state", path );
+	} else {
+		char *cnname = clean_strdup( svars->box_name[N] );
+		if (chan->sync_state) {
+			nfasprintf( &svars->dname, "%s%s", chan->sync_state, cnname );
+		} else {
+			char c = FieldDelimiter;
+			char *cfname = clean_strdup( svars->box_name[F] );
+			nfasprintf( &svars->dname, "%s%c%s%c%s_%c%s%c%s", global_conf.sync_state,
+			            c, chan->stores[F]->name, c, cfname, c, chan->stores[N]->name, c, cnname );
+			free( cfname );
+		}
+		free( cnname );
+		char *s;
+		if (!(s = strrchr( svars->dname, '/' ))) {
+			error( "Error: invalid SyncState location '%s'\n", svars->dname );
+			return 0;
+		}
+		// Note that this may be shorter than the configuration value,
+		// as that may contain a filename prefix.
+		*s = 0;
+		if (mkdir_p( svars->dname, s - svars->dname )) {
+			sys_error( "Error: cannot create SyncState directory '%s'", svars->dname );
+			return 0;
+		}
+		*s = '/';
+	}
+	nfasprintf( &svars->jname, "%s.journal", svars->dname );
+	nfasprintf( &svars->nname, "%s.new", svars->dname );
+	nfasprintf( &svars->lname, "%s.lock", svars->dname );
+	return 1;
+}
+
+int
+lock_state( sync_vars_t *svars )
+{
+	struct flock lck;
+
+	if (DFlags & DRYRUN)
+		return 1;
+
+	if (svars->lfd >= 0)
+		return 1;
+	memset( &lck, 0, sizeof(lck) );
+#if SEEK_SET != 0
+	lck.l_whence = SEEK_SET;
+#endif
+#if F_WRLCK != 0
+	lck.l_type = F_WRLCK;
+#endif
+	if ((svars->lfd = open( svars->lname, O_WRONLY | O_CREAT, 0666 )) < 0) {
+		sys_error( "Error: cannot create lock file %s", svars->lname );
+		return 0;
+	}
+	if (fcntl( svars->lfd, F_SETLK, &lck )) {
+		error( "Error: channel :%s:%s-:%s:%s is locked\n",
+		       svars->chan->stores[F]->name, svars->orig_name[F], svars->chan->stores[N]->name, svars->orig_name[N] );
+		close( svars->lfd );
+		svars->lfd = -1;
+		return 0;
+	}
+	return 1;
+}
+
+static uchar
+parse_flags( const char *buf )
+{
+	uchar flags = 0;
+	for (uint i = 0, d = 0; i < as(MsgFlags); i++) {
+		if (buf[d] == MsgFlags[i]) {
+			flags |= (1 << i);
+			d++;
+		}
+	}
+	return flags;
+}
+
+int
+load_state( sync_vars_t *svars )
+{
+	sync_rec_t *srec, *nsrec;
+	FILE *jfp;
+	uint ll;
+	uint maxxnuid = 0;
+	char fbuf[16];  // enlarge when support for keywords is added
+	char buf[128], buf1[64], buf2[64];
+
+	int xt = svars->chan->expire_side;
+	if ((jfp = fopen( svars->dname, "r" ))) {
+		if (!lock_state( svars ))
+			goto jbail;
+		debug( "reading sync state %s ...\n", svars->dname );
+		int line = 0;
+		while (fgets( buf, sizeof(buf), jfp )) {
+			line++;
+			if (!(ll = strlen( buf )) || buf[ll - 1] != '\n') {
+				error( "Error: incomplete sync state header entry at %s:%d\n", svars->dname, line );
+			  jbail:
+				fclose( jfp );
+				return 0;
+			}
+			if (ll == 1)
+				goto gothdr;
+			if (line == 1 && isdigit( buf[0] )) {  // Pre-1.1 legacy
+				if (sscanf( buf, "%63s %63s", buf1, buf2 ) != 2 ||
+				    sscanf( buf1, "%u:%u", &svars->uidval[F], &svars->maxuid[F] ) < 2 ||
+				    sscanf( buf2, "%u:%u:%u", &svars->uidval[N], &maxxnuid, &svars->maxuid[N] ) < 3) {
+					error( "Error: invalid sync state header in %s\n", svars->dname );
+					goto jbail;
+				}
+				if (maxxnuid && xt != N)
+					goto sidefail;
+				goto gothdr;
+			}
+			uint uid;
+			if (sscanf( buf, "%63s %u", buf1, &uid ) != 2) {
+				error( "Error: malformed sync state header entry at %s:%d\n", svars->dname, line );
+				goto jbail;
+			}
+			if (!strcmp( buf1, "FarUidValidity" ) || !strcmp( buf1, "MasterUidValidity" ) /* Pre-1.4 legacy */) {
+				svars->uidval[F] = uid;
+			} else if (!strcmp( buf1, "NearUidValidity" ) || !strcmp( buf1, "SlaveUidValidity" ) /* Pre-1.4 legacy */) {
+				svars->uidval[N] = uid;
+			} else if (!strcmp( buf1, "MaxPulledUid" )) {
+				svars->maxuid[F] = uid;
+			} else if (!strcmp( buf1, "MaxPushedUid" )) {
+				svars->maxuid[N] = uid;
+			} else if (!strcmp( buf1, "MaxExpiredFarUid" ) || !strcmp( buf1, "MaxExpiredMasterUid" ) /* Pre-1.4 legacy */) {
+				if (xt != N) {
+				  sidefail:
+					error( "Error: state file %s does not match ExpireSide setting\n", svars->dname );
+					goto jbail;
+				}
+				svars->maxxfuid = uid;
+			} else if (!strcmp( buf1, "MaxExpiredNearUid" )) {
+				if (xt != F)
+					goto sidefail;
+				svars->maxxfuid = uid;
+			} else if (!strcmp( buf1, "MaxExpiredSlaveUid" )) {  // Pre-1.3 legacy
+				if (xt != N)
+					goto sidefail;
+				maxxnuid = uid;
+			} else {
+				error( "Error: unrecognized sync state header entry at %s:%d\n", svars->dname, line );
+				goto jbail;
+			}
+		}
+		error( "Error: unterminated sync state header in %s\n", svars->dname );
+		goto jbail;
+	  gothdr:
+		debug( "  uid val %u/%u, max uid %u/%u, max expired %u\n",
+		       svars->uidval[F], svars->uidval[N], svars->maxuid[F], svars->maxuid[N], svars->maxxfuid );
+		while (fgets( buf, sizeof(buf), jfp )) {
+			line++;
+			if (!(ll = strlen( buf )) || buf[--ll] != '\n') {
+				error( "Error: incomplete sync state entry at %s:%d\n", svars->dname, line );
+				goto jbail;
+			}
+			buf[ll] = 0;
+			fbuf[0] = 0;
+			uint t1, t2;
+			if (sscanf( buf, "%u %u %15s", &t1, &t2, fbuf ) < 2) {
+				error( "Error: invalid sync state entry at %s:%d\n", svars->dname, line );
+				goto jbail;
+			}
+			srec = nfzalloc( sizeof(*srec) );
+			srec->uid[F] = t1;
+			srec->uid[N] = t2;
+			char *s = fbuf;
+			if (*s == '<') {
+				s++;
+				srec->status = S_DUMMY(F);
+			} else if (*s == '>') {
+				s++;
+				srec->status = S_DUMMY(N);
+			}
+			if (*s == '^') {  // Pre-1.4 legacy
+				s++;
+				srec->status = S_SKIPPED;
+			} else if (*s == '~' || *s == 'X' /* Pre-1.3 legacy */) {
+				s++;
+				srec->status = S_EXPIRE | S_EXPIRED;
+			} else if (srec->uid[F] == (uint)-1) {  // Pre-1.3 legacy
+				srec->uid[F] = 0;
+				srec->status = S_SKIPPED;
+			} else if (srec->uid[N] == (uint)-1) {
+				srec->uid[N] = 0;
+				srec->status = S_SKIPPED;
+			}
+			srec->flags = parse_flags( s );
+			debug( "  entry (%u,%u,%s,%s)\n", srec->uid[F], srec->uid[N],
+			       fmt_flags( srec->flags ).str, fmt_sts( srec->status ).str );
+			*svars->srecadd = srec;
+			svars->srecadd = &srec->next;
+			svars->nsrecs++;
+		}
+		fclose( jfp );
+		svars->existing = 1;
+	} else {
+		if (errno != ENOENT) {
+			sys_error( "Error: cannot read sync state %s", svars->dname );
+			return 0;
+		}
+		svars->existing = 0;
+	}
+
+	// This is legacy support for pre-1.3 sync states.
+	if (maxxnuid) {
+		uint minwuid = UINT_MAX;
+		for (srec = svars->srecs; srec; srec = srec->next) {
+			if ((srec->status & (S_DEAD | S_SKIPPED | S_PENDING)) || !srec->uid[F])
+				continue;
+			if (srec->status & S_EXPIRED) {
+				if (!srec->uid[N]) {
+					// The expired message was already gone.
+					continue;
+				}
+				// The expired message was not expunged yet, so re-examine it.
+				// This will happen en masse, so just extend the bulk fetch.
+			} else {
+				if (srec->uid[N] && maxxnuid >= srec->uid[N]) {
+					// The non-expired message is in the generally expired range,
+					// so don't make it contribute to the bulk fetch.
+					continue;
+				}
+				// Usual non-expired message.
+			}
+			if (minwuid > srec->uid[F])
+				minwuid = srec->uid[F];
+		}
+		svars->maxxfuid = minwuid - 1;
+	}
+
+	svars->newmaxuid[F] = svars->maxuid[F];
+	svars->newmaxuid[N] = svars->maxuid[N];
+	int line = 0;
+	if ((jfp = fopen( svars->jname, "r" ))) {
+		if (!lock_state( svars ))
+			goto jbail;
+		struct stat st;
+		if (!stat( svars->nname, &st ) && fgets( buf, sizeof(buf), jfp )) {
+			debug( "recovering journal ...\n" );
+			if (!(ll = strlen( buf )) || buf[--ll] != '\n') {
+				error( "Error: incomplete journal header in %s\n", svars->jname );
+				goto jbail;
+			}
+			buf[ll] = 0;
+			if (!equals( buf, (int)ll, JOURNAL_VERSION, strlen(JOURNAL_VERSION) )) {
+				error( "Error: incompatible journal version"
+				       " (got %s, expected " JOURNAL_VERSION ")\n", buf );
+				goto jbail;
+			}
+			srec = NULL;
+			line = 1;
+			while (fgets( buf, sizeof(buf), jfp )) {
+				line++;
+				if (!(ll = strlen( buf )) || buf[--ll] != '\n') {
+					error( "Error: incomplete journal entry at %s:%d\n", svars->jname, line );
+					goto jbail;
+				}
+				buf[ll] = 0;
+				char c;
+				int tn, bad;
+				uint t1, t2, t3, t4;
+				switch ((c = buf[0])) {
+				case '#':
+					tn = 0;
+					bad = (sscanf( buf + 2, "%u %u %n", &t1, &t2, &tn ) < 2) || !tn || (ll - (uint)tn != TUIDL + 2);
+					break;
+				case 'N':
+				case 'F':
+				case 'T':
+				case 'P':
+				case '+':
+				case '&':
+				case '-':
+				case '_':
+				case '|':
+					bad = sscanf( buf + 2, "%u %u", &t1, &t2 ) != 2;
+					break;
+				case '<':
+				case '>':
+				case '*':
+				case '%':
+				case '~':
+				case '^':
+					bad = sscanf( buf + 2, "%u %u %u", &t1, &t2, &t3 ) != 3;
+					break;
+				case '$':
+					bad = sscanf( buf + 2, "%u %u %u %u", &t1, &t2, &t3, &t4 ) != 4;
+					break;
+				default:
+					error( "Error: unrecognized journal entry at %s:%d\n", svars->jname, line );
+					goto jbail;
+				}
+				if (bad) {
+					error( "Error: malformed journal entry at %s:%d\n", svars->jname, line );
+					goto jbail;
+				}
+				if (c == 'N') {
+					svars->maxuid[t1] = svars->newmaxuid[t1] = t2;
+					debug( "  maxuid of %s now %u\n", str_fn[t1], t2 );
+				} else if (c == 'F') {
+					svars->finduid[t1] = t2;
+					debug( "  saved UIDNEXT of %s now %u\n", str_fn[t1], t2 );
+				} else if (c == 'T') {
+					*uint_array_append( &svars->trashed_msgs[t1] ) = t2;
+					debug( "  trashed %u from %s\n", t2, str_fn[t1] );
+				} else if (c == '|') {
+					svars->uidval[F] = t1;
+					svars->uidval[N] = t2;
+					debug( "  UIDVALIDITYs now %u/%u\n", t1, t2 );
+				} else if (c == '+') {
+					srec = nfzalloc( sizeof(*srec) );
+					srec->uid[F] = t1;
+					srec->uid[N] = t2;
+					if (svars->newmaxuid[F] < t1)
+						svars->newmaxuid[F] = t1;
+					if (svars->newmaxuid[N] < t2)
+						svars->newmaxuid[N] = t2;
+					debug( "  new entry(%u,%u)\n", t1, t2 );
+					srec->status = S_PENDING;
+					*svars->srecadd = srec;
+					svars->srecadd = &srec->next;
+					svars->nsrecs++;
+				} else {
+					for (nsrec = srec; srec; srec = srec->next)
+						if (srec->uid[F] == t1 && srec->uid[N] == t2)
+							goto syncfnd;
+					for (srec = svars->srecs; srec != nsrec; srec = srec->next)
+						if (srec->uid[F] == t1 && srec->uid[N] == t2)
+							goto syncfnd;
+					error( "Error: journal entry at %s:%d refers to non-existing sync state entry\n", svars->jname, line );
+					goto jbail;
+				  syncfnd:
+					debugn( "  entry(%u,%u) ", srec->uid[F], srec->uid[N] );
+					switch (c) {
+					case '-':
+						debug( "killed\n" );
+						srec->status = S_DEAD;
+						break;
+					case '#':
+						memcpy( srec->tuid, buf + tn + 2, TUIDL );
+						debug( "TUID now %." stringify(TUIDL) "s\n", srec->tuid );
+						break;
+					case '&':
+						debug( "TUID %." stringify(TUIDL) "s lost\n", srec->tuid );
+						srec->tuid[0] = 0;
+						break;
+					case '<':
+						debug( "far side now %u\n", t3 );
+						assign_uid( svars, srec, F, t3 );
+						break;
+					case '>':
+						debug( "near side now %u\n", t3 );
+						assign_uid( svars, srec, N, t3 );
+						break;
+					case '*':
+						srec->flags = (uchar)t3;
+						debug( "flags now %s\n", fmt_lone_flags( t3 ).str );
+						break;
+					case 'P':
+						debug( "deleted dummy\n" );
+						srec->aflags[F] = srec->aflags[N] = 0;  // Clear F_DELETED
+						srec->status = (srec->status & ~S_PURGE) | S_PURGED;
+						break;
+					case '%':
+						srec->pflags = (uchar)t3;
+						debug( "pending flags now %s\n", fmt_lone_flags( t3 ).str );
+						break;
+					case '~':
+						srec->status = (srec->status & ~S_LOGGED) | t3;
+						if ((srec->status & S_EXPIRED) && svars->maxxfuid < srec->uid[xt^1])
+							svars->maxxfuid = srec->uid[xt^1];
+						debug( "status now %s\n", fmt_sts( srec->status ).str );
+						break;
+					case '_':
+						debug( "has placeholder now\n" );
+						srec->status = S_PENDING | (!srec->uid[F] ? S_DUMMY(F) : S_DUMMY(N));
+						break;
+					case '^':
+						tn = (srec->status & S_DUMMY(F)) ? F : N;
+						srec->pflags = (uchar)t3;
+						debug( "upgrading %s placeholder, dummy's flags %s\n",
+						       str_fn[tn], fmt_lone_flags( t3 ).str );
+						srec = upgrade_srec( svars, srec, tn );
+						break;
+					case '$':
+						tn = !srec->uid[F] ? F : N;
+						srec->aflags[tn] = (uchar)t3;
+						srec->dflags[tn] = (uchar)t4;
+						debug( "flag update for %s now +%s -%s\n",
+						       str_fn[tn], fmt_flags( t3 ).str, fmt_flags( t4 ).str );
+						break;
+					default:
+						assert( !"Unhandled journal entry" );
+					}
+				}
+			}
+		}
+		fclose( jfp );
+		sort_uint_array( svars->trashed_msgs[F].array );
+		sort_uint_array( svars->trashed_msgs[N].array );
+	} else {
+		if (errno != ENOENT) {
+			sys_error( "Error: cannot read journal %s", svars->jname );
+			return 0;
+		}
+	}
+	svars->replayed = line;
+
+	return 1;
+}
+
+static void
+create_state( sync_vars_t *svars )
+{
+	if (!(svars->nfp = fopen( svars->nname, "w" ))) {
+		sys_error( "Error: cannot create new sync state %s", svars->nname );
+		exit( 1 );
+	}
+}
+
+void
+jFprintf( sync_vars_t *svars, const char *msg, ... )
+{
+	va_list va;
+
+	if (!svars->jfp) {
+		if (DFlags & DRYRUN)
+			goto dryout;
+		create_state( svars );
+		if (!(svars->jfp = fopen( svars->jname, svars->replayed ? "a" : "w" ))) {
+			sys_error( "Error: cannot create journal %s", svars->jname );
+			exit( 1 );
+		}
+		setlinebuf( svars->jfp );
+		if (!svars->replayed)
+			Fprintf( svars->jfp, JOURNAL_VERSION "\n" );
+	}
+	va_start( va, msg );
+	vFprintf( svars->jfp, msg, va );
+	va_end( va );
+  dryout:
+	countStep();
+	JCount++;
+}
+
+void
+save_state( sync_vars_t *svars )
+{
+	// If no change was made, the state is also unmodified.
+	if (!svars->jfp && !svars->replayed)
+		return;
+
+	// jfp is NULL in this case anyway, but we might have replayed.
+	if (DFlags & DRYRUN)
+		return;
+
+	if (!svars->nfp)
+		create_state( svars );
+	Fprintf( svars->nfp,
+	         "FarUidValidity %u\nNearUidValidity %u\nMaxPulledUid %u\nMaxPushedUid %u\n",
+	         svars->uidval[F], svars->uidval[N], svars->maxuid[F], svars->maxuid[N] );
+	if (svars->maxxfuid)
+		Fprintf( svars->nfp,
+		         svars->chan->expire_side == N ? "MaxExpiredFarUid %u\n" : "MaxExpiredNearUid %u\n",
+		         svars->maxxfuid );
+	Fprintf( svars->nfp, "\n" );
+	for (sync_rec_t *srec = svars->srecs; srec; srec = srec->next) {
+		if (srec->status & S_DEAD)
+			continue;
+		Fprintf( svars->nfp, "%u %u %s%s%s\n", srec->uid[F], srec->uid[N],
+		         (srec->status & S_DUMMY(F)) ? "<" : (srec->status & S_DUMMY(N)) ? ">" : "",
+		         (srec->status & S_SKIPPED) ? "^" : (srec->status & S_EXPIRED) ? "~" : "",
+		         fmt_flags( srec->flags ).str );
+	}
+
+	Fclose( svars->nfp, 1 );
+	if (svars->jfp)
+		Fclose( svars->jfp, 0 );
+	if (!(DFlags & KEEPJOURNAL)) {
+		// Order is important!
+		if (rename( svars->nname, svars->dname ))
+			warn( "Warning: cannot commit sync state %s\n", svars->dname );
+		else if (unlink( svars->jname ))
+			warn( "Warning: cannot delete journal %s\n", svars->jname );
+	}
+}
+
+void
+delete_state( sync_vars_t *svars )
+{
+	if (DFlags & DRYRUN)
+		return;
+
+	unlink( svars->nname );
+	unlink( svars->jname );
+	if (unlink( svars->dname ) || unlink( svars->lname )) {
+		sys_error( "Error: channel %s: sync state cannot be deleted", svars->chan->name );
+		svars->ret = SYNC_FAIL;
+	}
+}
+
+
+void
+assign_uid( sync_vars_t *svars, sync_rec_t *srec, int t, uint uid )
+{
+	srec->uid[t] = uid;
+	if (uid == svars->newmaxuid[t] + 1)
+		svars->newmaxuid[t] = uid;
+	if (uid) {
+		if (srec->status & S_UPGRADE) {
+			srec->flags = (srec->flags | srec->aflags[t]) & ~srec->dflags[t];
+			srec->aflags[t] = srec->dflags[t] = 0;  // Cleanup after journal replay
+		} else {
+			srec->flags = srec->pflags;
+		}
+	}
+	srec->status &= ~(S_PENDING | S_UPGRADE);
+	srec->tuid[0] = 0;
+}
+
+void
+assign_tuid( sync_vars_t *svars, sync_rec_t *srec )
+{
+	for (uint i = 0; i < TUIDL; i++) {
+		uchar c = arc4_getbyte() & 0x3f;
+		srec->tuid[i] = (char)(c < 26 ? c + 'A' : c < 52 ? c + 'a' - 26 :
+		                       c < 62 ? c + '0' - 52 : c == 62 ? '+' : '/');
+	}
+	JLOG( "# %u %u %." stringify(TUIDL) "s", (srec->uid[F], srec->uid[N], srec->tuid), "new TUID" );
+}
+
+int
+match_tuids( sync_vars_t *svars, int t, message_t *msgs )
+{
+	message_t *tmsg, *ntmsg = NULL;
+	const char *diag;
+	int num_lost = 0;
+
+	for (sync_rec_t *srec = svars->srecs; srec; srec = srec->next) {
+		if (srec->status & S_DEAD)
+			continue;
+		if (!srec->uid[t] && srec->tuid[0]) {
+			debug( "pair(%u,%u) TUID %." stringify(TUIDL) "s\n", srec->uid[F], srec->uid[N], srec->tuid );
+			for (tmsg = ntmsg; tmsg; tmsg = tmsg->next) {
+				if (tmsg->status & M_DEAD)
+					continue;
+				if (tmsg->tuid[0] && !memcmp( tmsg->tuid, srec->tuid, TUIDL )) {
+					diag = (tmsg == ntmsg) ? "adjacently" : "after gap";
+					goto mfound;
+				}
+			}
+			for (tmsg = msgs; tmsg != ntmsg; tmsg = tmsg->next) {
+				if (tmsg->status & M_DEAD)
+					continue;
+				if (tmsg->tuid[0] && !memcmp( tmsg->tuid, srec->tuid, TUIDL )) {
+					diag = "after reset";
+					goto mfound;
+				}
+			}
+			JLOG( "& %u %u", (srec->uid[F], srec->uid[N]), "TUID lost" );
+			// Note: status remains S_PENDING.
+			srec->tuid[0] = 0;
+			num_lost++;
+			continue;
+		  mfound:
+			tmsg->srec = srec;
+			srec->msg[t] = tmsg;
+			ntmsg = tmsg->next;
+			ASSIGN_UID( srec, t, tmsg->uid, "TUID matched %s", diag );
+		}
+	}
+	return num_lost;
+}
+
+sync_rec_t *
+upgrade_srec( sync_vars_t *svars, sync_rec_t *srec, int t )
+{
+	// Create an entry and append it to the current one.
+	sync_rec_t *nsrec = nfzalloc( sizeof(*nsrec) );
+	nsrec->next = srec->next;
+	srec->next = nsrec;
+	if (svars->srecadd == &srec->next)
+		svars->srecadd = &nsrec->next;
+	svars->nsrecs++;
+	// Move the placeholder to the new entry.
+	nsrec->uid[t] = srec->uid[t];
+	srec->uid[t] = 0;
+	if (srec->msg[t]) {  // NULL during journal replay; is assigned later.
+		nsrec->msg[t] = srec->msg[t];
+		nsrec->msg[t]->srec = nsrec;
+		srec->msg[t] = NULL;
+	}
+	// Mark the original entry for upgrade.
+	srec->status = (srec->status & ~(S_DUMMY(F) | S_DUMMY(N))) | S_PENDING | S_UPGRADE;
+	// Mark the placeholder for nuking.
+	nsrec->status = S_PURGE | (srec->status & (S_DEL(F) | S_DEL(N)));
+	nsrec->aflags[t] = F_DELETED;
+	return nsrec;
+}

src/tst_imap_msgs.c

@@ -0,0 +1,166 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// isync test suite
+//
+
+#include "imap_p.h"
+
+static imap_messages_t smsgs;
+
+// from driver.c
+void
+free_generic_messages( message_t *msgs )
+{
+	message_t *tmsg;
+
+	for (; msgs; msgs = tmsg) {
+		tmsg = msgs->next;
+		// free( msgs->msgid );
+		free( msgs );
+	}
+}
+
+static void
+dump_messages( void )
+{
+	print( "=>" );
+	uint seq = 0;
+	for (imap_message_t *msg = smsgs.head; msg; msg = msg->next) {
+		seq += msg->seq;
+		if (msg->status & M_DEAD)
+			print( " (%u:%u)", seq, msg->uid );
+		else
+			print( " %u:%u", seq, msg->uid );
+	}
+	print( "\n" );
+}
+
+static void
+init( uint *in )
+{
+	reset_imap_messages( &smsgs );
+	for (; *in; in++) {
+		imap_message_t *msg = imap_new_msg( &smsgs );
+		msg->seq = *in;
+		// We (ab)use the initial sequence number as the UID. That's not
+		// exactly realistic, but it's valid, and saves us redundant data.
+		msg->uid = *in;
+	}
+}
+
+static void
+modify( uint *in )
+{
+	for (; *in; in++) {
+		imap_expunge_msg( &smsgs, *in );
+#ifdef DEBUG_IMAP_MSGS
+		dump_messages();
+#endif
+	}
+}
+
+static void
+verify( uint *in, const char *name )
+{
+	int fails = 0;
+	imap_message_t *msg = smsgs.head;
+	for (;;) {
+		if (msg && *in && msg->uid == *in) {
+			if (msg->status & M_DEAD) {
+				printf( "*** %s: message %u is dead\n", name, msg->uid );
+				fails++;
+			} else {
+				assert( msg->seq );
+			}
+			msg = msg->next;
+			in++;
+		} else if (*in && (!msg || msg->uid > *in)) {
+			printf( "*** %s: message %u is missing\n", name, *in );
+			fails++;
+			in++;
+		} else if (msg) {
+			if (!(msg->status & M_DEAD)) {
+				printf( "*** %s: excess message %u\n", name, msg->uid );
+				fails++;
+			}
+			msg = msg->next;
+		} else {
+			assert( !*in );
+			break;
+		}
+	}
+	if (fails)
+		dump_messages();
+}
+
+static void
+test( uint *ex, uint *out, const char *name )
+{
+	printf( "test %s ...\n", name );
+	modify( ex );
+	verify( out, name );
+}
+
+int
+main( void )
+{
+	static uint arr_0[] = { 0 };
+	static uint arr_1[] = { 1, 0 };
+
+	static uint full_in[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0 };
+	init( full_in );
+#if 0
+	static uint nop[] = { 0 };
+	static uint nop_out[] = { /* 1, */ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, /* 17, */ 18 /*!*/, 0 };
+	test( nop, nop_out, "self-test" );
+#endif
+	static uint full_ex_fw1[] = { 18, 13, 13, 13, 1, 1, 1, 0 };
+	static uint full_out_fw1[] = { 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 0 };
+	test( full_ex_fw1, full_out_fw1, "full, forward 1" );
+	static uint full_ex_fw2[] = { 10, 10, 0 };
+	static uint full_out_fw2[] = { 4, 5, 6, 7, 8, 9, 10, 11, 12, 0 };
+	test( full_ex_fw2, full_out_fw2, "full, forward 2" );
+
+	init( full_in );
+	static uint full_ex_bw1[] = { 18, 17, 16, 15, 14, 13, 5, 4, 3, 0 };
+	static uint full_out_bw1[] = { 1, 2, 6, 7, 8, 9, 10, 11, 12, 0 };
+	test( full_ex_bw1, full_out_bw1, "full, backward 1" );
+	static uint full_ex_bw2[] = { 2, 1, 0 };
+	static uint full_out_bw2[] = { 6, 7, 8, 9, 10, 11, 12, 0 };
+	test( full_ex_bw2, full_out_bw2, "full, backward 2" );
+
+	static uint hole_wo1_in[] = { 10, 11, 12, 20, 21, 31, 32, 33, 34, 35, 36, 37, 0 };
+	init( hole_wo1_in );
+	static uint hole_wo1_ex_1[] = { 31, 30, 29, 28, 22, 21, 11, 2, 1, 0 };
+	static uint hole_wo1_out_1[] = { 10, 12, 20, 32, 33, 34, 35, 36, 37, 0 };
+	test( hole_wo1_ex_1, hole_wo1_out_1, "hole w/o 1, backward" );
+
+	init( hole_wo1_in );
+	static uint hole_wo1_ex_2[] = { 1, 1, 9, 18, 18, 23, 23, 23, 23, 0 };
+	test( hole_wo1_ex_2, hole_wo1_out_1, "hole w/o 1, forward" );
+	test( arr_1, hole_wo1_out_1, "hole w/o 1, forward 2" );
+	static uint hole_wo1_ex_4[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 };
+	static uint hole_wo1_out_4[] = { 37, 0 };
+	test( hole_wo1_ex_4, hole_wo1_out_4, "hole w/o 1, forward 3" );
+	test( arr_1, arr_0, "hole w/o 1, forward 4" );
+	test( arr_1, arr_0, "hole w/o 1, forward 5" );
+
+	static uint hole_w1_in[] = { 1, 10, 11, 12, 0 };
+	init( hole_w1_in );
+	static uint hole_w1_ex_1[] = { 11, 10, 2, 1, 0 };
+	static uint hole_w1_out_1[] = { 12, 0 };
+	test( hole_w1_ex_1, hole_w1_out_1, "hole w/ 1, backward" );
+	test( arr_1, hole_w1_out_1, "hole w/ 1, backward 2" );
+
+	init( hole_w1_in );
+	static uint hole_w1_ex_2[] = { 1, 1, 8, 8, 0 };
+	test( hole_w1_ex_2, hole_w1_out_1, "hole w/ 1, forward" );
+	static uint hole_w1_ex_4[] = { 1, 1, 1, 1, 1, 1, 1, 0 };
+	static uint hole_w1_out_4[] = { 12, 0 };
+	test( hole_w1_ex_4, hole_w1_out_4, "hole w/ 1, forward 2" );
+	test( arr_1, arr_0, "hole w/ 1, forward 3" );
+	test( arr_1, arr_0, "hole w/ 1, forward 4" );
+
+	return 0;
+}

src/tst_imap_utf7.c

@@ -0,0 +1,116 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// isync test suite
+//
+
+#include "imap_p.h"
+
+static struct {
+	const char *utf8, *utf7;
+} data[] = {
+	{ u8"", "" },
+	{ u8"1", "1" },
+	{ u8"word", "word" },
+	{ u8"&", "&-" },
+	{ NULL, "&" },
+	{ NULL, "&-&" },
+	{ u8"&&", "&-&-" },
+	{ u8"1&1", "1&-1" },
+	{ u8"&1&", "&-1&-" },
+	{ u8"\t", "&AAk-" },
+	{ NULL, "&AAk" },
+	{ NULL, "&AA-" },
+	{ NULL, "&*Ak-" },
+	{ NULL, "&&-" },
+	{ u8"m\x7f""ll", "m&AH8-ll" },
+	{ u8"\t&", "&AAk-&-" },
+	{ u8"\t&\t", "&AAk-&-&AAk-" },
+	{ u8"&\t", "&-&AAk-" },
+	{ u8"&\t&", "&-&AAk-&-" },
+	{ u8"ä", "&AOQ-" },
+	{ u8"\x83\x84", NULL },
+	{ u8"\xc3\xc4", NULL },
+	{ u8"\xc3", NULL },
+	{ u8"äö", "&AOQA9g-" },
+	{ u8"äöü", "&AOQA9gD8-" },
+	{ u8"Ḁ", "&HgA-" },
+	{ u8"\xe1\xc8\x80", NULL },
+	{ u8"\xe1\xb8\xf0", NULL },
+	{ u8"\xe1\xb8", NULL },
+	{ u8"\xe1", NULL },
+	{ u8"Ḁḁ", "&HgAeAQ-" },
+	{ u8"😂", "&2D3eAg-" },
+	{ u8"\xf8\x9f\x98\x82", NULL },
+	{ u8"\xf0\xcf\x98\x82", NULL },
+	{ u8"\xf0\x9f\xd8\x82", NULL },
+	{ u8"\xf0\x9f\x98\xe2", NULL },
+	{ u8"\xf0\x9f\x98", NULL },
+	{ u8"\xf0\x9f", NULL },
+	{ u8"\xf0", NULL },
+	{ NULL, "&2D0-" },
+	{ u8"😈😎", "&2D3eCNg93g4-" },
+	{ u8"müll", "m&APw-ll" },
+	{ u8"mü", "m&APw-" },
+	{ u8"über", "&APw-ber" },
+};
+
+int
+main( void )
+{
+	int ret = 0;
+
+	for (uint i = 0; i < as(data); i++) {
+		if (!data[i].utf8)
+			continue;
+		xprintf( "To UTF-7 \"%s\" (\"%!s\") ...\n", data[i].utf8, data[i].utf8 );
+		char *utf7 = imap_utf8_to_utf7( data[i].utf8 );
+		if (utf7) {
+			if (!data[i].utf7) {
+				xprintf( "Unexpected success: \"%s\" (\"%!s\")\n", utf7, utf7 );
+				ret = 1;
+			} else if (strcmp( utf7, data[i].utf7 )) {
+				xprintf( "Mismatch, got \"%s\" (\"%!s\"), want \"%!s\"\n",
+				         utf7, utf7, data[i].utf7 );
+				ret = 1;
+			}
+			free( utf7 );
+		} else {
+			if (data[i].utf7) {
+				xprintf( "Conversion failure.\n" );
+				ret = 1;
+			}
+		}
+	}
+
+	for (uint i = 0; i < as(data); i++) {
+		if (!data[i].utf7)
+			continue;
+		xprintf( "From UTF-7 \"%!s\" ...\n", data[i].utf7 );
+		int utf7len = strlen( data[i].utf7 );
+		char utf8buf[1000];
+		int utf8len = imap_utf7_to_utf8( data[i].utf7, utf7len, utf8buf );
+		if (utf8len >= 0) {
+			if (!data[i].utf8) {
+				xprintf( "Unexpected success: \"%.*s\" (\"%.*!s\")\n",
+				         utf8len, utf8buf, utf8len, utf8buf );
+				ret = 1;
+			} else {
+				int wantlen = strlen( data[i].utf8 );
+				if (utf8len != wantlen || memcmp( utf8buf, data[i].utf8, utf8len )) {
+					xprintf( "Mismatch, got \"%.*s\" (\"%.*!s\"), want \"%s\" (\"%!s\")\n",
+					         utf8len, utf8buf, utf8len, utf8buf, data[i].utf8, data[i].utf8 );
+					ret = 1;
+				}
+			}
+			assert( utf8len < utf7len * 9 / 8 + 1 );
+		} else {
+			if (data[i].utf8) {
+				xprintf( "Conversion failure.\n" );
+				ret = 1;
+			}
+		}
+	}
+
+	return ret;
+}

src/tst_msg_cvt.c

@@ -0,0 +1,306 @@
+// SPDX-FileCopyrightText: 2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// isync test suite
+//
+
+#include "sync_p.h"
+
+#define TUID "one two tuid"
+
+static_assert( sizeof(TUID) - 1 == TUIDL, "TUID size mismatch" );
+
+static size_t
+strip_cr( char *buf )
+{
+	size_t i, j = 0;
+	char c, pc = 0;
+	for (i = 0; (c = buf[i]); i++) {
+		if (c == '\n' && pc == '\r')
+			j--;
+		buf[j++] = c;
+		pc = c;
+	}
+	buf[j] = 0;
+	return j;
+}
+
+#define NL_UNIX 0
+#define NL_ANY 1
+
+#define AS_IS 0
+#define ADD_TUID 1
+
+#define FULL 0
+#define MINIMAL 1
+
+#define REGULAR 0
+#define FLAGGED 1
+
+#define OK_HEADER 0
+#define PARTIAL_HEADER 1
+
+#define BIG_SIZE 2345687
+#define BIG_SIZE_STR "2.2MiB"
+
+#define SEP "============="
+
+static void
+test( const char *name, const char *in, int scr, int rscr, const char *out, int tcr, int rtcr, int add_tuid, int minimal, int flagged )
+{
+	assert( !rscr || scr );
+	assert( !rtcr || tcr );
+	assert( !minimal || add_tuid );
+	assert( !flagged || minimal );
+
+	printf( "Testing %s, %s (%s) => %s (%s)%s%s%s ...\n", name,
+	        rscr ? "CRLF" : "LF", scr ? "Any" : "Unix", rtcr ? "CRLF" : "LF", tcr ? "Any" : "Unix",
+	        add_tuid ? ", add TUID" : "", minimal ? ", minimal" : "", flagged ? ", flagged" : "" );
+
+	sync_rec_t srec;
+	message_t msg;
+	copy_vars_t vars;
+	vars.minimal = minimal;
+	if (add_tuid) {
+		vars.srec = &srec;
+		memcpy( vars.srec->tuid, TUID, TUIDL );
+		if (minimal) {
+			vars.msg = &msg;
+			vars.msg->size = BIG_SIZE;
+			vars.data.flags = flagged ? F_FLAGGED : 0;
+		}
+	} else {
+		vars.srec = 0;
+	}
+	vars.data.data = strdup( in );
+	vars.data.len = rscr ? strlen( in ) : strip_cr( vars.data.data );
+	char *orig = strdup( vars.data.data );
+	const char *err = copy_msg_convert( scr, tcr, &vars );
+	if (err) {
+		printf( "FAIL: %s!\n", err );
+		exit( 1 );
+	}
+	if (!rtcr) {
+		char *tout = strdup( out );
+		size_t toutl = strip_cr( tout );
+		if (toutl != vars.data.len || memcmp( vars.data.data, tout, toutl )) {
+			xprintf( "FAIL!\n"
+			         SEP " Input " SEP "\n%!&s\n"
+			         SEP " Expected output " SEP "\n%!&s\n"
+			         SEP " Output " SEP "\n%.*!&s\n" SEP "\n",
+			         orig, tout, vars.data.len, vars.data.data );
+			exit( 1 );
+		}
+		free( tout );
+	} else {
+		size_t outl = strlen( out );
+		if (outl != vars.data.len || memcmp( vars.data.data, out, outl )) {
+			xprintf( "FAIL!\n"
+			         SEP " Input " SEP "\n%!&s\n"
+			         SEP " Expected output (%u bytes) " SEP "\n%!&s\n"
+			         SEP " Actual output (%u bytes) " SEP "\n%.*!&s\n" SEP "\n",
+			         orig, outl, out, vars.data.len, vars.data.len, vars.data.data );
+			exit( 1 );
+		}
+	}
+	free( orig );
+	free( vars.data.data );
+}
+
+static void
+tests( const char *name, const char *in, const char *out, int add_tuid, int minimal, int flagged, int hdr_sts )
+{
+	test( name, in, NL_UNIX, NL_UNIX, out, NL_ANY, NL_ANY, add_tuid, minimal, flagged );
+	test( name, in, NL_ANY, NL_UNIX, out, NL_UNIX, NL_UNIX, add_tuid, minimal, flagged );
+	test( name, in, NL_ANY, NL_ANY, out, NL_UNIX, NL_UNIX, add_tuid, minimal, flagged );
+	// Skip if (scr == tcr && !srec), like copy_msg() does.
+	if (add_tuid) {
+		test( name, in, NL_UNIX, NL_UNIX, out, NL_UNIX, NL_UNIX, ADD_TUID, minimal, flagged );
+		if (hdr_sts == OK_HEADER) {
+			test( name, in, NL_ANY, NL_UNIX, out, NL_ANY, NL_UNIX, ADD_TUID, minimal, flagged );
+		} else {
+			// If there are no line breaks to detect the style, the output defaults to CRLF.
+			test( name, in, NL_ANY, NL_UNIX, out, NL_ANY, NL_ANY, ADD_TUID, minimal, flagged );
+		}
+		test( name, in, NL_ANY, NL_ANY, out, NL_ANY, NL_ANY, ADD_TUID, minimal, flagged );
+	}
+}
+
+static void
+fulltests( const char *name, const char *in, const char *out, int add_tuid )
+{
+	tests( name, in, out, add_tuid, FULL, REGULAR, OK_HEADER );
+}
+
+static void
+fulltests_ih( const char *name, const char *in, const char *out, int add_tuid )
+{
+	tests( name, in, out, add_tuid, FULL, REGULAR, PARTIAL_HEADER );
+}
+
+static void
+mintests( const char *name, const char *in, const char *out, int flagged )
+{
+	tests( name, in, out, ADD_TUID, MINIMAL, flagged, OK_HEADER );
+}
+
+static void
+mintests_ih( const char *name, const char *in, const char *out, int flagged )
+{
+	tests( name, in, out, ADD_TUID, MINIMAL, flagged, PARTIAL_HEADER );
+}
+
+#define FROM "From: de\rvil\r\n"
+#define R_TO "To: me"
+#define TO R_TO "\r\n"
+#define R_IN_TUID "X-TUID: garbage"
+#define IN_TUID R_IN_TUID "\r\n"
+#define OUT_TUID "X-TUID: " TUID "\r\n"
+#define R_SUBJECT "Subject: hell"
+#define SUBJECT R_SUBJECT "\r\n"
+#define PH_SUBJECT "Subject: [placeholder] hell\r\n"
+#define NO_SUBJECT "Subject: [placeholder] (No Subject)\r\n"
+#define BODY "\r\nHi,\r\n\r\n...\r\n"
+#define PH_BODY "\r\nHaving a size of 2.2MiB, this message is over the MaxSize limit.\r\n" \
+	"Flag it and sync again (Sync mode Upgrade) to fetch its real contents.\r\n"
+#define FLAGGED_PH_BODY PH_BODY "\r\nThe original message is flagged as important.\r\n"
+
+#define scc static const char
+
+int
+main( void )
+{
+	scc in_from_to[] = FROM TO BODY;
+	fulltests( "from / to", in_from_to, in_from_to, AS_IS );
+	scc out_from_to[] = FROM TO OUT_TUID BODY;
+	fulltests( "from / to", in_from_to, out_from_to, ADD_TUID );
+	scc in_from_tuid_to[] = FROM IN_TUID TO BODY;
+	scc out_from_tuid_to[] = FROM OUT_TUID TO BODY;
+	fulltests( "from / tuid / to", in_from_tuid_to, out_from_tuid_to, ADD_TUID );
+
+	scc out_from_to_ph[] = FROM TO OUT_TUID NO_SUBJECT PH_BODY;
+	mintests( "from / to", in_from_to, out_from_to_ph, REGULAR );
+	scc out_from_to_flagged_ph[] = FROM TO OUT_TUID NO_SUBJECT FLAGGED_PH_BODY;
+	mintests( "from / to", in_from_to, out_from_to_flagged_ph, FLAGGED );
+	scc out_from_tuid_to_ph[] = FROM OUT_TUID TO NO_SUBJECT PH_BODY;
+	mintests( "from / tuid / to", in_from_tuid_to, out_from_tuid_to_ph, REGULAR );
+	scc in_from_subj_to[] = FROM SUBJECT TO BODY;
+	scc out_from_subj_to[] = FROM PH_SUBJECT TO OUT_TUID PH_BODY;
+	mintests( "from / subject / to", in_from_subj_to, out_from_subj_to, REGULAR );
+	scc in_from_subj_tuid_to[] = FROM SUBJECT IN_TUID TO BODY;
+	scc out_from_subj_tuid_to[] = FROM PH_SUBJECT OUT_TUID TO PH_BODY;
+	mintests( "from / subject / tuid / to", in_from_subj_tuid_to, out_from_subj_tuid_to, REGULAR );
+	scc in_subj_from_tuid_to[] = SUBJECT FROM IN_TUID TO BODY;
+	scc out_subj_from_tuid_to[] = PH_SUBJECT FROM OUT_TUID TO PH_BODY;
+	mintests( "subject / from / tuid / to", in_subj_from_tuid_to, out_subj_from_tuid_to, REGULAR );
+	scc in_from_tuid_subj_to[] = FROM IN_TUID SUBJECT TO BODY;
+	scc out_from_tuid_subj_to[] = FROM OUT_TUID PH_SUBJECT TO PH_BODY;
+	mintests( "from / tuid / subject / to", in_from_tuid_subj_to, out_from_tuid_subj_to, REGULAR );
+	scc in_tuid_from_subj_to[] = IN_TUID FROM SUBJECT TO BODY;
+	scc out_tuid_from_subj_to[] = OUT_TUID FROM PH_SUBJECT TO PH_BODY;
+	mintests( "tuid / from / subject / to", in_tuid_from_subj_to, out_tuid_from_subj_to, REGULAR );
+
+
+	scc in_from_to_b1[] = FROM TO;
+	fulltests( "from / to w/o end", in_from_to_b1, in_from_to_b1, AS_IS );
+	scc out_from_to_b1[] = FROM TO OUT_TUID;
+	fulltests( "from / to w/o end", in_from_to_b1, out_from_to_b1, ADD_TUID );
+	scc in_from_tuid_to_b1[] = FROM IN_TUID TO;
+	scc out_from_tuid_to_b1[] = FROM OUT_TUID TO;
+	fulltests( "from / tuid / to w/o end", in_from_tuid_to_b1, out_from_tuid_to_b1, ADD_TUID );
+	scc in_from_to_tuid_b1[] = FROM TO IN_TUID;
+	scc out_from_to_tuid_b1[] = FROM TO OUT_TUID;
+	fulltests( "from / to / tuid w/o end", in_from_to_tuid_b1, out_from_to_tuid_b1, ADD_TUID );
+
+	mintests( "from / to w/o end", in_from_to_b1, out_from_to_ph, REGULAR );
+	mintests( "from / tuid / to w/o end", in_from_tuid_to_b1, out_from_tuid_to_ph, REGULAR );
+	scc in_from_subj_to_b1[] = FROM SUBJECT TO;
+	mintests( "from / subject / to w/o end", in_from_subj_to_b1, out_from_subj_to, REGULAR );
+	scc in_from_subj_tuid_to_b1[] = FROM SUBJECT IN_TUID TO;
+	mintests( "from / subject / tuid / to w/o end", in_from_subj_tuid_to_b1, out_from_subj_tuid_to, REGULAR );
+	scc in_from_subj_to_tuid_b1[] = FROM SUBJECT TO IN_TUID;
+	scc out_from_subj_to_tuid_b1[] = FROM PH_SUBJECT TO OUT_TUID PH_BODY;
+	mintests( "from / subject / to / tuid w/o end", in_from_subj_to_tuid_b1, out_from_subj_to_tuid_b1, REGULAR );
+	scc in_from_tuid_subj_to_b1[] = FROM IN_TUID SUBJECT TO;
+	mintests( "from / tuid / subject / to w/o end", in_from_tuid_subj_to_b1, out_from_tuid_subj_to, REGULAR );
+	scc in_from_tuid_to_subj_b1[] = FROM IN_TUID TO SUBJECT;
+	scc out_from_tuid_to_subj_b1[] = FROM OUT_TUID TO PH_SUBJECT PH_BODY;
+	mintests( "from / tuid / to / subject w/o end", in_from_tuid_to_subj_b1, out_from_tuid_to_subj_b1, REGULAR );
+
+
+	scc in_from_to_b2[] = FROM R_TO "\r";
+	fulltests( "from / to w/o lf", in_from_to_b2, in_from_to_b2, AS_IS );
+	scc out_from_to_b2[] = FROM TO OUT_TUID "\r";
+	fulltests( "from / to w/o lf", in_from_to_b2, out_from_to_b2, ADD_TUID );
+	scc in_from_tuid_to_b2[] = FROM IN_TUID R_TO "\r";
+	scc out_from_tuid_to_b2[] = FROM OUT_TUID R_TO "\r";
+	fulltests( "from / tuid / to w/o lf", in_from_tuid_to_b2, out_from_tuid_to_b2, ADD_TUID );
+	scc in_from_to_tuid_b2[] = FROM TO R_IN_TUID "\r";
+	fulltests( "from / to / tuid w/o lf", in_from_to_tuid_b2, out_from_to_tuid_b1, ADD_TUID );
+
+	mintests( "from / to w/o lf", in_from_to_b2, out_from_to_ph, REGULAR );
+	mintests( "from / tuid / to w/o lf", in_from_tuid_to_b2, out_from_tuid_to_ph, REGULAR );
+	scc in_from_subj_to_b2[] = FROM SUBJECT R_TO "\r";
+	mintests( "from / subject / to w/o lf", in_from_subj_to_b2, out_from_subj_to, REGULAR );
+	scc in_from_subj_tuid_to_b2[] = FROM SUBJECT IN_TUID R_TO "\r";
+	mintests( "from / subject / tuid / to w/o lf", in_from_subj_tuid_to_b2, out_from_subj_tuid_to, REGULAR );
+	scc in_from_subj_to_tuid_b2[] = FROM SUBJECT TO R_IN_TUID "\r";
+	mintests( "from / subject / to / tuid w/o lf", in_from_subj_to_tuid_b2, out_from_subj_to_tuid_b1, REGULAR );
+	scc in_from_tuid_subj_to_b2[] = FROM IN_TUID SUBJECT R_TO "\r";
+	mintests( "from / tuid / subject / to w/o lf", in_from_tuid_subj_to_b2, out_from_tuid_subj_to, REGULAR );
+	scc in_from_tuid_to_subj_b2[] = FROM IN_TUID TO R_SUBJECT "\r";
+	mintests( "from / tuid / to / subject w/o lf", in_from_tuid_to_subj_b2, out_from_tuid_to_subj_b1, REGULAR );
+
+
+	scc in_from_to_b3[] = FROM R_TO;
+	fulltests( "from / to w/o crlf", in_from_to_b3, in_from_to_b3, AS_IS );
+	fulltests( "from / to w/o crlf", in_from_to_b3, out_from_to_b1, ADD_TUID );
+	scc in_from_tuid_to_b3[] = FROM IN_TUID R_TO;
+	scc out_from_tuid_to_b3[] = FROM OUT_TUID R_TO;
+	fulltests( "from / tuid / to w/o crlf", in_from_tuid_to_b3, out_from_tuid_to_b3, ADD_TUID );
+	scc in_from_to_tuid_b3[] = FROM TO R_IN_TUID;
+	fulltests( "from / to / tuid w/o crlf", in_from_to_tuid_b3, out_from_to_tuid_b1, ADD_TUID );
+
+	mintests( "from / to w/o crlf", in_from_to_b3, out_from_to_ph, REGULAR );
+	mintests( "from / tuid / to w/o crlf", in_from_tuid_to_b3, out_from_tuid_to_ph, REGULAR );
+	scc in_from_subj_to_b3[] = FROM SUBJECT R_TO;
+	mintests( "from / subject / to w/o crlf", in_from_subj_to_b3, out_from_subj_to, REGULAR );
+	scc in_from_subj_tuid_to_b3[] = FROM SUBJECT IN_TUID R_TO;
+	mintests( "from / subject / tuid / to w/o crlf", in_from_subj_tuid_to_b3, out_from_subj_tuid_to, REGULAR );
+	scc in_from_subj_to_tuid_b3[] = FROM SUBJECT TO R_IN_TUID;
+	mintests( "from / subject / to / tuid w/o crlf", in_from_subj_to_tuid_b3, out_from_subj_to_tuid_b1, REGULAR );
+	scc in_from_tuid_subj_to_b3[] = FROM IN_TUID SUBJECT R_TO;
+	mintests( "from / tuid / subject / to w/o crlf", in_from_tuid_subj_to_b3, out_from_tuid_subj_to, REGULAR );
+	scc in_from_tuid_to_subj_b3[] = FROM IN_TUID TO R_SUBJECT;
+	mintests( "from / tuid / to / subject w/o crlf", in_from_tuid_to_subj_b3, out_from_tuid_to_subj_b1, REGULAR );
+
+	scc in_to_b1[] = R_TO "\r";
+	fulltests_ih( "to w/o lf", in_to_b1, in_to_b1, AS_IS );
+	scc out_to_b1[] = TO OUT_TUID "\r";
+	fulltests_ih( "to w/o lf", in_to_b1, out_to_b1, ADD_TUID );
+	scc out_to_b1_ph[] = TO OUT_TUID NO_SUBJECT PH_BODY;
+	mintests_ih( "to w/o lf", in_to_b1, out_to_b1_ph, REGULAR );
+
+	scc in_to_b2[] = R_TO;
+	fulltests_ih( "to w/o crlf", in_to_b2, in_to_b2, AS_IS );
+	scc out_to_b2[] = TO OUT_TUID;
+	fulltests_ih( "to w/o crlf", in_to_b2, out_to_b2, ADD_TUID );
+	scc out_to_b2_ph[] = TO OUT_TUID NO_SUBJECT PH_BODY;
+	mintests_ih( "to w/o crlf", in_to_b2, out_to_b2_ph, REGULAR );
+
+	scc in_no_hdr[] = BODY;
+	fulltests( "no header", in_no_hdr, in_no_hdr, AS_IS );
+	scc out_no_hdr[] = OUT_TUID BODY;
+	fulltests( "no header", in_no_hdr, out_no_hdr, ADD_TUID );
+	scc out_no_hdr_ph[] = OUT_TUID NO_SUBJECT PH_BODY;
+	mintests( "no header", in_no_hdr, out_no_hdr_ph, REGULAR );
+
+	scc in_empty[] = "";
+	fulltests_ih( "empty", in_empty, in_empty, AS_IS );
+	scc out_empty[] = OUT_TUID;
+	fulltests_ih( "empty", in_empty, out_empty, ADD_TUID );
+	scc out_empty_ph[] = OUT_TUID NO_SUBJECT PH_BODY;
+	mintests_ih( "empty", in_empty, out_empty_ph, REGULAR );
+
+	return 0;
+}

src/tst_timers.c

@@ -1,38 +1,15 @@
-/*
- * mbsync - mailbox synchronizer
- * Copyright (C) 2014 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
- */
+// SPDX-FileCopyrightText: 2014-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// isync test suite
+//
 
 #include "common.h"
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-
-/* Just to satisfy the references in util.c */
-int DFlags;
-const char *Home;
-
 typedef struct {
 	int id;
 	int first, other, morph_at, morph_to;
-	time_t start;
+	int64_t start;
 	wakeup_t timer;
 	wakeup_t morph_timer;
 } tst_t;
@@ -41,7 +18,7 @@ static void
 timer_start( tst_t *timer, int to )
 {
 	printf( "starting timer %d, should expire after %d\n", timer->id, to );
-	time( &timer->start );
+	timer->start  = get_now();
 	conf_wakeup( &timer->timer, to );
 }
 
@@ -51,7 +28,7 @@ timed_out( void *aux )
 	tst_t *timer = (tst_t *)aux;
 
 	printf( "timer %d expired after %d, repeat %d\n",
-	        timer->id, (int)(time( 0 ) - timer->start), timer->other );
+	        timer->id, (int)(get_now() - timer->start), timer->other );
 	if (timer->other >= 0) {
 		timer_start( timer, timer->other );
 	} else {
@@ -67,7 +44,7 @@ morph_timed_out( void *aux )
 	tst_t *timer = (tst_t *)aux;
 
 	printf( "morphing timer %d after %d\n",
-	        timer->id, (int)(time( 0 ) - timer->start) );
+	        timer->id, (int)(get_now() - timer->start) );
 	timer_start( timer, timer->morph_to );
 }
 
@@ -78,6 +55,7 @@ main( int argc, char **argv )
 {
 	int i;
 
+	init_timers();
 	for (i = 1; i < argc; i++) {
 		char *val = argv[i];
 		tst_t *timer = nfmalloc( sizeof(*timer) );

src/util.c

@@ -1,38 +1,29 @@
+// SPDX-FileCopyrightText: 2000-2002 Michael R. Elkins <me@mutt.org>
+// SPDX-FileCopyrightText: 2002-2022 Oswald Buddenhagen <ossi@users.sf.net>
+// SPDX-License-Identifier: GPL-2.0-or-later WITH LicenseRef-isync-GPL-exception
 /*
  * mbsync - mailbox synchronizer
- * Copyright (C) 2000-2002 Michael R. Elkins <me@mutt.org>
- * Copyright (C) 2002-2006,2011,2012 Oswald Buddenhagen <ossi@users.sf.net>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, mbsync may be linked with the OpenSSL library,
- * despite that library's more restrictive license.
  */
 
 #include "common.h"
 
-#include <assert.h>
-#include <stddef.h>
-#include <stdlib.h>
-#include <unistd.h>
 #include <fcntl.h>
+#include <sys/stat.h>
 #include <errno.h>
-#include <string.h>
 #include <ctype.h>
 #include <pwd.h>
+#include <sys/time.h>
 
-static int need_nl;
+int Verbosity = TERSE;
+int DFlags;
+int JLimit, JCount;
+int UseFSync = 1;
+
+int Pid;
+char Hostname[256];
+const char *Home;
+
+static int need_nl, need_del;
 
 void
 flushn( void )
@@ -41,38 +32,52 @@ flushn( void )
 		putchar( '\n' );
 		fflush( stdout );
 		need_nl = 0;
+	} else if (need_del) {
+		static const char delstr[] =
+		        "                                                            "
+		        "                                                            ";
+		if (need_del > (int)sizeof(delstr) - 1)
+			need_del = (int)sizeof(delstr) - 1;
+		// We could use ^[[K instead, but we assume a dumb terminal.
+		printf( "\r%.*s\r", need_del, delstr );
+		fflush( stdout );
+		need_del = 0;
 	}
 }
 
 static void ATTR_PRINTFLIKE(1, 0)
-printn( const char *msg, va_list va )
+vprint( const char *msg, va_list va )
 {
-	if (*msg == '\v')
-		msg++;
-	else
-		flushn();
 	vprintf( msg, va );
 	fflush( stdout );
+	need_nl = 0;
 }
 
 void
-vdebug( int cat, const char *msg, va_list va )
+print( const char *msg, ... )
 {
-	if (DFlags & cat) {
-		vprintf( msg, va );
-		fflush( stdout );
-		need_nl = 0;
-	}
+	va_list va;
+
+	va_start( va, msg );
+	vprint( msg, va );
+	va_end( va );
+}
+
+static void ATTR_PRINTFLIKE(1, 0)
+vprintn( const char *msg, va_list va )
+{
+	vprint( msg, va );
+	need_nl = 1;
 }
 
 void
-vdebugn( int cat, const char *msg, va_list va )
+printn( const char *msg, ... )
 {
-	if (DFlags & cat) {
-		vprintf( msg, va );
-		fflush( stdout );
-		need_nl = 1;
-	}
+	va_list va;
+
+	va_start( va, msg );
+	vprintn( msg, va );
+	va_end( va );
 }
 
 void
@@ -81,10 +86,19 @@ progress( const char *msg, ... )
 	va_list va;
 
 	va_start( va, msg );
-	vprintf( msg, va );
+	need_del = vprintf( msg, va ) - 1;
 	va_end( va );
 	fflush( stdout );
-	need_nl = 1;
+}
+
+static void ATTR_PRINTFLIKE(1, 0)
+nvprint( const char *msg, va_list va )
+{
+	if (*msg == '\v')
+		msg++;
+	else
+		flushn();
+	vprint( msg, va );
 }
 
 void
@@ -92,11 +106,10 @@ info( const char *msg, ... )
 {
 	va_list va;
 
-	if (DFlags & VERBOSE) {
+	if (Verbosity >= VERBOSE) {
 		va_start( va, msg );
-		printn( msg, va );
+		nvprint( msg, va );
 		va_end( va );
-		need_nl = 0;
 	}
 }
 
@@ -105,9 +118,9 @@ infon( const char *msg, ... )
 {
 	va_list va;
 
-	if (DFlags & VERBOSE) {
+	if (Verbosity >= VERBOSE) {
 		va_start( va, msg );
-		printn( msg, va );
+		nvprint( msg, va );
 		va_end( va );
 		need_nl = 1;
 	}
@@ -118,11 +131,10 @@ notice( const char *msg, ... )
 {
 	va_list va;
 
-	if (!(DFlags & QUIET)) {
+	if (Verbosity >= TERSE) {
 		va_start( va, msg );
-		printn( msg, va );
+		nvprint( msg, va );
 		va_end( va );
-		need_nl = 0;
 	}
 }
 
@@ -131,7 +143,7 @@ warn( const char *msg, ... )
 {
 	va_list va;
 
-	if (!(DFlags & VERYQUIET)) {
+	if (Verbosity >= QUIET) {
 		flushn();
 		va_start( va, msg );
 		vfprintf( stderr, msg, va );
@@ -173,6 +185,225 @@ sys_error( const char *msg, ... )
 	va_end( va );
 }
 
+// Minimal printf() replacement with custom format sequence(s):
+// - %\\s
+//   Print backslash-escaped string literals. Note that this does not
+//   automatically add quotes around the printed string, so it is
+//   possible to concatenate multiple segments.
+// - %!s
+//   Same as %\\s, but non-ASCII characters are (hex-)escaped as well.
+// - %!&s
+//   Same as %!s, but linefeeds are also printed verbatim for legibility.
+
+// TODO: Trade off segments vs. buffer capacity dynamically.
+#define QPRINTF_SEGS 16
+#ifndef QPRINTF_BUFF
+# define QPRINTF_BUFF 1000
+#endif
+
+typedef void (*printf_cb)( const char **segs, uint *segls, int nsegs, uint totlen, void *aux );
+
+static void
+xvprintf_core( const char *fmt, va_list ap, printf_cb cb, void *cb_aux )
+{
+	int nsegs = 0;
+	uint totlen = 0;
+	const char *segs[QPRINTF_SEGS];
+	uint segls[QPRINTF_SEGS];
+	char buf[QPRINTF_BUFF];
+
+#define ADD_SEG(p, l) \
+		do { \
+			if (nsegs == QPRINTF_SEGS) \
+				oob(); \
+			segs[nsegs] = p; \
+			segls[nsegs++] = l; \
+			totlen += l; \
+		} while (0)
+
+	char *d = buf;
+	char *ed = d + sizeof(buf);
+	const char *s = fmt;
+	for (;;) {
+		char c = *fmt;
+		if (!c || c == '%') {
+			uint l = fmt - s;
+			if (l)
+				ADD_SEG( s, l );
+			if (!c)
+				break;
+			uint maxlen = UINT_MAX;
+			c = *++fmt;
+			if (c == '.') {
+				c = *++fmt;
+				if (c != '*') {
+					fputs( "Fatal: unsupported string length specification. Please report a bug.\n", stderr );
+					abort();
+				}
+				maxlen = va_arg( ap, uint );
+				c = *++fmt;
+			}
+			int escaped = 0;
+			if (c == '\\') {
+				escaped = 1;
+				c = *++fmt;
+			} else if (c == '!') {
+				escaped = 2;
+				c = *++fmt;
+				if (c == '&') {
+					escaped = 3;
+					c = *++fmt;
+				}
+			}
+			if (c == 'c') {
+				if (d + 1 > ed)
+					oob();
+				ADD_SEG( d, 1 );
+				*d++ = (char)va_arg( ap, int );
+			} else if (c == 's') {
+				s = va_arg( ap, const char * );
+				if (escaped) {
+					char *bd = d;
+					for (l = 0; l < maxlen && (c = *s); l++, s++) {
+						if (c == '\\' || c == '"') {
+							if (d >= ed)
+								oob();
+							*d++ = '\\';
+						} else if (escaped >= 2 && (c < 32 || c > 126)) {
+							switch (c) {
+							case '\r': c = 'r'; break;
+							case '\t': c = 't'; break;
+							case '\a': c = 'a'; break;
+							case '\b': c = 'b'; break;
+							case '\v': c = 'v'; break;
+							case '\f': c = 'f'; break;
+							case '\n':
+								if (escaped == 2) {
+									c = 'n';
+									break;
+								}
+								if (d + 2 >= ed)
+									oob();
+								*d++ = '\\';
+								*d++ = 'n';
+								*d++ = c;  // Keep the actual line break for legibility.
+								continue;
+							default:
+								d += nfsnprintf( d, ed - d, "\\x%02x", (uchar)c );
+								continue;
+							}
+							if (d >= ed)
+								oob();
+							*d++ = '\\';
+						}
+						if (d >= ed)
+							oob();
+						*d++ = c;
+					}
+					l = d - bd;
+					if (l)
+						ADD_SEG( bd, l );
+				} else {
+					l = strnlen( s, maxlen );
+					if (l)
+						ADD_SEG( s, l );
+				}
+			} else if (c == 'd') {
+				l = nfsnprintf( d, ed - d, "%d", va_arg( ap, int ) );
+				ADD_SEG( d, l );
+				d += l;
+			} else if (c == 'u') {
+				l = nfsnprintf( d, ed - d, "%u", va_arg( ap, uint ) );
+				ADD_SEG( d, l );
+				d += l;
+			} else {
+				fputs( "Fatal: unsupported format specifier. Please report a bug.\n", stderr );
+				abort();
+			}
+			s = ++fmt;
+		} else {
+			fmt++;
+		}
+	}
+	cb( segs, segls, nsegs, totlen, cb_aux );
+}
+
+static void
+xasprintf_cb( const char **segs, uint *segls, int nsegs, uint totlen, void *aux )
+{
+	char *d = nfmalloc( totlen + 1 );
+	*(char **)aux = d;
+	for (int i = 0; i < nsegs; i++) {
+		memcpy( d, segs[i], segls[i] );
+		d += segls[i];
+	}
+	*d = 0;
+}
+
+char *
+xvasprintf( const char *fmt, va_list ap )
+{
+	char *out;
+	xvprintf_core( fmt, ap, xasprintf_cb, &out );
+	return out;
+}
+
+#ifndef HAVE_FWRITE_UNLOCKED
+# define flockfile(f)
+# define funlockfile(f)
+# define fwrite_unlocked(b, l, n, f) fwrite(b, l, n, f)
+#endif
+
+static void
+xprintf_cb( const char **segs, uint *segls, int nsegs, uint totlen ATTR_UNUSED, void *aux ATTR_UNUSED )
+{
+	flockfile( stdout );
+	for (int i = 0; i < nsegs; i++)
+		fwrite_unlocked( segs[i], 1, segls[i], stdout );
+	funlockfile( stdout );
+}
+
+void
+xprintf( const char *fmt, ... )
+{
+	va_list va;
+
+	va_start( va, fmt );
+	xvprintf_core( fmt, va, xprintf_cb, NULL );
+	va_end( va );
+}
+
+void
+vFprintf( FILE *f, const char *msg, va_list va )
+{
+	int r;
+
+	r = vfprintf( f, msg, va );
+	if (r < 0) {
+		sys_error( "Error: cannot write file" );
+		exit( 1 );
+	}
+}
+
+void
+Fprintf( FILE *f, const char *msg, ... )
+{
+	va_list va;
+
+	va_start( va, msg );
+	vFprintf( f, msg, va );
+	va_end( va );
+}
+
+void
+Fclose( FILE *f, int safe )
+{
+	if ((safe && (fflush( f ) || (UseFSync && fdatasync( fileno( f ) )))) || fclose( f ) == EOF) {
+		sys_error( "Error: cannot close file" );
+		exit( 1 );
+	}
+}
+
 void
 add_string_list_n( string_list_t **list, const char *str, uint len )
 {
@@ -242,6 +473,13 @@ strnlen( const char *str, size_t maxlen )
 
 #endif
 
+void
+to_upper( char *str, uint len )
+{
+	for (uint i = 0; i < len; i++)
+		str[i] = toupper( str[i] );
+}
+
 int
 starts_with( const char *str, int strl, const char *cmp, uint cmpl )
 {
@@ -250,6 +488,15 @@ starts_with( const char *str, int strl, const char *cmp, uint cmpl )
 	return ((uint)strl >= cmpl) && !memcmp( str, cmp, cmpl );
 }
 
+static int
+equals_upper_impl( const char *str, const char *cmp, uint cmpl )
+{
+	for (uint i = 0; i < cmpl; i++)
+		if (toupper( str[i] ) != cmp[i])
+			return 0;
+	return 1;
+}
+
 int
 starts_with_upper( const char *str, int strl, const char *cmp, uint cmpl )
 {
@@ -257,10 +504,7 @@ starts_with_upper( const char *str, int strl, const char *cmp, uint cmpl )
 		strl = strnlen( str, cmpl + 1 );
 	if ((uint)strl < cmpl)
 		return 0;
-	for (uint i = 0; i < cmpl; i++)
-		if (str[i] != cmp[i] && toupper( str[i] ) != cmp[i])
-			return 0;
-	return 1;
+	return equals_upper_impl( str, cmp, cmpl );
 }
 
 int
@@ -271,6 +515,16 @@ equals( const char *str, int strl, const char *cmp, uint cmpl )
 	return ((uint)strl == cmpl) && !memcmp( str, cmp, cmpl );
 }
 
+int
+equals_upper( const char *str, int strl, const char *cmp, uint cmpl )
+{
+	if (strl < 0)
+		strl = strnlen( str, cmpl + 1 );
+	if ((uint)strl != cmpl)
+		return 0;
+	return equals_upper_impl( str, cmp, cmpl );
+}
+
 #ifndef HAVE_TIMEGM
 /*
    Converts struct tm to time_t, assuming the data in tm is UTC rather
@@ -336,6 +590,21 @@ timegm( struct tm *t )
 }
 #endif
 
+void
+fmt_bits( uint bits, uint num_bits, const char *bit_str, const int *bit_off, char *buf )
+{
+	uint d = 0;
+	for (uint i = 0, val = 1; i < num_bits; i++, val <<= 1) {
+		if (bits & val) {
+			if (d)
+				buf[d++] = ',';
+			for (const char *s = bit_str + bit_off[i]; *s; s++)
+				buf[d++] = *s;
+		}
+	}
+	buf[d] = 0;
+}
+
 void
 oob( void )
 {
@@ -374,7 +643,7 @@ nfmalloc( size_t sz )
 }
 
 void *
-nfcalloc( size_t sz )
+nfzalloc( size_t sz )
 {
 	void *ret;
 
@@ -449,53 +718,23 @@ cur_user( void )
 }
 */
 
-char *
-expand_strdup( const char *s )
-{
-	struct passwd *pw;
-	const char *p, *q;
-	char *r;
-
-	if (*s == '~') {
-		s++;
-		if (!*s) {
-			p = NULL;
-			q = Home;
-		} else if (*s == '/') {
-			p = s;
-			q = Home;
-		} else {
-			if ((p = strchr( s, '/' ))) {
-				r = nfstrndup( s, (size_t)(p - s) );
-				pw = getpwnam( r );
-				free( r );
-			} else
-				pw = getpwnam( s );
-			if (!pw)
-				return NULL;
-			q = pw->pw_dir;
-		}
-		nfasprintf( &r, "%s%s", q, p ? p : "" );
-		return r;
-	} else
-		return nfstrdup( s );
-}
-
 /* Return value: 0 = ok, -1 = out found in arg, -2 = in found in arg but no out specified */
 int
-map_name( const char *arg, char **result, uint reserve, const char *in, const char *out )
+map_name( const char *arg, int l, char **result, uint reserve, const char *in, const char *out )
 {
 	char *p;
-	uint i, l, ll, num, inl, outl;
+	int i, ll, num, inl, outl;
 
 	assert( arg );
-	l = strlen( arg );
+	if (l < 0)
+		l = strlen( arg );
 	assert( in );
 	inl = strlen( in );
 	if (!inl) {
 	  copy:
 		*result = nfmalloc( reserve + l + 1 );
-		memcpy( *result + reserve, arg, l + 1 );
+		memcpy( *result + reserve, arg, l );
+		(*result)[reserve + l] = 0;
 		return 0;
 	}
 	assert( out );
@@ -503,6 +742,8 @@ map_name( const char *arg, char **result, uint reserve, const char *in, const ch
 	if (equals( in, (int)inl, out, outl ))
 		goto copy;
 	for (num = 0, i = 0; i < l; ) {
+		if (i + inl > l)
+			goto fout;
 		for (ll = 0; ll < inl; ll++)
 			if (arg[i + ll] != in[ll])
 				goto fout;
@@ -511,6 +752,8 @@ map_name( const char *arg, char **result, uint reserve, const char *in, const ch
 		continue;
 	  fout:
 		if (outl) {
+			if (i + outl > l)
+				goto fnexti;
 			for (ll = 0; ll < outl; ll++)
 				if (arg[i + ll] != out[ll])
 					goto fnexti;
@@ -526,6 +769,8 @@ map_name( const char *arg, char **result, uint reserve, const char *in, const ch
 	*result = nfmalloc( reserve + l + num * (outl - inl) + 1 );
 	p = *result + reserve;
 	for (i = 0; i < l; ) {
+		if (i + inl > l)
+			goto rnexti;
 		for (ll = 0; ll < inl; ll++)
 			if (arg[i + ll] != in[ll])
 				goto rnexti;
@@ -540,6 +785,21 @@ map_name( const char *arg, char **result, uint reserve, const char *in, const ch
 	return 0;
 }
 
+int
+mkdir_p( char *path, int len )
+{
+	if (!mkdir( path, 0700 ) || errno == EEXIST)
+		return 0;
+	char *p = memrchr( path, '/', (size_t)len );
+	*p = 0;
+	if (mkdir_p( path, (int)(p - path) )) {
+		*p = '/';
+		return -1;
+	}
+	*p = '/';
+	return mkdir( path, 0700 );
+}
+
 static int
 compare_uints( const void *l, const void *r )
 {
@@ -741,10 +1001,41 @@ wipe_notifier( notifier_t *sn )
 #endif
 }
 
-static time_t
+#if _POSIX_TIMERS - 0 > 0
+static clockid_t clkid;
+#endif
+
+void
+init_timers( void )
+{
+#if _POSIX_TIMERS - 0 > 0
+	struct timespec ts;
+# ifdef CLOCK_BOOTTIME
+	if (!clock_gettime( CLOCK_BOOTTIME, &ts )) {
+		clkid = CLOCK_BOOTTIME;
+	} else
+# endif
+# ifdef CLOCK_MONOTONIC_COARSE
+	if (!clock_gettime( CLOCK_MONOTONIC_COARSE, &ts )) {
+		clkid = CLOCK_MONOTONIC_COARSE;
+	} else
+# endif
+	clkid = CLOCK_MONOTONIC;
+#endif
+}
+
+int64_t
 get_now( void )
 {
-	return time( NULL );
+#if _POSIX_TIMERS - 0 > 0
+	struct timespec ts;
+	clock_gettime( clkid, &ts );
+	return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000;
+#else
+	struct timeval tv;
+	gettimeofday( &tv, NULL );
+	return tv.tv_sec * 1000LL + tv.tv_usec / 1000;
+#endif
 }
 
 static list_head_t timers = { &timers, &timers };
@@ -773,7 +1064,7 @@ conf_wakeup( wakeup_t *tmr, int to )
 		if (tmr->links.next)
 			list_unlink( &tmr->links );
 	} else {
-		time_t timeout = to;
+		int64_t timeout = to;
 		if (!to) {
 			/* We always prepend null timers, to cluster related events. */
 			succ = timers.next;
@@ -807,18 +1098,20 @@ event_wait( void )
 	int timeout = -1;
 	if ((head = timers.next) != &timers) {
 		wakeup_t *tmr = (wakeup_t *)head;
-		time_t delta = tmr->timeout;
+		int64_t delta = tmr->timeout;
 		if (!delta || (delta -= get_now()) <= 0) {
 			list_unlink( head );
 			tmr->cb( tmr->aux );
 			return;
 		}
-		timeout = (int)delta * 1000;
+		timeout = (int)delta;
 	}
 	switch (poll( pollfds, npolls, timeout )) {
 	case 0:
 		return;
 	case -1:
+		if (errno == EINTR)
+			return;
 		perror( "poll() failed in event loop" );
 		abort();
 	default:
@@ -843,14 +1136,14 @@ event_wait( void )
 
 	if ((head = timers.next) != &timers) {
 		wakeup_t *tmr = (wakeup_t *)head;
-		time_t delta = tmr->timeout;
+		int64_t delta = tmr->timeout;
 		if (!delta || (delta -= get_now()) <= 0) {
 			list_unlink( head );
 			tmr->cb( tmr->aux );
 			return;
 		}
-		to_tv.tv_sec = delta;
-		to_tv.tv_usec = 0;
+		to_tv.tv_sec = delta / 1000;
+		to_tv.tv_usec = delta * 1000;
 		timeout = &to_tv;
 	}
 	FD_ZERO( &rfds );
@@ -871,6 +1164,8 @@ event_wait( void )
 	case 0:
 		return;
 	case -1:
+		if (errno == EINTR)
+			return;
 		perror( "select() failed in event loop" );
 		abort();
 	default: