Merge pull request systemd#130 from poettering/feature-flags

Numerous changes, including support for quota project IDs and .gitignore-like files

TODO

@@ -4,7 +4,6 @@ SHORT-TERM:
 
 TO MAKE IT USEFUL FOR BACKUPS:
 * encryption: aes256 of rotating hash function + HMAC for identifying chunks + individually encrypted chunks
-* speed up repeated image generation: maintain persistent cache for directory trees that permits lookups by a path location as key, returning a chunk id and "newest covering mtime"
 
 LATER:
 * verify
@@ -26,22 +25,27 @@ LATER:
 * implicitly generate index + chunks when accessing base trees or archives through native protocol
 * rework caindex to read multiple chunks per read (use reallocbuffer like elsewhere)
 * permit 511 (or 4095?) redundant NUL bytes at the end of archive and index files, so that they could in theory stored on block devices
-* seed: cache GOODBYE name table data so that we can regenerate the right bits when needed
 * optionally import from/export to classic tar ball (and zip?)
 * optionally interpret aufs/union mount whiteout files?
-* fuse: expose acls and fcaps, selinux
+* fuse: expose acls and fcaps, selinux, quota projid
 * fuse: possibly translate user names on access
 * fuse: provide "mount.casync" compat symlink so that people can list casync mounts in /etc/fstab
 * encoder: change seeking to be more like decoder's seeking (i.e. delay returned events until the next ca_encoder_step() call)
 * rename offset accessor functions (drop the "archive")
 * send progress information via sd_notify(), so that people can wrap casync nicely in UIs
 * maybe turn "recursive" mode into a numeric value specifying how far to descend?
 * make "casync stat" work on a directory with a subpath
-* save/restore xfs/ext4 projid
 * tweak chunker: shift cut to last "marker".
 * define sane errors we can show user messages about
 * introduce a --best-effort mode when replaying, which means we'll ignore what we can't apply
-* add "--cache-auto" mode, that works like "--cache" but finds a suitable place for the cache automatically
-* use CaNameTable logic in seed, so that we can safely regenerate *any* chunk
 * when building the cache, also build a seed
-* make sure that "casync list" on a directory doesn't read any files
+* recognize NULL chunks and handle them specially
+* make archive digest generation optional
+* add "index" digest
+* maybe add a concept of "packfiles", to contain multiple chunks in one?
+* add libsmbclient backend (so that Lennart can backup to his synology NAS in the easiest way)
+* make sure "casync list /etc/fstab" does something useful
+* rework CaSeed logic to use CaCache as backend, and then add a new command "casync cache" or so, to explicitly generate a cache/seed
+* support blake2 as hashes
+* parallelize image generation: when storing chunks in the store do so in a thread
+* in "casync stat" output show which flags enable what

doc/casync.rst

@@ -159,16 +159,20 @@ General options:
 --digest=<DIGEST>               Pick digest algorithm (sha512-256 or sha256)
 --compression=<COMPRESSION>     Pick compression algorithm (zstd, xz or gzip)
 --seed=<PATH>                   Additional file or directory to use as seed
+--cache=<PATH>                  Directory to use as encoder cache
+--cache-auto, -c                Pick encoder cache directory automatically
 --rate-limit-bps=<LIMIT>        Maximum bandwidth in bytes/s for remote communication
 --exclude-nodump=no             Don't exclude files with chattr(1)'s +d **nodump** flag when creating archive
 --exclude-submounts=yes         Exclude submounts when creating archive
+--exclude-file=no               Don't respect .caexclude files in the file tree
 --reflink=no                    Don't create reflinks from seeds when extracting
 --hardlink=yes                  Create hardlinks from seeds when extracting
 --punch-holes=no                Don't create sparse files when extracting
 --delete=no                     Don't delete existing files not listed in archive after extraction
 --undo-immutable=yes            When removing existing files, undo chattr(1)'s +i 'immutable' flag when extracting
 --seed-output=no                Don't implicitly add pre-existing output as seed when extracting
 --recursive=no                  List non-recursively
+--mkdir=no                      Don't automatically create mount directory if it is missing
 --uid-shift=<yes|SHIFT>         Shift UIDs/GIDs
 --uid-range=<RANGE>             Restrict UIDs/GIDs to range
 
@@ -232,6 +236,7 @@ Individual archive features:
 --with=<acl>               Store file access control lists
 --with=<selinux>           Store SElinux file labels
 --with=<fcaps>             Store file capabilities
+--with=<quota-projid>      Store ext4/XFS quota project ID
 
 (and similar: ``--without=16bit-uids``, ``--without=32bit-uids``, ...)
 
@@ -241,3 +246,47 @@ Archive features
 The various ``--with=`` and ``--without=`` parameters control the precise set
 of metadata to store in the archive, or restore when extracting. These flags
 only apply if ``casync`` operates on the file system level.
+
+Excluding Files and Directories from Archiving
+----------------------------------------------
+
+When generating an archive or index from a file system directory tree, some
+files and directories are excluded by default and others may optionally be
+excluded:
+
+1. Files and directories of virtual API file systems exposed by the kernel
+   (i.e. procfs, sysfs, cgroupfs, devpts … — but not tmpfs/devtmpfs) are
+   excluded unconditionally.
+
+2. Depending on whether symlinks, device nodes, fifos and sockets are enabled
+   for archiving with ``--with=`` and ``--without=``, file nodes of these types
+   are excluded.
+
+3. By default, files and directories with the ``+d`` chattr(1) flag set are
+   excluded, however this behaviour may be turned off with
+   ``--exclude-nodump=no``.
+
+4. Optionally, files and directories contained in submounts of the specified
+   file system tree are excluded, if ``--exclude-submounts=yes`` is specified.
+
+5. By default, any files and directories listed in ``.caexclude`` files in the
+   file hierarchy are excluded, however interpretation of these files may be
+   turned off with ``--exclude-file=no``. These files operate similar to
+   ``git``'s ``.gitignore`` concept: they are read as text file where each line
+   is either empty/starts with ``#`` (in which case they have no effect, which
+   may be used for commenting), or list a globbing path pattern of
+   files/directories to ignore. If a line contains no ``/`` character the line
+   applies to the directory the ``.caexclude`` file is located in as well as
+   all child directories of it. If it contains at least one ``/`` character it
+   is considered stricly relative to the directory the ``.caexclude`` file is
+   located in. ``.caexclude`` files may appear in any directory of the file
+   system tree that is archived, however they have no effect when placed in
+   directories that are marked for exclusion via ``.caexclude`` files placed
+   further up in the directory tree. When a line ends in a ``/`` character it
+   applies to directories only, and not regular files or other file node
+   types. If a line is prefixed with a ``!`` character matching files are
+   excluded from the exclusion, i.e. the effect of other matching lines that
+   are not prefixed like this is cancelled for matching files. ``!`` lines
+   unconditionally take precedence over lines not marked like this. Moreover,
+   lines prefixed with ``!`` also cancel the effect of patterns in
+   ``.caexclude`` files placed in directories further up the tree.

meson.build

@@ -295,10 +295,13 @@ test_sources = '''
         test-cachunker-histogram
         test-cadigest
         test-caencoder
+        test-calocation
         test-camakebst
+        test-camatch
         test-caorigin
         test-casync
         test-cautil
+        test-feature-flags
         test-util
 '''.split()
 

src/cacache.c

@@ -276,7 +276,7 @@ int ca_cache_get(CaCache *c, CaLocation *location, CaChunkID *ret_chunk_id, CaOr
                 return -EINVAL;
 
         /* The first item of the origin must match our lookup key. If it doesn't something's bad. */
-        if (!ca_location_equal(location, ca_origin_get(origin, 0), false))
+        if (!ca_location_equal(location, ca_origin_get(origin, 0), CA_LOCATION_WITH_MTIME|CA_LOCATION_WITH_FEATURE_FLAGS))
                 return -EINVAL;
 
         if (ret_chunk_id)
@@ -335,7 +335,7 @@ int ca_cache_put(CaCache *c, CaOrigin *origin, const CaChunkID *chunk_id) {
                 const char *f;
                 char *p;
 
-                /* If there's only a single item, then let's try to create this as symlink, is it is the cheapest
+                /* If there's only a single item, then let's try to create this as symlink, it is the cheapest
                  * option */
 
                 f = ca_location_format(first_location);
@@ -367,7 +367,7 @@ int ca_cache_put(CaCache *c, CaOrigin *origin, const CaChunkID *chunk_id) {
         if (r < 0)
                 return r;
 
-        fd = openat(c->fd, temp, O_CREAT|O_WRONLY|O_CLOEXEC|O_EXCL|O_NOFOLLOW|O_EXCL, 0666);
+        fd = openat(c->fd, temp, O_CREAT|O_EXCL|O_WRONLY|O_CLOEXEC|O_NOFOLLOW, 0666);
         if (fd < 0)
                 return -errno;