Skip to content

Commit

Permalink
casync-tool: gc verb
Browse files Browse the repository at this point in the history 
Fixes systemd#43.
  • Loading branch information
@keszybz
keszybz committed Nov 24, 2017
1 parent 7aa62e3 commit 758566a
Show file tree
Hide file tree
Showing 8 changed files with 439 additions and 10 deletions.
4 changes: 3 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,9 @@ but there are other systems that use similar algorithms, in particular:
## Maintenance

```
# casync gc /var/lib/backup.castr /home/lennart.caidx /home/foobar.caidx ... (NOT IMPLEMENTED YET)
# casync gc /home/lennart-20170101.caidx /home/lennart-20170102.caidx /home/lennart-20170103.caidx
# casync gc --backup /var/lib/backup/backup.castr /home/lennart-*.caidx
# casync make /home/lennart.catab /home/lennart (NOT IMPLEMENTED)
```

Expand Down
28 changes: 20 additions & 8 deletions doc/casync.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Synopsis
| **casync** [*OPTIONS*...] digest [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
| **casync** [*OPTIONS*...] mount [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
| **casync** [*OPTIONS*...] mkdev [*BLOB* | *BLOB_INDEX*] [*NODE*]
| **casync** [*OPTIONS*...] gc *BLOB_INDEX* | *ARCHIVE_INDEX* ...
Description
-----------
Expand All @@ -42,8 +43,8 @@ The metadata included in the archive is controlled by the ``--with-*`` and
``--without-*`` options.

|
| **casync** extract [*ARCHIVE* | *ARCHIVE_INDEX*] [*DIRECTORY*]
| **casync** extract *BLOB_INDEX* *FILE* | *DEVICE*
| **casync** **extract** [*ARCHIVE* | *ARCHIVE_INDEX*] [*DIRECTORY*]
| **casync** **extract** *BLOB_INDEX* *FILE* | *DEVICE*
This will extract the contents of a .catar archive or .caidx index
into the specified *DIRECTORY*, or the contents specified by *BLOB_INDEX*
Expand All @@ -54,7 +55,7 @@ The metadata replayed from the archive is controlled by the ``--with-*`` and
``--without-*`` options.

|
| **casync** list [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
| **casync** **list** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
This will list all the files and directories in the specified .catar
archive or .caidx index, or the directory. The argument is optional,
Expand All @@ -68,7 +69,7 @@ The output includes the permission mask and file names::
-rw-r--r-- TODO

|
| **casync** mtree [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
| **casync** **mtree** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*]
This is similar to **list**, but includes information about each entry in the
key=value format defined by BSD mtree(5)::
Expand All @@ -79,7 +80,7 @@ key=value format defined by BSD mtree(5)::
TODO type=file mode=0644 size=2395 uid=0 gid=0 time=1498175562.000000000 sha256digest=316f11a03c08ec39f0328ab1f7446bd048507d3fbeafffe7c32fad4942244b7d

|
| **casync** stat [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*] [*PATH*]
| **casync** **stat** [*ARCHIVE* | *ARCHIVE_INDEX* | *DIRECTORY*] [*PATH*]
This will show detailed information about a file or directory *PATH*, as found
in either *ARCHIVE* or *ARCHIVE_INDEX* or underneath *DIRECTORY*. Both arguments
Expand All @@ -99,7 +100,7 @@ Example output::
Group: zbyszek (1000)

|
| **casync** digest [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
| **casync** **digest** [*ARCHIVE* | *BLOB* | *ARCHIVE_INDEX* | *BLOB_INDEX* | *DIRECTORY*]
This will compute and print the checksum of the argument.
The argument is optional and defaults to the current directory::
Expand All @@ -111,13 +112,13 @@ The argument is optional and defaults to the current directory::
d1698b0c4c27163284abea5d1e369b92e89dd07cb74378638849800e0406baf7

|
| **casync** mount [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
| **casync** **mount** [*ARCHIVE* | *ARCHIVE_INDEX*] *PATH*
This will mount the specified .catar archive or .caidx index at the
specified *PATH*, using the FUSE protocol.

|
| **casync** mkdev [*BLOB* | *BLOB_INDEX*] [*NODE*]
| **casync** **mkdev** [*BLOB* | *BLOB_INDEX*] [*NODE*]
This will create a block device *NODE* with the contents specified
by the .caibx *BLOB_INDEX* or just the file or block device *BLOB*,
Expand All @@ -134,13 +135,24 @@ Example::

When ``casync mkdev`` is killed, the device is destroyed.

|
| **casync** **gc** *ARCHIVE_INDEX* | *BLOB_INDEX* ...
This will remove all chunks that are not used by one of the specified indices
(one or more blob and archive indices can be given). If ``--store`` is not
given, the default store for the first index will be used.

This command can be used to prune unused chunks from a shared chunk
store.

Options
-------

General options:

--help, -h Show terse help output
--verbose, -v Show terse status information during runtime
--dry-run, -n Only print what would be removed with **gc**
--store=PATH The primary chunk store to use
--extra-store=<PATH> Additional chunk store to look for chunks in
--chunk-size=<[MIN:]AVG[:MAX]> The minimal/average/maximum number of bytes in a chunk
Expand Down
95 changes: 95 additions & 0 deletions src/castore.c
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
/* SPDX-License-Identifier: LGPL-2.1+ */

#include <dirent.h>
#include <fcntl.h>
#include <lzma.h>
#include <sys/stat.h>
#include <unistd.h>

#include "castore.h"
#include "def.h"
#include "dirent-util.h"
#include "realloc-buffer.h"
#include "rm-rf.h"
#include "util.h"
Expand Down Expand Up @@ -34,6 +36,14 @@ struct CaStore {
uint64_t n_request_bytes;
};

struct CaStoreIterator {
CaStore *store;

DIR *rootdir;
struct dirent *subdir_de;
DIR *subdir;
};

CaStore* ca_store_new(void) {
CaStore *store;

Expand Down Expand Up @@ -312,3 +322,88 @@ int ca_store_set_digest_type(CaStore *s, CaDigestType type) {

return 0;
}

CaStoreIterator* ca_store_iterator_new(CaStore *store) {
CaStoreIterator *it;

it = new0(CaStoreIterator, 1);
if (!it)
return NULL;

it->store = store;

return it;
}

CaStoreIterator* ca_store_iterator_unref(CaStoreIterator *iter) {
if (!iter)
return NULL;

if (iter->rootdir)
closedir(iter->rootdir);
if (iter->subdir)
closedir(iter->subdir);
return mfree(iter);
}

int ca_store_iterator_next(
CaStoreIterator *iter,
int *rootdir_fd,
const char **subdir,
int *subdir_fd,
const char **chunk) {

struct dirent *de;

if (!iter->rootdir) {
iter->rootdir = opendir(iter->store->root);
if (!iter->rootdir)
return -errno;
}

while (true) {
if (!iter->subdir) {
int fd;

errno = 0;
iter->subdir_de = readdir(iter->rootdir);
if (!iter->subdir_de) {
if (errno > 0)
return -errno;
return 0; /* done */
}

fd = openat(dirfd(iter->rootdir), iter->subdir_de->d_name,
O_RDONLY|O_CLOEXEC|O_DIRECTORY);
if (fd < 0) {
if (errno == EISDIR)
continue;
return -errno;
}

iter->subdir = fdopendir(fd);
if (!iter->subdir) {
safe_close(fd);
return -errno;
}
}

FOREACH_DIRENT_ALL(de, iter->subdir, return -errno) {
if (!dirent_is_file_with_suffix(de, ".cacnk"))
continue;

if (rootdir_fd)
*rootdir_fd = dirfd(iter->rootdir);
if (subdir)
*subdir = iter->subdir_de->d_name;
if (subdir_fd)
*subdir_fd = dirfd(iter->subdir);
if (chunk)
*chunk = de->d_name;
return 1; /* success */
}

assert_se(closedir(iter->subdir) == 0);
iter->subdir = NULL;
}
}
16 changes: 16 additions & 0 deletions src/castore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,14 @@
#include "cautil.h"

typedef struct CaStore CaStore;
typedef struct CaStoreIterator CaStoreIterator;

CaStore* ca_store_new(void);
CaStore *ca_store_new_cache(void);
CaStore* ca_store_unref(CaStore *store);
static inline void ca_store_unrefp(CaStore **store) {
ca_store_unref(*store);
}

int ca_store_set_path(CaStore *store, const char *path);
int ca_store_set_compression(CaStore *store, CaChunkCompression c);
Expand All @@ -26,4 +30,16 @@ int ca_store_get_request_bytes(CaStore *s, uint64_t *ret);

int ca_store_set_digest_type(CaStore *s, CaDigestType type);

CaStoreIterator* ca_store_iterator_new(CaStore *store);
CaStoreIterator* ca_store_iterator_unref(CaStoreIterator *iter);
static inline void ca_store_iterator_unrefp(CaStoreIterator **iter) {
ca_store_iterator_unref(*iter);
}
int ca_store_iterator_next(
CaStoreIterator *iter,
int *rootdir_fd,
const char **subdir,
int *subdir_fd,
const char **chunk);

#endif
74 changes: 73 additions & 1 deletion src/casync-tool.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "caremote.h"
#include "castore.h"
#include "casync.h"
#include "gc.h"
#include "notify.h"
#include "parse-util.h"
#include "signal-handler.h"
Expand All @@ -35,6 +36,7 @@ static enum arg_what {
_WHAT_INVALID = -1,
} arg_what = _WHAT_INVALID;
static bool arg_verbose = false;
static bool arg_dry_run = false;
static bool arg_exclude_nodump = true;
static bool arg_exclude_submounts = false;
static bool arg_reflink = true;
Expand Down Expand Up @@ -74,6 +76,8 @@ static void help(void) {
" -h --help Show this help\n"
" --version Show brief version information\n"
" -v --verbose Show terse status information during runtime\n"
" -n --dry-run When garbage collecting, only print what would\n"
" be done\n"
" --store=PATH The primary chunk store to use\n"
" --extra-store=PATH Additional chunk store to look for chunks in\n"
" --chunk-size=[MIN:]AVG[:MAX]\n"
Expand Down Expand Up @@ -312,6 +316,7 @@ static int parse_argv(int argc, char *argv[]) {
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, ARG_VERSION },
{ "verbose", no_argument, NULL, 'v' },
{ "dry-run", no_argument, NULL, 'n' },
{ "store", required_argument, NULL, ARG_STORE },
{ "extra-store", required_argument, NULL, ARG_EXTRA_STORE },
{ "chunk-size", required_argument, NULL, ARG_CHUNK_SIZE },
Expand Down Expand Up @@ -345,7 +350,7 @@ static int parse_argv(int argc, char *argv[]) {
if (getenv_bool("CASYNC_VERBOSE") > 0)
arg_verbose = true;

while ((c = getopt_long(argc, argv, "hv", options, NULL)) >= 0) {
while ((c = getopt_long(argc, argv, "hvn", options, NULL)) >= 0) {

switch (c) {

Expand All @@ -361,6 +366,10 @@ static int parse_argv(int argc, char *argv[]) {
arg_verbose = true;
break;

case 'n':
arg_dry_run = true;
break;

case ARG_STORE: {
char *p;

Expand Down Expand Up @@ -3807,6 +3816,67 @@ static int verb_udev(int argc, char *argv[]) {
return 0;
}

static int verb_gc(int argc, char *argv[]) {
int i, r;
_cleanup_(ca_chunk_collection_unrefp) CaChunkCollection *coll = NULL;
_cleanup_(ca_store_unrefp) CaStore *store = NULL;

if (argc < 2) {
fprintf(stderr, "Expected at least one argument.\n");
return -EINVAL;
}

coll = ca_chunk_collection_new();
if (!coll)
return log_oom();

/* This sets the same store for all indices, based on the first index. */
r = set_default_store(argv[1]);
if (r < 0)
return r;

if (!arg_store) {
fprintf(stderr, "Failed to determine store, use --store= to set store.\n");
return -EINVAL;
}

store = ca_store_new();
if (!store)
return log_oom();

r = ca_store_set_path(store, arg_store);
if (r < 0) {
fprintf(stderr, "Set to set store to \"%s\": %s", arg_store, strerror(-r));
return r;
}

for (i = 1; i < argc; i++) {
const char *path = argv[i];

r = ca_chunk_collection_add_index(coll, path);
if (r < 0)
return r;
}

{
size_t usage, size;

assert_se(ca_chunk_collection_usage(coll, &usage) == 0);
assert_se(ca_chunk_collection_size(coll, &size) == 0);
if (arg_verbose)
printf("Chunk store usage: %zu references, %zu chunks\n", usage, size);
}

r = ca_gc_cleanup_unused(store, coll,
arg_verbose * CA_GC_VERBOSE |
arg_dry_run * CA_GC_DRY_RUN);
if (r < 0)
fprintf(stderr, "Chunk cleanup failed: %s\n", strerror(-r));

return r;
}


static int dispatch_verb(int argc, char *argv[]) {
int r;

Expand Down Expand Up @@ -3836,6 +3906,8 @@ static int dispatch_verb(int argc, char *argv[]) {
r = verb_push(argc, argv);
else if (streq(argv[0], "udev")) /* "Secret" verb, only to be called by the udev nbd rules */
r = verb_udev(argc, argv);
else if (streq(argv[0], "gc"))
r = verb_gc(argc, argv);
else {
fprintf(stderr, "Unknown verb '%s'. (Invoke '%s --help' for a list of available verbs.)\n", argv[0], program_invocation_short_name);
r = -EINVAL;
Expand Down
Loading

0 comments on commit 758566a

Please sign in to comment.