From 83c7c0fce52631f59c374f1744b138bddec2af40 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 22 Jan 2025 13:54:58 +0100 Subject: [PATCH] Added option to print rsync performance stats in cf-net The performance stats are also logged. This way we can also study the performance through `cf-agent`'s debug logs. The following is an example of running cf-net with the `--stats` option: ``` $ sudo /var/cfengine/bin/cf-net --stats --host 192.168.56.10 get /var/cfengine/masterfiles/promises.cf Send signature statistics: 16970 bytes in (read from 'promises.cf') 2424 bytes out (sent to server) Receive delta statistics: 9 bytes in (received from server) 16970 bytes out (written to 'promises.cf.cfnew') ``` Changelog: Title Signed-off-by: Lars Erik Wik --- cf-net/cf-net.c | 15 ++++++++++-- libcfnet/client_code.c | 2 +- libcfnet/file_stream.c | 52 ++++++++++++++++++++++++++++++++++++++---- libcfnet/file_stream.h | 7 +++++- libcfnet/protocol.c | 10 ++++---- libcfnet/protocol.h | 6 +++-- 6 files changed, 77 insertions(+), 15 deletions(-) diff --git a/cf-net/cf-net.c b/cf-net/cf-net.c index 448e8dd67a..1573847e15 100644 --- a/cf-net/cf-net.c +++ b/cf-net/cf-net.c @@ -56,6 +56,7 @@ typedef struct char *min_tls_version; char *allow_ciphers; char *use_protocol_version; + bool print_stats; } CFNetOptions; //******************************************************************* @@ -114,6 +115,7 @@ static const struct option OPTIONS[] = {"tls-version", required_argument, 0, 't'}, {"ciphers", required_argument, 0, 'c'}, {"protocol", required_argument, 0, 'p'}, + {"stats", no_argument, 0, 's'}, {NULL, 0, 0, '\0'} }; @@ -129,6 +131,7 @@ static const char *const HINTS[] = "Minimum TLS version to use", "TLS ciphers to use (comma-separated list)", "Specify CFEngine protocol to use. Possible values: 'classic', 'tls', 'cookie', 'filestream', 'latest' (default)", + "Print rsync performance statistics to stderr", NULL }; @@ -234,6 +237,7 @@ static void CFNetSetDefault(CFNetOptions *opts){ opts->min_tls_version = NULL; opts->allow_ciphers = NULL; opts->use_protocol_version = NULL; + opts->print_stats = false; } static void CFNetOptionsClear(CFNetOptions *opts) @@ -289,7 +293,7 @@ static int CFNetParse(int argc, char **argv, *hostnames = NULL; int c = 0; int start_index = 1; - const char *optstr = "+hMg:H:p:dvI"; // + means stop for non opt arg. :) + const char *optstr = "+hMg:H:p:sdvI"; // + means stop for non opt arg. :) while ((c = getopt_long(argc, argv, optstr, OPTIONS, &start_index)) != -1) { @@ -361,6 +365,11 @@ static int CFNetParse(int argc, char **argv, opts->use_protocol_version = xstrdup(optarg); break; } + case 's': + { + opts->print_stats = true; + break; + } default: { // printf("Default optarg = '%s', c = '%c' = %i\n", @@ -725,6 +734,7 @@ static int invalid_command(const char *cmd) typedef struct _GetFileData { const char *hostname; const char *use_protocol_version; + bool print_stats; char remote_file[PATH_MAX]; char local_file[PATH_MAX]; bool ret; @@ -741,7 +751,7 @@ static void *CFNetGetFile(void *arg) } data->ret = ProtocolStatGet(conn, data->remote_file, - data->local_file, 0644); + data->local_file, 0644, data->print_stats); if (!data->ret) { printf("Could not stat: '%s'\n", data->remote_file); @@ -858,6 +868,7 @@ static int CFNetGet(ARG_UNUSED CFNetOptions *opts, const char *hostname, char ** threads[i]->data = (GetFileData*) xcalloc(1, sizeof(GetFileData)); threads[i]->data->hostname = hostname; threads[i]->data->use_protocol_version = opts->use_protocol_version; + threads[i]->data->print_stats = opts->print_stats; if (n_threads > 1) { if (strstr(local_file, "%d") != NULL) diff --git a/libcfnet/client_code.c b/libcfnet/client_code.c index 0d1dff506b..dfd9803c16 100644 --- a/libcfnet/client_code.c +++ b/libcfnet/client_code.c @@ -796,7 +796,7 @@ bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info); if (ProtocolSupportsFileStream(version)) { - return FileStreamFetch(conn->conn_info->ssl, basis, dest, mode); + return FileStreamFetch(conn->conn_info->ssl, basis, dest, mode, false); } int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode); diff --git a/libcfnet/file_stream.c b/libcfnet/file_stream.c index 7b21aa6732..0e3ad99821 100644 --- a/libcfnet/file_stream.c +++ b/libcfnet/file_stream.c @@ -672,13 +672,18 @@ static rs_long_t GetSizeOfFile(FILE *file) * * @param conn The SSL connection object * @param filename The name of the basis file + * @param print_stats Whether or not to print performance statistics * @return true on success, otherwise false */ -static bool SendSignature(SSL *conn, const char *filename) +static bool SendSignature(SSL *conn, const char *filename, bool print_stats) { assert(conn != NULL); assert(filename != NULL); + /* Variables used for performance statistics */ + size_t bytes_in = 0; + size_t bytes_out = 0; + /* In this case, the input buffer does not need to be twice the message * size, because we can control how much we read into it */ char in_buf[PROTOCOL_MESSAGE_SIZE], out_buf[PROTOCOL_MESSAGE_SIZE]; @@ -783,6 +788,7 @@ static bool SendSignature(SSL *conn, const char *filename) bufs.next_in = in_buf; bufs.avail_in += n_bytes; + bytes_in += n_bytes; } /* Iterate job */ @@ -813,6 +819,7 @@ static bool SendSignature(SSL *conn, const char *filename) bufs.next_out = out_buf; bufs.avail_out = PROTOCOL_MESSAGE_SIZE; + bytes_out += present; } else if (res == RS_DONE) { @@ -829,6 +836,16 @@ static bool SendSignature(SSL *conn, const char *filename) fclose(file); rs_job_free(job); + const char *msg = + "Send signature statistics:\n" + " %zu bytes in (read from '%s')\n" + " %zu bytes out (sent to server)\n"; + Log(LOG_LEVEL_DEBUG, msg, bytes_in, filename, bytes_out); + if (print_stats) + { + fprintf(stderr, msg, bytes_in, filename, bytes_out); + } + return true; } @@ -839,15 +856,24 @@ static bool SendSignature(SSL *conn, const char *filename) * @param basis The name of basis file * @param dest The name of destination file * @param perms The desired file permissions of the destination file + * @param print_stats Whether or not to print performance statistics * @return true on success, otherwise false */ static bool RecvDelta( - SSL *conn, const char *basis, const char *dest, mode_t perms) + SSL *conn, + const char *basis, + const char *dest, + mode_t perms, + bool print_stats) { assert(conn != NULL); assert(basis != NULL); assert(dest != NULL); + /* Variables used for performance statistics */ + size_t bytes_in = 0; + size_t bytes_out = 0; + /* The input buffer has to be twice the message size, so that it can fit a * new message, as well as some tail data from the last job iteration */ char in_buf[PROTOCOL_MESSAGE_SIZE * 2], out_buf[PROTOCOL_MESSAGE_SIZE]; @@ -953,6 +979,7 @@ static bool RecvDelta( bufs.eof_in = eof ? 1 : 0; bufs.next_in = in_buf; bufs.avail_in += n_bytes; + bytes_in += n_bytes; } res = rs_job_iter(job, &bufs); @@ -998,6 +1025,7 @@ static bool RecvDelta( n_wrote_total += present; bufs.next_out = out_buf; bufs.avail_out = sizeof(out_buf); + bytes_out += present; } } while (res != RS_DONE); @@ -1012,11 +1040,25 @@ static bool RecvDelta( return false; } + const char *msg = + "Receive delta statistics:\n" + " %zu bytes in (received from server)\n" + " %zu bytes out (written to '%s')\n"; + Log(LOG_LEVEL_DEBUG, msg, bytes_in, bytes_out, dest); + if (print_stats) + { + fprintf(stderr, msg, bytes_in, bytes_out, dest); + } + return true; } bool FileStreamFetch( - SSL *conn, const char *basis, const char *dest, mode_t perms) + SSL *conn, + const char *basis, + const char *dest, + mode_t perms, + bool print_stats) { assert(conn != NULL); assert(basis != NULL); @@ -1032,7 +1074,7 @@ bool FileStreamFetch( Log(LOG_LEVEL_VERBOSE, "Computing- & sending signature of file '%s'...", basis); - if (!SendSignature(conn, basis)) + if (!SendSignature(conn, basis, print_stats)) { /* Error is already logged */ return false; @@ -1041,7 +1083,7 @@ bool FileStreamFetch( Log(LOG_LEVEL_VERBOSE, "Receiving delta & applying patch to file '%s'...", dest); - if (!RecvDelta(conn, basis, dest, perms)) + if (!RecvDelta(conn, basis, dest, perms, print_stats)) { /* Error is already logged */ return false; diff --git a/libcfnet/file_stream.h b/libcfnet/file_stream.h index 0252ef9fcb..8c7bd380d2 100644 --- a/libcfnet/file_stream.h +++ b/libcfnet/file_stream.h @@ -93,12 +93,17 @@ bool FileStreamServe(SSL *conn, const char *filename); * @param basis The name of the basis file * @param dest The name of the destination file * @param perms The desired permissions of the destination file + * @param print_stats Print performance statistics * @return true on success, otherwise false * * @note If the destination file is a symlink, this function fetches the * contents into the symlink target. */ bool FileStreamFetch( - SSL *conn, const char *basis, const char *dest, mode_t perms); + SSL *conn, + const char *basis, + const char *dest, + mode_t perms, + bool print_stats); #endif // FILE_STREAM_H diff --git a/libcfnet/protocol.c b/libcfnet/protocol.c index 54b0b7fa64..5f8e023ab3 100644 --- a/libcfnet/protocol.c +++ b/libcfnet/protocol.c @@ -92,7 +92,8 @@ Seq *ProtocolOpenDir(AgentConnection *conn, const char *path) } bool ProtocolGet(AgentConnection *conn, const char *remote_path, - const char *local_path, const uint32_t file_size, int perms) + const char *local_path, const uint32_t file_size, int perms, + bool print_stats) { assert(conn != NULL); assert(remote_path != NULL); @@ -128,7 +129,8 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path, if (ProtocolSupportsFileStream(version)) { /* Use file stream API if it is available */ - if (!FileStreamFetch(conn->conn_info->ssl, local_path, dest, perms)) + if (!FileStreamFetch(conn->conn_info->ssl, local_path, dest, perms, + print_stats)) { /* Error is already logged */ success = false; @@ -222,7 +224,7 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path, } bool ProtocolStatGet(AgentConnection *conn, const char *remote_path, - const char *local_path, int perms) + const char *local_path, int perms, bool print_stats) { assert(conn != NULL); assert(remote_path != NULL); @@ -237,7 +239,7 @@ bool ProtocolStatGet(AgentConnection *conn, const char *remote_path, return false; } - return ProtocolGet(conn, remote_path, local_path, sb.st_size, perms); + return ProtocolGet(conn, remote_path, local_path, sb.st_size, perms, print_stats); } bool ProtocolStat(AgentConnection *const conn, const char *const remote_path, diff --git a/libcfnet/protocol.h b/libcfnet/protocol.h index 23f2009708..b3c96b6b6d 100644 --- a/libcfnet/protocol.h +++ b/libcfnet/protocol.h @@ -84,6 +84,7 @@ Seq *ProtocolOpenDir(AgentConnection *conn, const char *path); * @param [in] local_path Path of received file * @param [in] file_size Size of file to get * @param [in] perms Permissions of local file + * @param [in] print_stats Print RSYNC performance statistics * @return True if file was successfully transferred, false otherwise * * Example (for printing each directory entry): @@ -104,7 +105,8 @@ Seq *ProtocolOpenDir(AgentConnection *conn, const char *path); * Translated to: GET /var/cfengine/masterfiles/update.cf */ bool ProtocolGet(AgentConnection *conn, const char *remote_path, - const char *local_path, const uint32_t file_size, int perms); + const char *local_path, const uint32_t file_size, int perms, + bool print_stats); /** @@ -114,7 +116,7 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path, * it. */ bool ProtocolStatGet(AgentConnection *conn, const char *remote_path, - const char *local_path, int perms); + const char *local_path, int perms, bool print_stats); /** * Receives statistics about a remote file.