diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml index 9eac099968..60132de1f0 100644 --- a/.github/workflows/acceptance_tests.yml +++ b/.github/workflows/acceptance_tests.yml @@ -5,13 +5,13 @@ on: jobs: acceptance_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: submodules: recursive - name: Install dependencies - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl libyaml-dev + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl libyaml-dev librsync-dev - name: Run autotools / configure run: ./autogen.sh --enable-debug - name: Compile and link (make) diff --git a/.github/workflows/asan_unit_tests.yml b/.github/workflows/asan_unit_tests.yml index 9f7ba2076e..e16a8a7256 100644 --- a/.github/workflows/asan_unit_tests.yml +++ b/.github/workflows/asan_unit_tests.yml @@ -5,16 +5,16 @@ on: jobs: asan_unit_tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: submodules: recursive - name: Install dependencies - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl librsync-dev - name: Run autotools / configure run: ./autogen.sh --enable-debug - name: Compile and link (make) run: make -j8 CFLAGS="-Werror -Wall -fsanitize=address" LDFLAGS="-fsanitize=address" - name: Run unit tests - run: make -C tests/unit CFLAGS="-fsanitize=address" LDFLAGS="-fsanitize=address" check + run: ASAN_OPTIONS=detect_odr_violation=0 make -C tests/unit CFLAGS="-fsanitize=address" LDFLAGS="-fsanitize=address" check diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4777259e8e..016276a748 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: - name: Install dependencies (C) if: ${{ matrix.language == 'cpp' }} - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl librsync-dev - name: Build (C) if: ${{ matrix.language == 'cpp' }} diff --git a/.github/workflows/job-static-check.yml b/.github/workflows/job-static-check.yml index 63d63116d7..e0d1481138 100644 --- a/.github/workflows/job-static-check.yml +++ b/.github/workflows/job-static-check.yml @@ -5,7 +5,7 @@ on: jobs: static_check: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout Core uses: actions/checkout@v3 @@ -41,7 +41,7 @@ jobs: sudo apt-get install -y dpkg-dev debhelper g++ libncurses6 pkg-config \ build-essential libpam0g-dev fakeroot gcc make autoconf buildah \ liblmdb-dev libacl1-dev libcurl4-openssl-dev libyaml-dev libxml2-dev \ - libssl-dev libpcre2-dev + libssl-dev libpcre2-dev librsync-dev - name: Run Autogen run: NO_CONFIGURE=1 PROJECT=community ./buildscripts/build-scripts/autogen diff --git a/.github/workflows/macos_unit_tests.yml b/.github/workflows/macos_unit_tests.yml index c29a1ce2e0..d79037f56c 100644 --- a/.github/workflows/macos_unit_tests.yml +++ b/.github/workflows/macos_unit_tests.yml @@ -11,15 +11,15 @@ jobs: with: submodules: recursive - name: Install dependencies - run: brew install lmdb automake openssl pcre2 autoconf libtool + run: brew install lmdb automake openssl pcre2 autoconf libtool librsync - name: Check tools run: command -v libtool && command -v automake && command -v autoconf - name: Check tools versions run: libtool -V && automake --version && autoconf --version - name: Run autotools / configure run: > - LDFLAGS="-L`brew --prefix lmdb`/lib -L`brew --prefix openssl`/lib -L`brew --prefix pcre2`/lib" - CPPFLAGS="-I`brew --prefix lmdb`/include -I`brew --prefix openssl`/include -I`brew --prefix pcre2`/include" + LDFLAGS="-L`brew --prefix lmdb`/lib -L`brew --prefix openssl`/lib -L`brew --prefix pcre2`/lib -L`brew --prefix librsync`/lib" + CPPFLAGS="-I`brew --prefix lmdb`/include -I`brew --prefix openssl`/include -I`brew --prefix pcre2`/include -I`brew --prefix librsync`/include" PATH="/opt/homebrew/opt/libtool/libexec/gnubin:$PATH" ./autogen.sh --enable-debug - name: Compile and link diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index adc97226e7..b5de17ec7c 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -6,13 +6,13 @@ on: jobs: unit_tests: name: Run shellcheck on shell scripts - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: submodules: recursive - name: Install dependencies - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl shellcheck + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl shellcheck librsync-dev - name: Run autotools / configure run: ./autogen.sh --enable-debug - name: Run shellcheck diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 932560b1ea..8115a911b6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -6,13 +6,13 @@ on: jobs: unit_tests: name: Run Unit Tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: submodules: recursive - name: Install dependencies - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl librsync-dev - name: Run autotools / configure run: ./autogen.sh --enable-debug - name: Compile and link (make) diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index db600e6556..6b0f780708 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -5,7 +5,7 @@ on: jobs: valgrind_tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 defaults: run: working-directory: core @@ -27,7 +27,7 @@ jobs: path: masterfiles submodules: recursive - name: Install dependencies - run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl libyaml-dev valgrind + run: sudo apt-get update -y && sudo apt-get install -y libssl-dev libpam0g-dev liblmdb-dev byacc curl libyaml-dev valgrind librsync-dev - name: Run autotools / configure run: ./autogen.sh --enable-debug --with-systemd-service - name: Compile and link (make) diff --git a/cf-agent/verify_files_utils.c b/cf-agent/verify_files_utils.c index 22ee22a18a..541c68bd89 100644 --- a/cf-agent/verify_files_utils.c +++ b/cf-agent/verify_files_utils.c @@ -1551,7 +1551,7 @@ bool CopyRegularFile(EvalContext *ctx, const char *source, const char *dest, con return false; } - if (!CopyRegularFileNet(source, ToChangesPath(new), + if (!CopyRegularFileNet(source, dest, ToChangesPath(new), sstat->st_size, attr->copy.encrypt, conn, sstat->st_mode)) { RecordFailure(ctx, pp, attr, "Failed to copy file '%s' from '%s'", diff --git a/cf-serverd/server_common.c b/cf-serverd/server_common.c index 6322a1a9d2..91ba9265b2 100644 --- a/cf-serverd/server_common.c +++ b/cf-serverd/server_common.c @@ -54,6 +54,7 @@ static const int CF_NOSIZE = -1; #include /* ThreadLock */ #include /* struct Stat */ #include /* GetUserID() */ +#include #include "server_access.h" @@ -402,6 +403,9 @@ static void FailedTransfer(ConnectionInfo *connection) void CfGetFile(ServerFileGetState *args) { + assert(args != NULL); + assert(args->conn != NULL); + int fd; off_t n_read, total = 0, sendlen = 0, count = 0; char sendbuffer[CF_BUFSIZE + 256], filename[CF_BUFSIZE - 128]; @@ -421,12 +425,23 @@ void CfGetFile(ServerFileGetState *args) if (!TransferRights(args->conn, filename, &sb)) { + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); + assert(ProtocolIsKnown(version)); + Log(LOG_LEVEL_INFO, "REFUSE access to file: %s", filename); + + if (ProtocolSupportsFileStream(version)) { + Log(LOG_LEVEL_VERBOSE, "REFUSAL to user='%s' of request: %s", + NULL_OR_EMPTY(args->conn->username) ? "?" : args->conn->username, + args->replyfile); + FileStreamRefuse(args->conn->conn_info->ssl); + return; + } + /* Else then handle older protocols */ + RefuseAccess(args->conn, args->replyfile); snprintf(sendbuffer, CF_BUFSIZE, "%s", CF_FAILEDSTR); - const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); - assert(ProtocolIsKnown(version)); if (ProtocolIsClassic(version)) { SendSocketStream(ConnectionInfoSocket(conn_info), sendbuffer, args->buf_size); @@ -440,6 +455,12 @@ void CfGetFile(ServerFileGetState *args) /* File transfer */ + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn_info); + if (ProtocolSupportsFileStream(version)) { + FileStreamServe(conn_info->ssl, filename); + return; + } + if ((fd = safe_open(filename, O_RDONLY)) == -1) { Log(LOG_LEVEL_ERR, "Open error of file '%s'. (open: %s)", diff --git a/configure.ac b/configure.ac index 96609095da..ab9699009b 100644 --- a/configure.ac +++ b/configure.ac @@ -521,6 +521,20 @@ CF3_WITH_LIBRARY(pcre2, [ )] ) +dnl librsync + +AC_ARG_WITH([librsync], [AS_HELP_STRING([--with-librsync[[=PATH]]], [Specify librsync path])], [], [with_librsync=yes]) + +if test "x$with_librsync" = "xno"; then + AC_MSG_ERROR([librsync is required]) +fi + +CF3_WITH_LIBRARY(librsync, [ + AC_CHECK_HEADERS([librsync.h], [], AC_MSG_ERROR(Cannot find librsync)) + AC_CHECK_LIB(rsync, rs_job_iter, [], [AC_MSG_ERROR(Cannot find librsync)]) + ] +) + dnl defined for libntech AC_DEFINE(WITH_PCRE2, 1, [Define if PCRE2 is being used]) @@ -1317,10 +1331,10 @@ dnl ###################################################################### dnl Collect all the options dnl ###################################################################### -CORE_CPPFLAGS="$LMDB_CPPFLAGS $TOKYOCABINET_CPPFLAGS $QDBM_CPPFLAGS $PCRE2_CPPFLAGS $OPENSSL_CPPFLAGS $SQLITE3_CPPFLAGS $LIBACL_CPPFLAGS $LIBCURL_CPPFLAGS $LIBYAML_CPPFLAGS $POSTGRESQL_CPPFLAGS $MYSQL_CPPFLAGS $LIBXML2_CPPFLAGS $CPPFLAGS $CFECOMPAT_CPPFLAGS" -CORE_CFLAGS="$LMDB_CFLAGS $TOKYOCABINET_CFLAGS $QDBM_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $SQLITE3_CFLAGS $LIBACL_CFLAGS $LIBCURL_CFLAGS $LIBYAML_CFLAGS $POSTGRESQL_CFLAGS $MYSQL_CFLAGS $LIBXML2_CFLAGS $CFLAGS" -CORE_LDFLAGS="$LMDB_LDFLAGS $TOKYOCABINET_LDFLAGS $QDBM_LDFLAGS $PCRE2_LDFLAGS $OPENSSL_LDFLAGS $SQLITE3_LDFLAGS $LIBACL_LDFLAGS $LIBCURL_LDFLAGS $LIBYAML_LDFLAGS $POSTGRESQL_LDFLAGS $MYSQL_LDFLAGS $LIBXML2_LDFLAGS $LDFLAGS" -CORE_LIBS="$LMDB_LIBS $TOKYOCABINET_LIBS $QDBM_LIBS $PCRE2_LIBS $OPENSSL_LIBS $SQLITE3_LIBS $LIBACL_LIBS $LIBCURL_LIBS $LIBYAML_LIBS $POSTGRESQL_LIBS $MYSQL_LIBS $LIBXML2_LIBS $LIBS" +CORE_CPPFLAGS="$LMDB_CPPFLAGS $TOKYOCABINET_CPPFLAGS $QDBM_CPPFLAGS $PCRE2_CPPFLAGS $OPENSSL_CPPFLAGS $SQLITE3_CPPFLAGS $LIBACL_CPPFLAGS $LIBCURL_CPPFLAGS $LIBRSYNC_CPPFLAGS $LIBYAML_CPPFLAGS $POSTGRESQL_CPPFLAGS $MYSQL_CPPFLAGS $LIBXML2_CPPFLAGS $CPPFLAGS $CFECOMPAT_CPPFLAGS" +CORE_CFLAGS="$LMDB_CFLAGS $TOKYOCABINET_CFLAGS $QDBM_CFLAGS $PCRE2_CFLAGS $OPENSSL_CFLAGS $SQLITE3_CFLAGS $LIBACL_CFLAGS $LIBCURL_CFLAGS $LIBRSYNC_CFLAGS $LIBYAML_CFLAGS $POSTGRESQL_CFLAGS $MYSQL_CFLAGS $LIBXML2_CFLAGS $CFLAGS" +CORE_LDFLAGS="$LMDB_LDFLAGS $TOKYOCABINET_LDFLAGS $QDBM_LDFLAGS $PCRE2_LDFLAGS $OPENSSL_LDFLAGS $SQLITE3_LDFLAGS $LIBACL_LDFLAGS $LIBCURL_LDFLAGS $LIBRSYNC_LDFLAGS $LIBYAML_LDFLAGS $POSTGRESQL_LDFLAGS $MYSQL_LDFLAGS $LIBXML2_LDFLAGS $LDFLAGS" +CORE_LIBS="$LMDB_LIBS $TOKYOCABINET_LIBS $QDBM_LIBS $PCRE2_LIBS $OPENSSL_LIBS $SQLITE3_LIBS $LIBACL_LIBS $LIBCURL_LIBS $LIBRSYNC_LIBS $LIBYAML_LIBS $POSTGRESQL_LIBS $MYSQL_LIBS $LIBXML2_LIBS $LIBS" dnl ###################################################################### dnl Make them available to subprojects. diff --git a/libcfnet/Makefile.am b/libcfnet/Makefile.am index 3e5cff8619..36ad3639f3 100644 --- a/libcfnet/Makefile.am +++ b/libcfnet/Makefile.am @@ -37,6 +37,7 @@ libcfnet_la_SOURCES = \ communication.c communication.h \ connection_info.c connection_info.h \ conn_cache.c conn_cache.h \ + file_stream.c file_stream.h \ key.c key.h \ misc.c \ net.c net.h \ diff --git a/libcfnet/README.md b/libcfnet/README.md index 568b920ef1..77fbf4b537 100644 --- a/libcfnet/README.md +++ b/libcfnet/README.md @@ -15,6 +15,7 @@ Names of protocol versions: 1. `"classic"` - Legacy, pre-TLS, protocol. Not enabled or allowed by default. 2. `"tls"` - TLS Protocol using OpenSSL. Encrypted and 2-way authentication. 3. `"cookie"` - TLS Protocol with cookie command for duplicate host detection. +3. `"filestream"` - Introduces a new streaming API for get file request (powered by librsync). Wanted protocol version can be specified from policy: @@ -59,3 +60,30 @@ Both server and client will then set `conn_info->protocol` to `2`, and use proto There is currently no way to require a specific version number (only allow / disallow version 1). This is because version 2 and 3 are practically identical. Downgrade from version 3 to 2 happens seamlessly, but crucially, it doesn't downgrade to version 1 inside the TLS code. + +## Commands + +### `GET ` (protocol v4) + +The following is a description of the `GET ` command, modified in +protocol version v4 (introduced in CFEngine 3.25). + +The initial motivation for creating a new protocol version `filestream` was +due to a race condition found in the `GET ` request. It relied on the +file size aquired by `STAT `. However, if the file size increased +between the two requests, the client would think that the remaining data at the +offset of the aquired file size is a new protocol header. This situation would lead +to undefined behaviour. Hence, we needed a new protocol to send files. Instead +of reinventing the wheel, we decided to use librsync which utilizes the RSYNC +protocol to transmit files. + +The server implementation is found in function +[CfGet()](../cf-serverd/server_common.c). Client impementations are found in +[CopyRegularFileNet()](client_code.c) and [ProtocolGet()](protocol.c) + +Similar to before, the client issues a `GET ` request. However, +instead of continuing to execute the old protocol, the client immediately calls +`FileStreamFetch()` from the "File Stream API". Upon receiving such a request, +the server calls either `FileStreamRefuse()` (to refuse the request) or +`FileStreamServe()` (to comply with the request). The internal workings of the +File Stream API is well explained in [file_stream.h](file_stream.h). diff --git a/libcfnet/client_code.c b/libcfnet/client_code.c index 7dccb88324..0d1dff506b 100644 --- a/libcfnet/client_code.c +++ b/libcfnet/client_code.c @@ -45,6 +45,7 @@ #include /* ProgrammingError */ #include /* PRINTSIZE */ #include /* LastSaw */ +#include #define CFENGINE_SERVICE "cfengine" @@ -749,9 +750,11 @@ static void FlushFileStream(int sd, int toget) /* TODO finalise socket or TLS session in all cases that this function fails * and the transaction protocol is out of sync. */ -bool CopyRegularFileNet(const char *source, const char *dest, off_t size, +bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size, bool encrypt, AgentConnection *conn, mode_t mode) { + assert(conn != NULL); + char *buf, workbuf[CF_BUFSIZE], cfchangedstr[265]; const int buf_size = 2048; @@ -774,23 +777,12 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size, unlink(dest); /* To avoid link attacks */ - int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode); - if (dd == -1) - { - Log(LOG_LEVEL_ERR, - "Copy from server '%s' to destination '%s' failed (open: %s)", - conn->this_server, dest, GetErrorStr()); - unlink(dest); - return false; - } - workbuf[0] = '\0'; int tosend = snprintf(workbuf, CF_BUFSIZE, "GET %d %s", buf_size, source); if (tosend <= 0 || tosend >= CF_BUFSIZE) { Log(LOG_LEVEL_ERR, "Failed to compose GET command for file %s", source); - close(dd); return false; } @@ -799,7 +791,21 @@ bool CopyRegularFileNet(const char *source, const char *dest, off_t size, if (SendTransaction(conn->conn_info, workbuf, tosend, CF_DONE) == -1) { Log(LOG_LEVEL_ERR, "Couldn't send GET command"); - close(dd); + return false; + } + + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info); + if (ProtocolSupportsFileStream(version)) { + return FileStreamFetch(conn->conn_info->ssl, basis, dest, mode); + } + + int dd = safe_open_create_perms(dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, mode); + if (dd == -1) + { + Log(LOG_LEVEL_ERR, + "Copy from server '%s' to destination '%s' failed (open: %s)", + conn->this_server, dest, GetErrorStr()); + unlink(dest); return false; } diff --git a/libcfnet/client_code.h b/libcfnet/client_code.h index 4aaaa5ff4c..8f9a7cef57 100644 --- a/libcfnet/client_code.h +++ b/libcfnet/client_code.h @@ -47,7 +47,7 @@ AgentConnection *ServerConnection(const char *server, const char *port, const Rl void DisconnectServer(AgentConnection *conn); bool CompareHashNet(const char *file1, const char *file2, bool encrypt, AgentConnection *conn); -bool CopyRegularFileNet(const char *source, const char *dest, off_t size, +bool CopyRegularFileNet(const char *source, const char *basis, const char *dest, off_t size, bool encrypt, AgentConnection *conn, mode_t mode); Item *RemoteDirList(const char *dirname, bool encrypt, AgentConnection *conn); diff --git a/libcfnet/file_stream.c b/libcfnet/file_stream.c new file mode 100644 index 0000000000..b840e8edca --- /dev/null +++ b/libcfnet/file_stream.c @@ -0,0 +1,1050 @@ +/* + Copyright 2024 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#include + +#include "file_stream.h" + +#include +#include +#include +#include +#include +#include + +/*********************************************************/ +/* Network protocol */ +/*********************************************************/ + +/** + * @brief Simple network protocol on top of SSL/TCP. Used for client-server + * communication during file stream. + * + * @details Header format: + * +----------+----------+----------+----------+ + * | SDU Len. | Reserved | EOF Flag | ERR Flag | + * +----------+----------+----------+----------+ + * | 12 bits | 2 bits | 1 bit | 1 bit | + * +----------+----------+----------+----------+ + * + * The header consists of 16 bits and the fields are defined as follows: + * SDU Length Length of the SDU (i.e. payload) encapsulated within + * this datagram. + * Reserved 2 bits reserved for future use. + * End-of-File flag Signals whether or not the receiver should expect to + * receive more datagrams. + * Error flag Signals that the transmission must be canceled due to + * unexpected error. + * + * @note If the End-of-File flag is set, there may still be data to process in + * in the payload. If the Error flag is set, there may be an error + * message in the payload. + */ +#define PROTOCOL_HEADER_SIZE 2 + +/** + * @note The TLS Generic API requires that the message length is less than + * CF_BUFSIZE. Furthermore, the protocol can only handle up to 4095 + * Bytes, because it's the largest unsigned integer you can represent + * with 12 bits (2^12 - 1 = 4095). + */ +#define PROTOCOL_MESSAGE_SIZE MIN(CF_BUFSIZE - 1, 4095) + +/** + * @brief Send a message using the file stream protocol + * @warning You probably want to use ProtocolSendMessage() or + * ProtocolSendError() instead + * + * @param conn The SSL connection object + * @param msg The message to send + * @param len The length of the message to send (must be less or equal to + * PROTOCOL_MESSAGE_SIZE Bytes) + * @param eof Set to true if this is the last message in a transaction, + * otherwise false + * @param err Set to true if transaction must be canceled (e.g., due to an + * unexpected error), otherwise false + * @note If the err parameter is set to true, the expected return value is + * still true. + * @return true on success, otherwise false + */ +static bool __ProtocolSendMessage( + SSL *conn, const char *msg, size_t len, bool eof, bool err) +{ + assert(conn != NULL); + assert(msg != NULL || len == 0); + assert(len <= PROTOCOL_MESSAGE_SIZE); + + /* Set message length */ + assert(sizeof(len) >= 3); /* It's probably guaranteed, but let's make sure + * to avoid potentially nasty surprises */ + uint16_t header = len << 4; + + /* Set Error flag */ + if (err) + { + header |= (1 << 0); + } + + /* Set End-of-File flag */ + if (eof) + { + header |= (1 << 1); + } + + /* Send header */ + header = htons(header); + int ret = TLSSend(conn, (char *) &header, PROTOCOL_HEADER_SIZE); + if (ret != PROTOCOL_HEADER_SIZE) + { + Log(LOG_LEVEL_ERR, + "Failed to send message header during file stream: " + "Expected to send %d bytes, but sent %d bytes", + ret, + PROTOCOL_HEADER_SIZE); + return false; + } + + if (len > 0) + { + /* Send payload */ + ret = TLSSend(conn, msg, len); + if (ret != (int) len) + { + Log(LOG_LEVEL_ERR, + "Failed to send message payload during file stream: " + "Expected to send %d bytes, but sent %zu bytes", + ret, + len); + return false; + } + } + + return true; +} + +/** + * @brief Send a message using the file stream protocol + * + * @param conn The SSL connection object + * @param msg The message to send + * @param len The length of the message to send (must be less or equal to + * PROTOCOL_MESSAGE_SIZE Bytes) + * @param eof Set to true if this is the last message in a transaction, + * otherwise false + * @return true on success, otherwise false + */ +static inline bool ProtocolSendMessage( + SSL *conn, const char *msg, size_t len, bool eof) +{ + assert(conn != NULL); + assert(msg != NULL || len == 0); + + return __ProtocolSendMessage(conn, msg, len, eof, false); +} + +/** + * @brief Receive a message using the file stream protocol + * + * @param conn The SSL connection object + * @param msg The message receive buffer (must be PROTOCOL_MESSAGE_SIZE bytes + * large) + * @param len The length of the reveived message + * @param eof Is set to true if this was the last message in the transaction + * @return true on success, otherwise false + * + * @note ProtocolRecvMessage fails if the communication is broken or if we + * received an error from the remote host. In both cases, we should not + * try to flush the stream. + */ +static bool ProtocolRecvMessage(SSL *conn, char *msg, size_t *len, bool *eof) +{ + assert(conn != NULL); + assert(msg != NULL); + assert(len != NULL); + assert(eof != NULL); + + /* TLSRecv() expects a buffer this size */ + char recv_buffer[CF_BUFSIZE]; + + /* Receive header */ + int ret = TLSRecv(conn, recv_buffer, PROTOCOL_HEADER_SIZE); + if (ret != PROTOCOL_HEADER_SIZE) + { + Log(LOG_LEVEL_ERR, + "Failed to receive message header during file stream: " + "Expected to receive %d bytes, but received %d bytes", + ret, + PROTOCOL_HEADER_SIZE); + return false; + } + + /* Why not receive the bytes directly into header in the TLSRecv()? + * Because it actually writes a NUL-Byte after the requested bytes which + * would cause memory violations. */ + uint16_t header; + memcpy(&header, recv_buffer, PROTOCOL_HEADER_SIZE); + header = ntohs(header); + + /* Extract Error flag */ + bool err = header & (1 << 0); + + /* Extract End-of-File flag */ + *eof = header & (1 << 1); + + /* Extract message length */ + assert(sizeof(*len) >= 2); /* It's probably guaranteed, but let's make + * sure to avoid potentially nasty surprises */ + *len = header >> 4; + + /* Read payload */ + if (*len > 0) + { + /* The TLSRecv() function's doc string says that the returned value + * may be less than the requested length if the other side completed a + * send with less bytes. I take it that this means that there is no + * short reads/recvs. Futhermore, TLSSend() says that its return value + * is always equal to the requested length as long as TLS is setup + * correctly. I take it that the same is true for TLSRecv(). Hence, we + * will interpret a shorter read than what we expect as an error. */ + ret = TLSRecv(conn, recv_buffer, *len); + if (ret != *len) + { + Log(LOG_LEVEL_ERR, + "Failed to receive message payload during file stream: " + "Expected to receive %d bytes, but received %zu bytes", + ret, + *len); + return false; + } + memcpy(msg, recv_buffer, *len); + + if (err) + { + /* If the error flag is set, then the payload contains an error + * message in the form of a NUL-byte terminated string. */ + Log(LOG_LEVEL_ERR, "Remote file stream error: %s", msg); + } + } + + return !err; +} + +/** + * @brief Flush the file stream + * + * It's used to prevent the remote host from blocking while sending the + * remaining data after we have experienced an unexpected error and need to + * abort the file stream. Once the stream has been successfully flushed, the + * remote host will be ready to receive our error message. + * + * @param conn The SSL connection object + * @return true on success, otherwise false + */ +static bool ProtocolFlushStream(SSL *conn) +{ + assert(conn != NULL); + + char msg[PROTOCOL_MESSAGE_SIZE]; + size_t len; + bool eof; + while (ProtocolRecvMessage(conn, msg, &len, &eof)) + { + if (eof) + { + return true; + } + } + + Log(LOG_LEVEL_ERR, "Remote file stream error: %s", msg); + return false; +} + +/** + * @brief Send an error message using the file stream protocol + * + * @param conn The SSL connection object + * @param flush Whether or not to flush the stream (see ProtocolFlushStream()) + * @param fmt The format string + * @param ... The format string arguments + * @return true on success, otherwise false + */ +static bool ProtocolSendError(SSL *conn, bool flush, const char *fmt, ...) + FUNC_ATTR_PRINTF(3, 4); + +static bool ProtocolSendError(SSL *conn, bool flush, const char *fmt, ...) +{ + assert(conn != NULL); + assert(fmt != NULL); + + va_list ap; + char msg[PROTOCOL_MESSAGE_SIZE]; + + va_start(ap, fmt); + int len = vsnprintf(msg, PROTOCOL_MESSAGE_SIZE, fmt, ap); + va_end(ap); + + assert(len >= 0); /* Let's make sure we detect this in debug builds */ + if (len < 0) + { + Log(LOG_LEVEL_ERR, + "Failed to format error message during file stream"); + len = 0; /* We still want to send the header */ + } + else if (len >= PROTOCOL_MESSAGE_SIZE) + { + Log(LOG_LEVEL_WARNING, + "Error message truncated during file stream: " + "Message is %d bytes, but maximum message size is %d bytes", + len, + PROTOCOL_MESSAGE_SIZE); + /* Add dots to indicate message truncation. We don't need the + * terminating NULL-byte in the buffer. Furthermore, TLSRecv() will + * append one, upon receiving the message */ + msg[PROTOCOL_MESSAGE_SIZE - 1] = '.'; + msg[PROTOCOL_MESSAGE_SIZE - 2] = '.'; + msg[PROTOCOL_MESSAGE_SIZE - 3] = '.'; + len = PROTOCOL_MESSAGE_SIZE; + } + + if (flush) + { + ProtocolFlushStream(conn); + } + + return __ProtocolSendMessage(conn, msg, (size_t) len, false, true); +} + +/*********************************************************/ +/* Server specific */ +/*********************************************************/ + +#define ERROR_MSG_UNSPECIFIED_SERVER_REFUSAL "Unspecified server refusal" +#define ERROR_MSG_INTERNAL_SERVER_ERROR "Internal server error" + +bool FileStreamRefuse(SSL *conn) +{ + return ProtocolSendError( + conn, false, ERROR_MSG_UNSPECIFIED_SERVER_REFUSAL); +} + +/** + * @brief Receive and load signature into memory + * + * @param conn The SSL connection object + * @param sig The signature of the outdated file + * @return true on success, otherwise false + */ +static bool RecvSignature(SSL *conn, rs_signature_t **sig) +{ + assert(conn != NULL); + assert(sig != NULL); + + /* The input buffer has to be twice the message size, so that it can fit a + * new message, as well as some tail data from the last job iteration */ + char in_buf[PROTOCOL_MESSAGE_SIZE * 2]; + + /* Start a job for loading a signature into memory */ + rs_job_t *job = rs_loadsig_begin(sig); + if (job == NULL) + { + Log(LOG_LEVEL_ERR, "Failed to begin job for loading signature"); + ProtocolSendError(conn, true, ERROR_MSG_INTERNAL_SERVER_ERROR); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + + rs_result res; + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in > PROTOCOL_MESSAGE_SIZE) + { + /* The job requires more data, but we cannot fit another + * message into the input buffer */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to receive file stream signature: " + "%zu of %zu bytes available, but %d bytes is required to fit another message", + sizeof(in_buf) - bufs.avail_in, + sizeof(in_buf), + PROTOCOL_MESSAGE_SIZE); + ProtocolSendError(conn, true, ERROR_MSG_INTERNAL_SERVER_ERROR); + + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes; + bool eof; + if (!ProtocolRecvMessage( + conn, in_buf + bufs.avail_in, &n_bytes, &eof)) + { + /* Error is already logged */ + rs_job_free(job); + return false; + } + + bufs.eof_in = eof ? 1 : 0; + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + Log(LOG_LEVEL_ERR, + "Failed to iterate job for loading signature: %s", + rs_strerror(res)); + ProtocolSendError( + conn, bufs.eof_in == 0, ERROR_MSG_INTERNAL_SERVER_ERROR); + rs_job_free(job); + return false; + } + } while (res != RS_DONE); + + rs_job_free(job); + + return true; +} + +/** + * @brief Compute and send delta based on the source file and the signature of + * the basis file + * + * @param conn The SSL connection object + * @param sig The signature of the basis file + * @param filename The name of the source file + * @return true on success, otherwise false + */ +static bool SendDelta(SSL *conn, rs_signature_t *sig, const char *filename) +{ + assert(conn != NULL); + assert(sig != NULL); + assert(filename != NULL); + + /* In this case, the input buffer does not need to be twice the message + * size, because we can control how much we read into it */ + char in_buf[PROTOCOL_MESSAGE_SIZE], out_buf[PROTOCOL_MESSAGE_SIZE]; + + /* Open source file */ + FILE *file = safe_fopen(filename, "rb"); + if (file == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open the source file '%s' for computing delta during file stream: %s", + filename, + GetErrorStr()); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + return false; + } + + /* Build hash table */ + rs_result res = rs_build_hash_table(sig); + if (res != RS_DONE) + { + Log(LOG_LEVEL_ERR, "Failed to build hash table: %s", rs_strerror(res)); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + fclose(file); + return false; + } + + /* Start generating delta */ + rs_job_t *job = rs_delta_begin(sig); + if (job == NULL) + { + Log(LOG_LEVEL_ERR, "Failed to begin job for generating delta"); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + fclose(file); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = + PROTOCOL_MESSAGE_SIZE; /* We cannot send more using the protocol */ + + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in >= sizeof(in_buf)) + { + /* The job requires more data, but the input buffer is full */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to compute delta: " + "%zu of %zu bytes available", + sizeof(in_buf) - bufs.avail_in, + sizeof(in_buf)); + ProtocolSendError( + conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes = fread( + in_buf + bufs.avail_in, + 1 /* Byte */, + sizeof(in_buf) - bufs.avail_in, + file); + if (n_bytes == 0) + { + if (ferror(file)) + { + Log(LOG_LEVEL_ERR, + "Failed to read the source file '%s' during file stream: %s", + filename, + GetErrorStr()); + ProtocolSendError( + conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* End-of-File reached */ + bufs.eof_in = feof(file); + assert(bufs.eof_in != 0); + } + + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + Log(LOG_LEVEL_ERR, + "Failed to iterate job for generating delta: %s", + rs_strerror(res)); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_SERVER_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + assert(present <= PROTOCOL_MESSAGE_SIZE); + if (!ProtocolSendMessage(conn, out_buf, present, res == RS_DONE)) + { + fclose(file); + rs_job_free(job); + return false; + } + + bufs.next_out = out_buf; + bufs.avail_out = PROTOCOL_MESSAGE_SIZE; + } + else if (res == RS_DONE) + { + /* Send End-of-File */ + if (!ProtocolSendMessage(conn, NULL, 0, 1)) + { + fclose(file); + rs_job_free(job); + return false; + } + } + } while (res != RS_DONE); + + fclose(file); + rs_job_free(job); + + return true; +} + +bool FileStreamServe(SSL *conn, const char *filename) +{ + assert(conn != NULL); + assert(filename != NULL); + + Log(LOG_LEVEL_VERBOSE, + "Receiving- & loading signature into memory for file '%s'...", + filename); + rs_signature_t *sig; + if (!RecvSignature(conn, &sig)) + { + /* Error is already logged */ + return false; + } + + Log(LOG_LEVEL_VERBOSE, + "Computing- & sending delta for file '%s'...", + filename); + if (!SendDelta(conn, sig, filename)) + { + /* Error is already logged */ + rs_free_sumset(sig); + return false; + } + + rs_free_sumset(sig); + return true; +} + +/*********************************************************/ +/* Client specific */ +/*********************************************************/ + +#define ERROR_MSG_INTERNAL_CLIENT_ERROR "Internal client error" + + +/** + * @brief Get the size of a file + * + * @param file The file pointer + * @return the file size or -1 on error + * @note -1 on error is quite handy, because rs_sig_args() interprets it as + * unknown file size + */ +static rs_long_t GetSizeOfFile(FILE *file) +{ + /* librsync has rs_file_size() as a utility/convenience function which + * basically does the exact same thing. However, it is not available in + * versions prior to librsync-2.1.0 which caused problems for some of the + * older platforms we support. Hence, we provide our own implementation in + * order to have some backwards compatibility in terms of librsync + * versions provided by package managers. */ + + int fd = fileno(file); + if (fd == -1) + { + return -1; + } + + struct stat sb; + if (fstat(fd, &sb) == -1) + { + return -1; + } + + return (rs_long_t) (S_ISREG(sb.st_mode) ? sb.st_size : -1); +} + +/** + * @brief Compute and send a signature of the basis file + * + * @param conn The SSL connection object + * @param filename The name of the basis file + * @return true on success, otherwise false + */ +static bool SendSignature(SSL *conn, const char *filename) +{ + assert(conn != NULL); + assert(filename != NULL); + + /* In this case, the input buffer does not need to be twice the message + * size, because we can control how much we read into it */ + char in_buf[PROTOCOL_MESSAGE_SIZE], out_buf[PROTOCOL_MESSAGE_SIZE]; + + /* Open basis file */ + FILE *file = safe_fopen(filename, "rb"); + if (file == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open the basis file '%s' for computing delta during file stream: %s", + filename, + GetErrorStr()); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + return false; + } + + /* Get file size */ + rs_long_t fsize = GetSizeOfFile(file); + + /* Get recommended arguments */ + rs_magic_number sig_magic = 0; + size_t block_len = 0, strong_len = 0; + rs_result res = rs_sig_args(fsize, &sig_magic, &block_len, &strong_len); + if (res != RS_DONE) + { + Log(LOG_LEVEL_ERR, + "Failed to get recommended signature arguments: %s", + rs_strerror(res)); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + fclose(file); + return false; + } + + /* Start generating signature */ + rs_job_t *job = rs_sig_begin(block_len, strong_len, sig_magic); + if (job == NULL) + { + Log(LOG_LEVEL_ERR, "Failed to begin job for generating signature"); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + fclose(file); + return false; + } + + /* Setup buffers */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = + PROTOCOL_MESSAGE_SIZE; /* We cannot send more using the protocol */ + + do + { + if (bufs.eof_in == 0) + { + if (bufs.avail_in >= sizeof(in_buf)) + { + /* The job requires more data, but the input buffer is full */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to compute delta: " + "%zu of %zu bytes available", + sizeof(in_buf) - bufs.avail_in, + sizeof(in_buf)); + ProtocolSendError( + conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + /* Fill input buffer */ + size_t n_bytes = fread( + in_buf + bufs.avail_in, + 1 /* Byte */, + sizeof(in_buf) - bufs.avail_in, + file); + if (n_bytes == 0) + { + if (ferror(file)) + { + Log(LOG_LEVEL_ERR, + "Failed to read the basis file '%s' during file stream: %s", + filename, + GetErrorStr()); + ProtocolSendError( + conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* End-of-File reached */ + bufs.eof_in = feof(file); + assert(bufs.eof_in != 0); + } + + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + /* Iterate job */ + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + Log(LOG_LEVEL_ERR, + "Failed to iterate job for generating signature: %s", + rs_strerror(res)); + ProtocolSendError(conn, false, ERROR_MSG_INTERNAL_CLIENT_ERROR); + + fclose(file); + rs_job_free(job); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + assert(present <= PROTOCOL_MESSAGE_SIZE); + if (!ProtocolSendMessage(conn, out_buf, present, res == RS_DONE)) + { + fclose(file); + rs_job_free(job); + return false; + } + + bufs.next_out = out_buf; + bufs.avail_out = PROTOCOL_MESSAGE_SIZE; + } + else if (res == RS_DONE) + { + /* Send End-of-File */ + if (!ProtocolSendMessage(conn, NULL, 0, 1)) + { + fclose(file); + rs_job_free(job); + return false; + } + } + } while (res != RS_DONE); + + fclose(file); + rs_job_free(job); + + return true; +} + +/** + * @brief Receive delta and apply patch to the outdated copy of the file + * + * @param conn The SSL connection object + * @param basis The name of basis file + * @param dest The name of destination file + * @param perms The desired file permissions of the destination file + * @return true on success, otherwise false + */ +static bool RecvDelta( + SSL *conn, const char *basis, const char *dest, mode_t perms) +{ + assert(conn != NULL); + assert(basis != NULL); + assert(dest != NULL); + + /* The input buffer has to be twice the message size, so that it can fit a + * new message, as well as some tail data from the last job iteration */ + char in_buf[PROTOCOL_MESSAGE_SIZE * 2], out_buf[PROTOCOL_MESSAGE_SIZE]; + + /* Open/create the destination file */ + int new = safe_open_create_perms( + dest, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL | O_BINARY, perms); + if (new == -1) + { + Log(LOG_LEVEL_ERR, + "Failed to open/create destination file '%s': %s", + dest, + GetErrorStr()); + + /* At this point the server will not be expecting any more messages + * from the client as far as the File Stream API is concerned. Hence, + * we don't have to send error message. Instead we just flush the + * stream. */ + ProtocolFlushStream(conn); + return false; + } + + /* Open the basis file */ + FILE *old = safe_fopen(basis, "rb"); + if (old == NULL) + { + Log(LOG_LEVEL_ERR, + "Failed to open basis file '%s': %s", + basis, + GetErrorStr()); + ProtocolFlushStream(conn); + close(new); + unlink(dest); + return false; + } + + /* Start a job for patching destination file */ + rs_job_t *job = rs_patch_begin(rs_file_copy_cb, old); + if (job == NULL) + { + Log(LOG_LEVEL_ERR, "Failed to begin job for patching"); + ProtocolFlushStream(conn); + close(new); + fclose(old); + unlink(dest); + return false; + } + + /* Setup buffers for the job */ + rs_buffers_t bufs = {0}; + bufs.next_out = out_buf; + bufs.avail_out = sizeof(out_buf); + + /* Sparse file specific */ + bool last_write_made_hole = false; + size_t n_wrote_total = 0; + + rs_result res; + do + { + /* Fill input buffers */ + if (bufs.eof_in == 0) + { + if (bufs.avail_in > PROTOCOL_MESSAGE_SIZE) + { + /* The job requires more data, but we cannot fit another + * message into the input buffer */ + Log(LOG_LEVEL_ERR, + "Insufficient buffer capacity to receive file stream delta: " + "%zu of %zu bytes available, but %d bytes is required to fit another message", + sizeof(in_buf) - bufs.avail_in, + sizeof(in_buf), + PROTOCOL_MESSAGE_SIZE); + ProtocolFlushStream(conn); + + close(new); + fclose(old); + rs_job_free(job); + unlink(dest); + return false; + } + + if (bufs.avail_in > 0) + { + /* Move leftover tail data to the front of the buffer */ + memmove(in_buf, bufs.next_in, bufs.avail_in); + } + + size_t n_bytes; + bool eof; + if (!ProtocolRecvMessage( + conn, in_buf + bufs.avail_in, &n_bytes, &eof)) + { + /* Error is already logged */ + close(new); + fclose(old); + rs_job_free(job); + unlink(dest); + return false; + } + + bufs.eof_in = eof ? 1 : 0; + bufs.next_in = in_buf; + bufs.avail_in += n_bytes; + } + + res = rs_job_iter(job, &bufs); + if (res != RS_DONE && res != RS_BLOCKED) + { + Log(LOG_LEVEL_ERR, + "Failed to iterate job for patching: %s", + rs_strerror(res)); + if (bufs.eof_in == 0) + { + ProtocolFlushStream(conn); + } + + close(new); + fclose(old); + rs_job_free(job); + unlink(dest); + return false; + } + + /* Drain output buffer, if there is data */ + size_t present = bufs.next_out - out_buf; + if (present > 0) + { + if (!FileSparseWrite(new, out_buf, present, &last_write_made_hole)) + { + Log(LOG_LEVEL_ERR, + "Failed to write to destination file '%s' during file stream: %s", + dest, + GetErrorStr()); + if (bufs.eof_in == 0) + { + ProtocolFlushStream(conn); + } + + close(new); + fclose(old); + rs_job_free(job); + unlink(dest); + return false; + } + + n_wrote_total += present; + bufs.next_out = out_buf; + bufs.avail_out = sizeof(out_buf); + } + } while (res != RS_DONE); + + fclose(old); + rs_job_free(job); + + if (!FileSparseClose( + new, dest, false, n_wrote_total, last_write_made_hole)) + { + /* Error is already logged */ + unlink(dest); + return false; + } + + return true; +} + +bool FileStreamFetch( + SSL *conn, const char *basis, const char *dest, mode_t perms) +{ + assert(conn != NULL); + assert(basis != NULL); + assert(dest != NULL); + + /* Let's make sure the basis file exists */ + FILE *file = safe_fopen_create_perms(basis, "wb", perms); + if (file != NULL) + { + fclose(file); + } + + Log(LOG_LEVEL_VERBOSE, + "Computing- & sending signature of file '%s'...", + basis); + if (!SendSignature(conn, basis)) + { + /* Error is already logged */ + return false; + } + + Log(LOG_LEVEL_VERBOSE, + "Receiving delta & applying patch to file '%s'...", + dest); + if (!RecvDelta(conn, basis, dest, perms)) + { + /* Error is already logged */ + return false; + } + + return true; +} diff --git a/libcfnet/file_stream.h b/libcfnet/file_stream.h new file mode 100644 index 0000000000..0252ef9fcb --- /dev/null +++ b/libcfnet/file_stream.h @@ -0,0 +1,104 @@ +/* + Copyright 2024 Northern.tech AS + + This file is part of CFEngine 3 - written and maintained by Northern.tech AS. + + This program is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + + To the extent this program is licensed as part of the Enterprise + versions of CFEngine, the applicable Commercial Open Source License + (COSL) may apply to this file if you as a licensee so wish it. See + included file COSL.txt. +*/ + +#ifndef FILE_STREAM_H +#define FILE_STREAM_H + +/** + * @file file_stream.h + * + * +---------+ +---------+ + * | client | | server | + * +----+----+ +----+----+ + * | | + * | sig = Compute(basis) | + * | | + * | Send(sig) ---------------->| sig = Recv() + * | | + * | | sig = BuildHashTable(sig) + * | | + * | | delta = Compute(sig, src) + * | | + * | delta = Recv() <-----------| Send(delta) + * | | + * | dest = Patch(delta, basis) | + * | | + * v v + * + * 1. Client generates a signature of the basis file (i.e., the "outdated" + * file) + * 2. Client sends signature to server + * 4. Server builds a hash table from the signature + * 5. Server generates delta from signature and the source file (i.e., the + * "up-to-date" file) + * 6. Server sends delta to client + * 7. Client applies delta on contents of the basis file in order to create + * the destination file + * + */ + +#include +#include +#include /* mode_t */ + + +/** + * @brief Reply with unspecified server refusal + * + * E.g., use this function when the resource does not exist or access is + * denied. We don't disinguish between these two for security reasons. + * + * @param conn The SSL connection object + * @return true on success, otherwise false + */ +bool FileStreamRefuse(SSL *conn); + +/** + * @brief Serve a file using the stream API + * + * @param conn The SSL connection object + * @param filename The name of the source file + * @return true on success, otherwise false + * + * @note If the source file is a symlink, this function serves the contents of + * the symlink target. + */ +bool FileStreamServe(SSL *conn, const char *filename); + +/** + * @brief Fetch a file using the stream API + * + * @param conn The SSL connection object + * @param basis The name of the basis file + * @param dest The name of the destination file + * @param perms The desired permissions of the destination file + * @return true on success, otherwise false + * + * @note If the destination file is a symlink, this function fetches the + * contents into the symlink target. + */ +bool FileStreamFetch( + SSL *conn, const char *basis, const char *dest, mode_t perms); + +#endif // FILE_STREAM_H diff --git a/libcfnet/protocol.c b/libcfnet/protocol.c index 3a6f9bcb37..37290c2194 100644 --- a/libcfnet/protocol.c +++ b/libcfnet/protocol.c @@ -31,6 +31,7 @@ #include #include #include +#include Seq *ProtocolOpenDir(AgentConnection *conn, const char *path) { @@ -124,6 +125,38 @@ bool ProtocolGet(AgentConnection *conn, const char *remote_path, return false; } + /* Use file stream API if it is available */ + const ProtocolVersion version = ConnectionInfoProtocolVersion(conn->conn_info); + if (ProtocolSupportsFileStream(version)) + { + fclose(file_ptr); + + char dest[PATH_MAX]; + ret = snprintf(dest, sizeof(dest), "%s.cfnew", local_path); + if (ret < 0 || (size_t)ret >= sizeof(dest)) + { + Log(LOG_LEVEL_ERR, "Truncation error: Path too long (%d >= %zu)", ret, sizeof(dest)); + return false; + } + + if (!FileStreamFetch(conn->conn_info->ssl, local_path, dest, perms)) + { + /* Error is already logged */ + return false; + } + + Log(LOG_LEVEL_VERBOSE, "Replacing file '%s' with '%s'...", dest, local_path); + if (rename(dest, local_path) == -1) + { + Log(LOG_LEVEL_ERR, "Failed to replace destination file '%s' with basis file '%s': %s", dest, local_path, GetErrorStr()); + return false; + } + + return true; + } + + /* Otherwise, use old protocol */ + char cfchangedstr[sizeof(CF_CHANGEDSTR1 CF_CHANGEDSTR2)]; snprintf(cfchangedstr, sizeof(cfchangedstr), "%s%s", CF_CHANGEDSTR1, CF_CHANGEDSTR2); diff --git a/libcfnet/protocol_version.c b/libcfnet/protocol_version.c index 31925e9a96..e99ef2c89d 100644 --- a/libcfnet/protocol_version.c +++ b/libcfnet/protocol_version.c @@ -33,6 +33,10 @@ ProtocolVersion ParseProtocolVersionPolicy(const char *const s) { return CF_PROTOCOL_COOKIE; } + else if (StringEqual(s, "4") || StringEqual(s, "filestream")) + { + return CF_PROTOCOL_FILESTREAM; + } else if (StringEqual(s, "latest")) { return CF_PROTOCOL_LATEST; diff --git a/libcfnet/protocol_version.h b/libcfnet/protocol_version.h index 06f7c32382..61e9cf3855 100644 --- a/libcfnet/protocol_version.h +++ b/libcfnet/protocol_version.h @@ -39,10 +39,11 @@ typedef enum /* --- Greater versions use TLS as secure communications layer --- */ CF_PROTOCOL_TLS = 2, CF_PROTOCOL_COOKIE = 3, + CF_PROTOCOL_FILESTREAM = 4, } ProtocolVersion; /* We use CF_PROTOCOL_LATEST as the default for new connections. */ -#define CF_PROTOCOL_LATEST CF_PROTOCOL_COOKIE +#define CF_PROTOCOL_LATEST CF_PROTOCOL_FILESTREAM static inline const char *ProtocolVersionString(const ProtocolVersion p) { @@ -54,6 +55,8 @@ static inline const char *ProtocolVersionString(const ProtocolVersion p) return "tls"; case CF_PROTOCOL_CLASSIC: return "classic"; + case CF_PROTOCOL_FILESTREAM: + return "filestream"; default: return "undefined"; } @@ -84,6 +87,11 @@ static inline bool ProtocolIsClassic(const ProtocolVersion p) return (p == CF_PROTOCOL_CLASSIC); } +static inline bool ProtocolSupportsFileStream(const ProtocolVersion p) +{ + return (p >= CF_PROTOCOL_FILESTREAM); +} + static inline bool ProtocolTerminateCSV(const ProtocolVersion p) { return (p < CF_PROTOCOL_COOKIE); diff --git a/libpromises/Makefile.am b/libpromises/Makefile.am index 4162972d18..b2123440de 100644 --- a/libpromises/Makefile.am +++ b/libpromises/Makefile.am @@ -30,7 +30,7 @@ AM_LDFLAGS = endif AM_LDFLAGS += $(CORE_LDFLAGS) $(LMDB_LDFLAGS) $(TOKYOCABINET_LDFLAGS) $(QDBM_LDFLAGS) \ - $(PCRE2_LDFLAGS) $(OPENSSL_LDFLAGS) $(SQLITE3_LDFLAGS) $(LIBACL_LDFLAGS) $(LIBYAML_LDFLAGS) $(LIBCURL_LDFLAGS) + $(PCRE2_LDFLAGS) $(OPENSSL_LDFLAGS) $(SQLITE3_LDFLAGS) $(LIBACL_LDFLAGS) $(LIBYAML_LDFLAGS) $(LIBCURL_LDFLAGS) $(LIBRSYNC_LDFLAGS) AM_CPPFLAGS = \ -I$(srcdir)/../libntech/libutils -I$(srcdir)/../libcfecompat \ @@ -39,16 +39,16 @@ AM_CPPFLAGS = \ -I$(srcdir)/../cf-check \ $(CORE_CPPFLAGS) $(ENTERPRISE_CPPFLAGS) \ $(LMDB_CPPFLAGS) $(TOKYOCABINET_CPPFLAGS) $(QDBM_CPPFLAGS) \ - $(PCRE2_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SQLITE3_CPPFLAGS) $(LIBACL_CPPFLAGS) $(LIBYAML_CPPFLAGS) $(LIBCURL_CPPFLAGS) + $(PCRE2_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SQLITE3_CPPFLAGS) $(LIBACL_CPPFLAGS) $(LIBYAML_CPPFLAGS) $(LIBCURL_CPPFLAGS) $(LIBRSYNC_CPPFLAGS) AM_CFLAGS = $(CORE_CFLAGS) $(ENTERPRISE_CFLAGS) \ $(LMDB_CFLAGS) $(TOKYOCABINET_CFLAGS) $(QDBM_CFLAGS) \ - $(PCRE2_CFLAGS) $(OPENSSL_CFLAGS) $(SQLITE3_CFLAGS) $(LIBACL_CFLAGS) $(LIBYAML_CFLAGS) $(LIBCURL_CFLAGS) + $(PCRE2_CFLAGS) $(OPENSSL_CFLAGS) $(SQLITE3_CFLAGS) $(LIBACL_CFLAGS) $(LIBYAML_CFLAGS) $(LIBCURL_CFLAGS) $(LIBRSYNC_CFLAGS) AM_YFLAGS = -d LIBS = $(LMDB_LIBS) $(TOKYOCABINET_LIBS) $(QDBM_LIBS) \ - $(PCRE2_LIBS) $(OPENSSL_LIBS) $(SQLITE3_LIBS) $(LIBACL_LIBS) $(LIBYAML_LIBS) $(LIBCURL_LIBS) + $(PCRE2_LIBS) $(OPENSSL_LIBS) $(SQLITE3_LIBS) $(LIBACL_LIBS) $(LIBYAML_LIBS) $(LIBCURL_LIBS) $(LIBRSYNC_LIBS) # The lib providing sd_listen_fds() is not needed in libpromises, it's actually # needed in libcfnet. But adding it here is an easy way to make sure it's diff --git a/tests/static-check/run.sh b/tests/static-check/run.sh index c73f06d325..913a65688f 100755 --- a/tests/static-check/run.sh +++ b/tests/static-check/run.sh @@ -19,7 +19,7 @@ fi function create_image() { local c=$(buildah from -q $BASE_IMG) buildah run $c -- dnf -q -y install "@C Development Tools and Libraries" clang cppcheck which diffutils file >/dev/null 2>&1 - buildah run $c -- dnf -q -y install pcre-devel pcre2-devel openssl-devel libxml2-devel pam-devel lmdb-devel libacl-devel libyaml-devel curl-devel libvirt-devel >/dev/null 2>&1 + buildah run $c -- dnf -q -y install pcre-devel pcre2-devel openssl-devel libxml2-devel pam-devel lmdb-devel libacl-devel libyaml-devel curl-devel libvirt-devel librsync-devel >/dev/null 2>&1 buildah run $c -- dnf clean all >/dev/null 2>&1 # Copy checksum of this file into container. We use the checksum to detect diff --git a/tests/valgrind-check/Containerfile b/tests/valgrind-check/Containerfile index a0ef77c9f5..cd4a35a749 100644 --- a/tests/valgrind-check/Containerfile +++ b/tests/valgrind-check/Containerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 AS build RUN DEBIAN_FRONTEND=noninteractive apt-get update -y -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libssl-dev libxml2-dev libpam0g-dev liblmdb-dev libacl1-dev libpcre2-dev +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libssl-dev libxml2-dev libpam0g-dev liblmdb-dev libacl1-dev libpcre2-dev librsync-dev RUN DEBIAN_FRONTEND=noninteractive apt-get install -y python3 git flex bison byacc automake make autoconf libtool valgrind COPY masterfiles masterfiles COPY core core