From 0c23e92edee6b68133520b2f91befb31112e6f26 Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 28 Oct 2024 13:40:37 +0100 Subject: [PATCH 1/9] Enable gzip compression on file rotation WIP Make dlt files compressed on file rotation so the current file is uncompressed but all other are compressed. This is so all files can be copied without issues as the current file otherwise is likely to be truncated. --- .../dlt_offline_logstorage.h | 7 +- .../dlt_offline_logstorage_behavior.c | 118 ++++++++++++++++++ ...dlt_offline_logstorage_behavior_internal.h | 4 + 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage.h b/src/offlinelogstorage/dlt_offline_logstorage.h index 37c85b3b2..dd1fde56c 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage.h +++ b/src/offlinelogstorage/dlt_offline_logstorage.h @@ -126,8 +126,11 @@ /* Offline Logstorage disable network routing */ #define DLT_LOGSTORAGE_GZIP_ERROR -1 /* error case */ #define DLT_LOGSTORAGE_GZIP_UNSET 0 /* not set */ -#define DLT_LOGSTORAGE_GZIP_OFF 1 /* default, enable network routing */ -#define DLT_LOGSTORAGE_GZIP_ON (1 << 1) /* disable network routing */ +#define DLT_LOGSTORAGE_GZIP_OFF 1 /* default, no compression */ +#define DLT_LOGSTORAGE_GZIP_ON \ + (1 << 1) /* enable gzip compression of all files */ +#define DLT_LOGSTORAGE_GZIP_FILE \ + (1 << 2) /* enable gzip compression on file rotation */ /* logstorage max cache */ extern unsigned int g_logstorage_cache_max; diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index fb9c95f2f..c2c1fc223 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -740,6 +740,24 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, } } + tmp = &config->records; + while ((*tmp)->next != NULL) { + int len = strlen((*tmp)->name); + const char *suffix = ".dlt"; + if (strcmp(&(*tmp)->name[len - strlen(suffix)], suffix) == 0) { + memset(absolute_file_path, 0, + sizeof(absolute_file_path) / sizeof(char)); + strcat(absolute_file_path, storage_path); + strncat(absolute_file_path, (*tmp)->name, len); + dlt_vlog(LOG_INFO, + "%s: Compressing '%s' (num_log_files: %d, " + "file_name:%s)\n", + __func__, absolute_file_path, num_log_files, + (*tmp)->name); + dlt_logstorage_compress_dlt_file(absolute_file_path); + } + tmp = &(*tmp)->next; + } } } @@ -769,6 +787,106 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, return ret; } +/** + * dlt_logstorage_compress_dlt_file + * + * Compress content of a file and remove the original file. + * compressed file name is the same as the original with .gz extension. + * + * @param file_path The file to compress + * @return 0 on success, -1 on error + */ +int dlt_logstorage_compress_dlt_file(char *file_path) +{ + char file_path_dest[DLT_OFFLINE_LOGSTORAGE_MAX_PATH_LEN + 1] = {'\0'}; + strcat(file_path_dest, file_path); + strcat(file_path_dest, ".gz"); + + FILE *source = fopen(file_path, "rb"); + if (source == NULL) { + dlt_vlog(LOG_ERR, "%s: could not open %s\n", __func__, file_path); + return -1; + } + + FILE *dest = fopen(file_path_dest, "wb"); + if (dest == NULL) { + dlt_vlog(LOG_ERR, "%s: could not open %s\n", __func__, file_path_dest); + return -1; + } + + gzFile gzfile = gzdopen(fileno(dest), "wb"); + if (dest == NULL) { + dlt_vlog(LOG_ERR, "%s: could not gz open %s\n", __func__, + file_path_dest); + return -1; + } + + int ret = dlt_logstorage_compress_fd(source, gzfile); + + fclose(source); + gzclose(gzfile); + fclose(dest); + + if (ret == 0 && remove(file_path) != 0) { + dlt_vlog(LOG_ERR, "%s: could not remove original file %s \n", __func__, + file_path); + return -1; + } + + return ret; +} + +/** + * dlt_logstorage_compress_fd + * + * Compress content of a file descriptor into a gzFile. + * + * @param source The file to compress + * @param dest The file to write to + * @return 0 on success, -1 on error + */ +int dlt_logstorage_compress_fd(FILE *source, gzFile *dest) +{ + int ret, flush; + z_stream strm; + unsigned char in[COMPRESS_CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, COMPRESS_CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + dlt_vlog(LOG_ERR, "%s: Can't open source\n", __func__); + return -1; + } + + ret = gzwrite(dest, in, strm.avail_in); + if (ret == 0) { + dlt_vlog(LOG_ERR, "%s: failed to write to log file\n", __func__); + return -1; + } + if (gzflush(dest, Z_SYNC_FLUSH) != 0) + dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__); + + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + // done when last data in file processed + } while (flush != Z_FINISH); + + // clean up and return + (void)deflateEnd(&strm); + return 0; +} + /** * dlt_logstorage_find_dlt_header * diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h index ec820d513..68dcaf035 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h @@ -75,6 +75,10 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, bool is_update_required, bool is_sync); +#define COMPRESS_CHUNK 16384 +int dlt_logstorage_compress_dlt_file(char *path); +int dlt_logstorage_compress_fd(FILE *source, gzFile *dest); + DLT_STATIC int dlt_logstorage_sync_to_file(DltLogStorageFilterConfig *config, DltLogStorageUserConfig *file_config, char *dev_path, From ce4dc32b58a13d77d624d61d3aefc1094c6e022f Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 28 Oct 2024 15:24:54 +0100 Subject: [PATCH 2/9] Enable file gzip config --- doc/dlt_offline_logstorage.md | 2 +- .../dlt_offline_logstorage.c | 13 ++- .../dlt_offline_logstorage.h | 2 +- .../dlt_offline_logstorage_behavior.c | 92 ++++++++++--------- 4 files changed, 58 insertions(+), 51 deletions(-) diff --git a/doc/dlt_offline_logstorage.md b/doc/dlt_offline_logstorage.md index 1bb3d1867..b385f0fc5 100644 --- a/doc/dlt_offline_logstorage.md +++ b/doc/dlt_offline_logstorage.md @@ -77,7 +77,7 @@ NOFiles= # Number of created files before oldest is SyncBehavior= # Specify sync strategy. Default: Sync'ed after every message. See Logstorage Ringbuffer Implementation below. EcuID= # Specify ECU identifier SpecificSize= # Store logs in storage devices after specific size is reached. -GzipCompression= # Write the logfiles with gzip compression. +GzipCompression= # Write the logfiles with gzip compression. ON: Continously, FILE: On file rotation OverwriteBehavior= # Specify overwrite strategy. Default: Delete oldest file and continue. See Logstorage Ringbuffer Implementation below. DisableNetwork= # Specify if the message shall be routed to network client. ``` diff --git a/src/offlinelogstorage/dlt_offline_logstorage.c b/src/offlinelogstorage/dlt_offline_logstorage.c index e65436cae..f7bdf53d0 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage.c +++ b/src/offlinelogstorage/dlt_offline_logstorage.c @@ -1337,17 +1337,22 @@ DLT_STATIC int dlt_logstorage_check_gzip_compression(DltLogStorageFilterConfig * if (strcasestr(value, "ON") != NULL) { config->gzip_compression = DLT_LOGSTORAGE_GZIP_ON; - } else if (strcasestr(value, "OFF") != NULL) { + } + else if (strcasestr(value, "FILE") != NULL) { + config->gzip_compression = DLT_LOGSTORAGE_GZIP_FILE; + } + else if (strcasestr(value, "OFF") != NULL) { config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF; - } else { + } + else { dlt_log(LOG_WARNING, "Unknown gzip compression flag. Set default OFF\n"); - config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF; + config->gzip_compression = DLT_LOGSTORAGE_GZIP_ERROR; return 1; } #else dlt_log(LOG_WARNING, "dlt-daemon not compiled with logstorage gzip support\n"); - config->gzip_compression = 0; + config->gzip_compression = DLT_LOGSTORAGE_GZIP_ERROR; #endif return 0; } diff --git a/src/offlinelogstorage/dlt_offline_logstorage.h b/src/offlinelogstorage/dlt_offline_logstorage.h index dd1fde56c..21e0a31e9 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage.h +++ b/src/offlinelogstorage/dlt_offline_logstorage.h @@ -123,7 +123,7 @@ #define DLT_LOGSTORAGE_DISABLE_NW_OFF 1 /* default, enable network routing */ #define DLT_LOGSTORAGE_DISABLE_NW_ON (1 << 1) /* disable network routing */ -/* Offline Logstorage disable network routing */ +/* Offline Logstorage enable gzip compression */ #define DLT_LOGSTORAGE_GZIP_ERROR -1 /* error case */ #define DLT_LOGSTORAGE_GZIP_UNSET 0 /* not set */ #define DLT_LOGSTORAGE_GZIP_OFF 1 /* default, no compression */ diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index c2c1fc223..11582a668 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -152,7 +152,7 @@ void dlt_logstorage_log_file_name(char *log_file_name, } dlt_logstorage_concat_logfile_name(log_file_name, ".dlt"); - if (filter_config->gzip_compression) { + if (filter_config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { dlt_logstorage_concat_logfile_name(log_file_name, ".gz"); } } @@ -370,15 +370,10 @@ int dlt_logstorage_storage_dir_info(DltLogStorageUserConfig *file_config, config->records = NULL; } - char* suffix = NULL; - for (i = 0; i < cnt; i++) { - if (config->gzip_compression) { - suffix = strdup(".dlt.gz"); - } - else { - suffix = strdup(".dlt"); - } + char *suffix = ".dlt.gz"; + char *gzsuffix = ".dlt"; + for (i = 0; i < cnt; i++) { int len = 0; len = strlen(file_name); @@ -389,10 +384,13 @@ int dlt_logstorage_storage_dir_info(DltLogStorageUserConfig *file_config, if (config->num_files == 1 && file_config->logfile_optional_counter) { /* .dlt or _.dlt */ if ((files[i]->d_name[len] == suffix[0]) || + (files[i]->d_name[len] == gzsuffix[0]) || (file_config->logfile_timestamp && - (files[i]->d_name[len] == file_config->logfile_delimiter))) { + (files[i]->d_name[len] == + file_config->logfile_delimiter))) { current_idx = 1; - } else { + } + else { continue; } } else { @@ -470,11 +468,6 @@ int dlt_logstorage_storage_dir_info(DltLogStorageUserConfig *file_config, free(files); - if (suffix) { - free(suffix); - suffix = NULL; - } - return ret; } @@ -497,12 +490,13 @@ DLT_STATIC void dlt_logstorage_open_log_output_file(DltLogStorageFilterConfig *c return; } config->fd = fileno(file); - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { #ifdef DLT_LOGSTORAGE_USE_GZIP dlt_vlog(LOG_DEBUG, "%s: Opening GZIP log file\n", __func__); config->gzlog = gzdopen(config->fd, mode); #endif - } else { + } + else { dlt_vlog(LOG_DEBUG, "%s: Opening log file\n", __func__); config->log = file; } @@ -740,23 +734,26 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, } } - tmp = &config->records; - while ((*tmp)->next != NULL) { - int len = strlen((*tmp)->name); - const char *suffix = ".dlt"; - if (strcmp(&(*tmp)->name[len - strlen(suffix)], suffix) == 0) { - memset(absolute_file_path, 0, - sizeof(absolute_file_path) / sizeof(char)); - strcat(absolute_file_path, storage_path); - strncat(absolute_file_path, (*tmp)->name, len); - dlt_vlog(LOG_INFO, - "%s: Compressing '%s' (num_log_files: %d, " - "file_name:%s)\n", - __func__, absolute_file_path, num_log_files, - (*tmp)->name); - dlt_logstorage_compress_dlt_file(absolute_file_path); + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_FILE) { + tmp = &config->records; + while ((*tmp)->next != NULL) { + int len = strlen((*tmp)->name); + const char *suffix = ".dlt"; + if (strcmp(&(*tmp)->name[len - strlen(suffix)], suffix) == + 0) { + memset(absolute_file_path, 0, + sizeof(absolute_file_path) / sizeof(char)); + strcat(absolute_file_path, storage_path); + strncat(absolute_file_path, (*tmp)->name, len); + dlt_vlog(LOG_INFO, + "%s: Compressing '%s' (num_log_files: %d, " + "file_name:%s)\n", + __func__, absolute_file_path, num_log_files, + (*tmp)->name); + dlt_logstorage_compress_dlt_file(absolute_file_path); + } + tmp = &(*tmp)->next; } - tmp = &(*tmp)->next; } } } @@ -953,9 +950,10 @@ DLT_STATIC int dlt_logstorage_write_to_log(void *ptr, size_t size, size_t nmemb, DltLogStorageFilterConfig *config) { #ifdef DLT_LOGSTORAGE_USE_GZIP - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { return gzfwrite(ptr, size, nmemb, config->gzlog); - } else { + } + else { return fwrite(ptr, size, nmemb, config->log); } #else @@ -980,26 +978,28 @@ DLT_STATIC void dlt_logstorage_check_write_ret(DltLogStorageFilterConfig *config } if (ret <= 0) { - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { #ifdef DLT_LOGSTORAGE_USE_GZIP const char *msg = gzerror(config->gzlog, &ret); if (msg != NULL) { dlt_vlog(LOG_ERR, "%s: failed to write cache into log file: %s\n", __func__, msg); } #endif - } else { + } + else { if (ferror(config->log) != 0) dlt_vlog(LOG_ERR, "%s: failed to write cache into log file\n", __func__); } } else { /* force sync */ - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { #ifdef DLT_LOGSTORAGE_USE_GZIP if (gzflush(config->gzlog, Z_SYNC_FLUSH) != 0) dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__); #endif - } else { + } + else { if (fflush(config->log) != 0) dlt_vlog(LOG_ERR, "%s: failed to flush log file\n", __func__); } @@ -1223,7 +1223,7 @@ int dlt_logstorage_prepare_on_msg(DltLogStorageFilterConfig *config, /* Sync only if on_msg */ if ((config->sync == DLT_LOGSTORAGE_SYNC_ON_MSG) || (config->sync == DLT_LOGSTORAGE_SYNC_UNSET)) { - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { if (fsync(fileno(config->gzlog)) != 0) { if (errno != ENOSYS) { dlt_vlog(LOG_ERR, "%s: failed to sync gzip log file\n", __func__); @@ -1319,10 +1319,11 @@ int dlt_logstorage_write_on_msg(DltLogStorageFilterConfig *config, dlt_log(LOG_WARNING, "Wrote less data than specified\n"); #ifdef DLT_LOGSTORAGE_USE_GZIP - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { gzerror(config->gzlog, &ret); return ret; - } else { + } + else { return ferror(config->log); } #else @@ -1353,12 +1354,13 @@ int dlt_logstorage_sync_on_msg(DltLogStorageFilterConfig *config, return -1; if (status == DLT_LOGSTORAGE_SYNC_ON_MSG) { /* sync on every message */ - if (config->gzip_compression) { + if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON) { #ifdef DLT_LOGSTORAGE_USE_GZIP if (gzflush(config->gzlog, Z_SYNC_FLUSH) != 0) dlt_vlog(LOG_ERR, "%s: failed to gzflush log file\n", __func__); #endif - } else { + } + else { if (fflush(config->log) != 0) dlt_vlog(LOG_ERR, "%s: failed to flush log file\n", __func__); } From 2808f007493f66db7ce64d753f75176a1c0555a8 Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 28 Oct 2024 15:44:11 +0100 Subject: [PATCH 3/9] Fix comment --- src/offlinelogstorage/dlt_offline_logstorage.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage.c b/src/offlinelogstorage/dlt_offline_logstorage.c index f7bdf53d0..177d08a2f 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage.c +++ b/src/offlinelogstorage/dlt_offline_logstorage.c @@ -1345,8 +1345,7 @@ DLT_STATIC int dlt_logstorage_check_gzip_compression(DltLogStorageFilterConfig * config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF; } else { - dlt_log(LOG_WARNING, - "Unknown gzip compression flag. Set default OFF\n"); + dlt_log(LOG_WARNING, "Unknown gzip compression flag\n"); config->gzip_compression = DLT_LOGSTORAGE_GZIP_ERROR; return 1; } From 4bcd6cc5d3f9e36ede67b45c43c56d739215f8c3 Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 4 Nov 2024 13:48:24 +0100 Subject: [PATCH 4/9] Put compression functions in ifdefs To enable compilation without gzip support --- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 6 ++++++ .../dlt_offline_logstorage_behavior_internal.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index 11582a668..e83954d93 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -734,6 +734,7 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, } } +#ifdef DLT_LOGSTORAGE_USE_GZIP if (config->gzip_compression == DLT_LOGSTORAGE_GZIP_FILE) { tmp = &config->records; while ((*tmp)->next != NULL) { @@ -755,6 +756,7 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, tmp = &(*tmp)->next; } } +#endif } } @@ -784,6 +786,7 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, return ret; } +#ifdef DLT_LOGSTORAGE_USE_GZIP /** * dlt_logstorage_compress_dlt_file * @@ -832,7 +835,9 @@ int dlt_logstorage_compress_dlt_file(char *file_path) return ret; } +#endif +#ifdef DLT_LOGSTORAGE_USE_GZIP /** * dlt_logstorage_compress_fd * @@ -883,6 +888,7 @@ int dlt_logstorage_compress_fd(FILE *source, gzFile *dest) (void)deflateEnd(&strm); return 0; } +#endif /** * dlt_logstorage_find_dlt_header diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h index 68dcaf035..b08fbf0ee 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h @@ -75,9 +75,11 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, bool is_update_required, bool is_sync); +#ifdef DLT_LOGSTORAGE_USE_GZIP #define COMPRESS_CHUNK 16384 int dlt_logstorage_compress_dlt_file(char *path); int dlt_logstorage_compress_fd(FILE *source, gzFile *dest); +#endif DLT_STATIC int dlt_logstorage_sync_to_file(DltLogStorageFilterConfig *config, DltLogStorageUserConfig *file_config, From c65afc37c0c638b49017b5683ffa58afa440dae0 Mon Sep 17 00:00:00 2001 From: gustav Date: Wed, 6 Nov 2024 12:09:48 +0100 Subject: [PATCH 5/9] Updates from internal review Fix closing of files in error case Change to Gzip OFF when disabled Fix function parameter gzfile --- src/offlinelogstorage/dlt_offline_logstorage.c | 2 +- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 6 +++++- .../dlt_offline_logstorage_behavior_internal.h | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage.c b/src/offlinelogstorage/dlt_offline_logstorage.c index 177d08a2f..d6a7fe07d 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage.c +++ b/src/offlinelogstorage/dlt_offline_logstorage.c @@ -1351,7 +1351,7 @@ DLT_STATIC int dlt_logstorage_check_gzip_compression(DltLogStorageFilterConfig * } #else dlt_log(LOG_WARNING, "dlt-daemon not compiled with logstorage gzip support\n"); - config->gzip_compression = DLT_LOGSTORAGE_GZIP_ERROR; + config->gzip_compression = DLT_LOGSTORAGE_GZIP_OFF; #endif return 0; } diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index e83954d93..6ff833da3 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -811,6 +811,7 @@ int dlt_logstorage_compress_dlt_file(char *file_path) FILE *dest = fopen(file_path_dest, "wb"); if (dest == NULL) { dlt_vlog(LOG_ERR, "%s: could not open %s\n", __func__, file_path_dest); + fclose(source); return -1; } @@ -818,6 +819,8 @@ int dlt_logstorage_compress_dlt_file(char *file_path) if (dest == NULL) { dlt_vlog(LOG_ERR, "%s: could not gz open %s\n", __func__, file_path_dest); + fclose(source); + fclose(dest); return -1; } @@ -847,7 +850,7 @@ int dlt_logstorage_compress_dlt_file(char *file_path) * @param dest The file to write to * @return 0 on success, -1 on error */ -int dlt_logstorage_compress_fd(FILE *source, gzFile *dest) +int dlt_logstorage_compress_fd(FILE *source, gzFile dest) { int ret, flush; z_stream strm; @@ -872,6 +875,7 @@ int dlt_logstorage_compress_fd(FILE *source, gzFile *dest) ret = gzwrite(dest, in, strm.avail_in); if (ret == 0) { + (void)deflateEnd(&strm); dlt_vlog(LOG_ERR, "%s: failed to write to log file\n", __func__); return -1; } diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h index b08fbf0ee..f2f6ba58a 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior_internal.h @@ -78,7 +78,7 @@ int dlt_logstorage_open_log_file(DltLogStorageFilterConfig *config, #ifdef DLT_LOGSTORAGE_USE_GZIP #define COMPRESS_CHUNK 16384 int dlt_logstorage_compress_dlt_file(char *path); -int dlt_logstorage_compress_fd(FILE *source, gzFile *dest); +int dlt_logstorage_compress_fd(FILE *source, gzFile dest); #endif DLT_STATIC int dlt_logstorage_sync_to_file(DltLogStorageFilterConfig *config, From 8253e266e86b91af067c98d237fdec9905d408e5 Mon Sep 17 00:00:00 2001 From: Erik Stenlund Date: Wed, 27 Nov 2024 10:25:29 +0100 Subject: [PATCH 6/9] Check gzlog in dlt_logstorage_sync_to_file Adds check in dlt_logstorage_sync_to_file that gzlog is NULL if the dlt-daemon is compiled with gzip, otherwise that statement will always return true if using gzip-compression on logs. --- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index 8c20b7f51..0378a09fd 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -1138,7 +1138,11 @@ DLT_STATIC int dlt_logstorage_sync_to_file(DltLogStorageFilterConfig *config, if ((start_index >= 0) && (count > 0)) { /* Prepare log file */ +#ifdef DLT_LOGSTORAGE_USE_GZIP + if (config->log == NULL && config->gzlog == NULL) +#else if (config->log == NULL) +#endif { if (dlt_logstorage_open_log_file(config, file_config, dev_path, count, true, false) != 0) From 2478992a2552655a630e47931df7a31a0e37f936 Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 28 Oct 2024 12:56:02 +0100 Subject: [PATCH 7/9] Fix file length calculation dlt_logstorage_log_file_name assumed the suffix always is .dlt which is not be the case if gzip compression is enabled. Changed to assume .dlt.gz when smax is calculated as it prevents buffer overflow if filenames are long and gzip is enabled. --- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index 0378a09fd..2458d9964 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -99,7 +99,9 @@ void dlt_logstorage_log_file_name(char *log_file_name, index_width = 0; } - const char * suffix = ".dlt"; + const char *suffix = + ".dlt.gz"; // Unknown here if gzip is used but we assume so as it leads + // to shorter avaialble space const int smax = DLT_MOUNT_PATH_MAX - strlen(suffix) - 1; int spos = 0; log_file_name[spos] = '\0'; From c860d92d2e93c3940177d34f93526c7986006b53 Mon Sep 17 00:00:00 2001 From: gustav Date: Mon, 28 Oct 2024 13:07:57 +0100 Subject: [PATCH 8/9] typo --- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index 2458d9964..d62305b92 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -101,7 +101,7 @@ void dlt_logstorage_log_file_name(char *log_file_name, const char *suffix = ".dlt.gz"; // Unknown here if gzip is used but we assume so as it leads - // to shorter avaialble space + // to shorter available space const int smax = DLT_MOUNT_PATH_MAX - strlen(suffix) - 1; int spos = 0; log_file_name[spos] = '\0'; From 8e18bcdd8d54115579f24b9576f251b889a1d3bc Mon Sep 17 00:00:00 2001 From: gustav Date: Thu, 21 Nov 2024 07:31:52 +0100 Subject: [PATCH 9/9] Make file suffix dependent on gzip compression --- src/offlinelogstorage/dlt_offline_logstorage_behavior.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c index d62305b92..76bc2b6d5 100644 --- a/src/offlinelogstorage/dlt_offline_logstorage_behavior.c +++ b/src/offlinelogstorage/dlt_offline_logstorage_behavior.c @@ -100,8 +100,8 @@ void dlt_logstorage_log_file_name(char *log_file_name, } const char *suffix = - ".dlt.gz"; // Unknown here if gzip is used but we assume so as it leads - // to shorter available space + filter_config->gzip_compression == DLT_LOGSTORAGE_GZIP_ON ? ".dlt.gz" + : ".dlt"; const int smax = DLT_MOUNT_PATH_MAX - strlen(suffix) - 1; int spos = 0; log_file_name[spos] = '\0';