diff --git a/CMakeLists.txt b/CMakeLists.txt index 3db7c863e..a7e4973e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,12 @@ SET( DEFAULT_EXECUTABLE_INSTALL_DIR bin/ ) SET( CMAKE_DEBUG_POSTFIX _debug ) SET( BUILD_SHARED_LIBS NO ) SET( ECC_IMPL secp256k1 CACHE STRING "secp256k1 or openssl or mixed" ) +SET( FC_USE_FULL_ZLIB FALSE CACHE BOOL "TRUE to try to use full zlib for compression, FALSE to use miniz.c") + +if( FC_USE_FULL_ZLIB ) + find_package( ZLIB REQUIRED ) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFC_USE_FULL_ZLIB") +endif() set(platformBitness 32) if(CMAKE_SIZEOF_VOID_P EQUAL 8) @@ -322,6 +328,7 @@ target_include_directories(fc ${CMAKE_CURRENT_SOURCE_DIR}/vendor/udt4/src ${CMAKE_CURRENT_SOURCE_DIR}/vendor/websocketpp ${CMAKE_CURRENT_SOURCE_DIR}/vendor/secp256k1-zkp + ${ZLIB_INCLUDE_DIRS} ) #target_link_libraries( fc PUBLIC udt ${Boost_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_SPECIFIC_LIBS} ${RPCRT4} ${CMAKE_DL_LIBS} ${rt_library} ${ECC_LIB} ) diff --git a/include/fc/compress/zlib.hpp b/include/fc/compress/zlib.hpp index 10a3a0e6c..baf01f5e8 100644 --- a/include/fc/compress/zlib.hpp +++ b/include/fc/compress/zlib.hpp @@ -1,10 +1,16 @@ #pragma once #include +#ifdef FC_USE_FULL_ZLIB +# include +#endif namespace fc { string zlib_compress(const string& in); +#ifdef FC_USE_FULL_ZLIB + void gzip_compress_file(const path& input_filename, const path& output_filename); +#endif } // namespace fc diff --git a/include/fc/log/file_appender.hpp b/include/fc/log/file_appender.hpp index a05fcd3e6..70a6e0ad0 100644 --- a/include/fc/log/file_appender.hpp +++ b/include/fc/log/file_appender.hpp @@ -18,6 +18,7 @@ class file_appender : public appender { bool rotate = false; microseconds rotation_interval; microseconds rotation_limit; + bool rotation_compression = false; }; file_appender( const variant& args ); ~file_appender(); @@ -31,4 +32,4 @@ class file_appender : public appender { #include FC_REFLECT( fc::file_appender::config, - (format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit) ) + (format)(filename)(flush)(rotate)(rotation_interval)(rotation_limit)(rotation_compression) ) diff --git a/src/compress/zlib.cpp b/src/compress/zlib.cpp index dd9d23bc7..a7099bab8 100644 --- a/src/compress/zlib.cpp +++ b/src/compress/zlib.cpp @@ -1,9 +1,77 @@ #include -#include "miniz.c" +#ifdef FC_USE_FULL_ZLIB +# include +# include +# include +#else +# include "miniz.c" +#endif namespace fc { +#ifdef FC_USE_FULL_ZLIB + string zlib_compress(const string& in) + { + unsigned long bufferLen = compressBound(in.size()); + std::unique_ptr buffer(new char[bufferLen]); + compress((unsigned char*)buffer.get(), &bufferLen, (const unsigned char*)in.c_str(), in.size()); + string result(buffer.get(), bufferLen); + return result; + } + + void gzip_compress_file(const path& input_filename, const path& output_filename) + { + std::ifstream infile(input_filename.generic_string().c_str(), std::ios::binary); + std::ofstream outfile(output_filename.generic_string().c_str(), std::ios::out | std::ios::binary); + unsigned bufferLen = 1024 * 1024; + std::unique_ptr inputBuffer(new char[bufferLen]); + std::unique_ptr outputBuffer(new char[bufferLen]); + + z_stream outputStream; + outputStream.zalloc = 0; + outputStream.zfree = 0; + outputStream.opaque = 0; + int windowBits = 15; + int GZIP_ENCODING = 16; + + deflateInit2(&outputStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, windowBits | GZIP_ENCODING, + 8, Z_DEFAULT_STRATEGY); + + if (infile) + { + do + { + infile.read(inputBuffer.get(), bufferLen); + int bytesRead = infile.gcount(); + if (bytesRead == 0) + break; + outputStream.avail_in = bytesRead; + outputStream.next_in = (unsigned char*)inputBuffer.get(); + do + { + outputStream.avail_out = bufferLen; + outputStream.next_out = (unsigned char*)outputBuffer.get(); + deflate(&outputStream, Z_NO_FLUSH); + int compressedBytesGenerated = bufferLen - outputStream.avail_out; + outfile.write(outputBuffer.get(), compressedBytesGenerated); + } + while (outputStream.avail_out == 0); + } + while (infile); + } + do + { + outputStream.avail_out = bufferLen; + outputStream.next_out = (unsigned char*)outputBuffer.get(); + deflate(&outputStream, Z_FINISH); + int compressedBytesGenerated = bufferLen - outputStream.avail_out; + outfile.write(outputBuffer.get(), compressedBytesGenerated); + } + while (outputStream.avail_out == 0); + deflateEnd(&outputStream); + } +#else string zlib_compress(const string& in) { size_t compressed_message_length; @@ -12,4 +80,5 @@ namespace fc free(compressed_message); return result; } +#endif } diff --git a/src/log/file_appender.cpp b/src/log/file_appender.cpp index 85b69c61b..e830b63dd 100644 --- a/src/log/file_appender.cpp +++ b/src/log/file_appender.cpp @@ -5,6 +5,9 @@ #include #include #include +#ifdef FC_USE_FULL_ZLIB +# include +#endif #include #include #include @@ -12,6 +15,8 @@ namespace fc { + static const string compression_extension( ".gz" ); + class file_appender::impl : public fc::retainable { public: @@ -22,6 +27,7 @@ namespace fc { private: future _rotation_task; time_point_sec _current_file_start_time; + std::unique_ptr _compression_thread; time_point_sec get_file_start_time( const time_point_sec& timestamp, const microseconds& interval ) { @@ -30,6 +36,28 @@ namespace fc { return time_point_sec( (uint32_t)(file_number * interval_seconds) ); } + void compress_file( const fc::path& filename ) + { +#ifdef FC_USE_FULL_ZLIB + FC_ASSERT( cfg.rotate && cfg.rotation_compression ); + FC_ASSERT( _compression_thread ); + if( !_compression_thread->is_current() ) + { + _compression_thread->async( [this, filename]() { compress_file( filename ); }, "compress_file" ).wait(); + return; + } + + try + { + gzip_compress_file( filename, filename.parent_path() / (filename.filename().string() + compression_extension) ); + remove_all( filename ); + } + catch( ... ) + { + } +#endif + } + public: impl( const config& c) : cfg( c ) { @@ -38,8 +66,10 @@ namespace fc { FC_ASSERT( cfg.rotation_interval >= seconds( 1 ) ); FC_ASSERT( cfg.rotation_limit >= cfg.rotation_interval ); - - +#ifdef FC_USE_FULL_ZLIB + if( cfg.rotation_compression ) + _compression_thread.reset( new thread( "compression") ); +#endif _rotation_task = async( [this]() { rotate_files( true ); }, "rotate_files(1)" ); } @@ -107,11 +137,16 @@ namespace fc { fc::time_point_sec current_timestamp = fc::time_point_sec::from_iso_string( current_timestamp_str ); if( current_timestamp < start_time ) { - if( current_timestamp < limit_time || file_size( current_filename ) <= 0 ) + if( current_timestamp < limit_time || file_size( link_filename.parent_path() / itr->filename() ) <= 0 ) { remove_all( *itr ); continue; } + if( !cfg.rotation_compression ) + continue; + if( current_filename.find( compression_extension ) != string::npos ) + continue; + compress_file( *itr ); } } catch (const fc::canceled_exception&) @@ -134,7 +169,8 @@ namespace fc { format( "${timestamp} ${thread_name} ${context} ${file}:${line} ${method} ${level}] ${message}" ), filename(p), flush(true), - rotate(false) + rotate(false), + rotation_compression(false) {} file_appender::file_appender( const variant& args ) : diff --git a/tests/compress/compress.cpp b/tests/compress/compress.cpp index a30255e98..a155d5459 100644 --- a/tests/compress/compress.cpp +++ b/tests/compress/compress.cpp @@ -38,9 +38,9 @@ BOOST_AUTO_TEST_CASE(smaz_test) BOOST_CHECK_EQUAL( decomp, line ); } +#ifndef FC_USE_FULL_ZLIB extern "C" { - enum { TINFL_FLAG_PARSE_ZLIB_HEADER = 1, @@ -85,5 +85,6 @@ BOOST_AUTO_TEST_CASE(zlib_test) std::string decomp = zlib_decompress( compressed ); BOOST_CHECK_EQUAL( decomp, line ); } +#endif BOOST_AUTO_TEST_SUITE_END()