diff --git a/lib/logstash/inputs/s3/stream_downloader.rb b/lib/logstash/inputs/s3/stream_downloader.rb index 2adb705..1aed28a 100644 --- a/lib/logstash/inputs/s3/stream_downloader.rb +++ b/lib/logstash/inputs/s3/stream_downloader.rb @@ -28,9 +28,30 @@ def self.fetcher(remote_file, logger) class CompressedStreamDownloader < StreamDownloader def fetch - original_file = super + compressed_file_io_object = super @logger.debug("Decompressing gzip file", :remote_object_key => @remote_object.key) - Zlib::GzipReader.new(original_file) + decompress_io_object(compressed_file_io_object) + end + + private + + def decompress_io_object(io_object) + # Shelling out is necessary here until logstash-oss is using JRuby 9.4 which includes + # the Zlib::GzipReader.zcat method + output = '' + IO.popen('zcat', 'r+') do |zcat| + writer_thread = Thread.new do + while chunk = io_object.read(65536) + zcat.write(chunk) + end + zcat.close_write + end + + output = zcat.read + writer_thread.join + end + + output end end end;end;end