[WIP] modules/zstd: Add ZSTD decoder

This commit adds a ZSTD decoder module that parses ZSTD frames. The provided tests examine the model using C++ API, which is a prerequisite for detailed tests using zstd library. Internal-tag: [#50221] Co-authored-by: Maciej Dudek <[email protected]> Co-authored-by: Pawel Czarnecki <[email protected]> Signed-off-by: Maciej Dudek <[email protected]> Signed-off-by: Pawel Czarnecki <[email protected]> Signed-off-by: Robert Winkler <[email protected]>
google · Feb 21, 2024 · 4bca36a · 4bca36a
1 parent ec7e648
commit 4bca36a
Show file tree

Hide file tree

Showing 9 changed files with 768 additions and 6 deletions.
diff --git a/xls/modules/zstd/BUILD b/xls/modules/zstd/BUILD
@@ -21,11 +21,9 @@ load(
     "//xls/build_rules:xls_build_defs.bzl",
     "xls_benchmark_ir",
     "xls_benchmark_verilog",
-    "xls_dslx_ir",
     "xls_dslx_library",
     "xls_dslx_test",
     "xls_dslx_verilog",
-    "xls_ir_opt_ir",
 )
 
 package(
@@ -774,3 +772,126 @@ place_and_route(
     synthesized_rtl = ":repacketizer_synth_asap7",
     target_die_utilization_percentage = "10",
 )
+
+xls_dslx_library(
+    name = "zstd_dec_dslx",
+    srcs = [
+        "zstd_dec.x",
+    ],
+    deps = [
+        ":block_dec_dslx",
+        ":block_header_dslx",
+        ":buffer_dslx",
+        ":common_dslx",
+        ":frame_header_dslx",
+        ":frame_header_test_dslx",
+        ":magic_dslx",
+        ":ram_printer_dslx",
+        ":repacketizer_dslx",
+        ":sequence_executor_dslx",
+        "//xls/examples:ram_dslx",
+    ],
+)
+
+xls_dslx_verilog(
+    name = "zstd_dec_verilog",
+    codegen_args = {
+        "module_name": "ZstdDecoder",
+        "generator": "pipeline",
+        "delay_model": "asap7",
+        "ram_configurations": ",".join([
+            "{ram_name}:1RW:{req}:{resp}:{wr_comp}:{latency}".format(
+                latency = 5,
+                ram_name = "ram{}".format(num),
+                req = "zstd_dec__req{}_s".format(num),
+                resp = "zstd_dec__resp{}_r".format(num),
+                wr_comp = "zstd_dec__wr_comp{}_r".format(num),
+            )
+            for num in range(7)
+        ]),
+        "pipeline_stages": "10",
+        "reset": "rst",
+        "reset_data_path": "true",
+        "reset_active_low": "false",
+        "reset_asynchronous": "true",
+        "flop_inputs": "false",
+        "flop_single_value_channels": "false",
+        "flop_outputs": "false",
+        "worst_case_throughput": "1",
+        "use_system_verilog": "false",
+    },
+    dslx_top = "ZstdDecoder",
+    library = ":zstd_dec_dslx",
+    # TODO: 2024-01-15: Workaround for https://github.com/google/xls/issues/869
+    # Force proc inlining for IR optimization
+    opt_ir_args = {
+        "inline_procs": "true",
+    },
+    verilog_file = "zstd_dec.v",
+)
+
+cc_test(
+    name = "zstd_dec_cc_test",
+    srcs = [
+        "zstd_dec_test.cc",
+    ],
+    data = [
+        ":zstd_dec_verilog.ir",
+    ],
+    #shard_count = 50,
+    deps = [
+        ":data_generator",
+        "//xls/common:xls_gunit_main",
+        "//xls/common/file:filesystem",
+        "//xls/common/file:get_runfile_path",
+        "//xls/common/status:matchers",
+        "//xls/interpreter:interpreter_proc_runtime",
+        "//xls/ir:events",
+        "//xls/ir:ir_parser",
+        "//xls/ir:value",
+        "@com_github_facebook_zstd//:zstd",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+xls_benchmark_ir(
+    name = "zstd_dec_opt_ir_benchmark",
+    src = ":zstd_dec_verilog.opt.ir",
+    benchmark_ir_args = {
+        #TODO: rewrite ram in opt_ir step to perform valid IR benchmark
+        "pipeline_stages": "1",
+        "delay_model": "asap7",
+    },
+)
+
+verilog_library(
+    name = "zstd_dec_verilog_lib",
+    srcs = [
+        ":zstd_dec.v",
+    ],
+)
+
+synthesize_rtl(
+    name = "zstd_dec_synth_asap7",
+    standard_cells = "@org_theopenroadproject_asap7sc7p5t_28//:asap7-sc7p5t_rev28_rvt",
+    top_module = "ZstdDecoder",
+    deps = [
+        ":zstd_dec_verilog_lib",
+    ],
+)
+
+benchmark_synth(
+    name = "zstd_dec_benchmark_synth",
+    synth_target = ":zstd_dec_synth_asap7",
+)
+
+place_and_route(
+    name = "zstd_dec_place_and_route",
+    clock_period = "750",
+    core_padding_microns = 2,
+    min_pin_distance = "0.5",
+    placement_density = "0.30",
+    skip_detailed_routing = True,
+    synthesized_rtl = ":zstd_dec_synth_asap7",
+    target_die_utilization_percentage = "10",
+)
diff --git a/xls/modules/zstd/common.x b/xls/modules/zstd/common.x
@@ -19,6 +19,7 @@ pub const BLOCK_SIZE_WIDTH = u32:21;
 pub const HISTORY_BUFFER_SIZE_KB = u32:64;
 pub const OFFSET_WIDTH = u32:22;
 pub const LENGTH_WIDTH = u32:22;
+pub const BUFFER_WIDTH = u32:128;
 
 pub type BlockData = bits[DATA_WIDTH];
 pub type BlockPacketLength = u32;

diff --git a/xls/modules/zstd/data_generator.cc b/xls/modules/zstd/data_generator.cc
@@ -107,4 +107,22 @@ absl::StatusOr<std::vector<uint8_t>> GenerateFrameHeader(int seed, bool magic) {
   return raw_data;
 }
 
+absl::StatusOr<std::vector<uint8_t>> GenerateFrame(int seed, BlockType btype) {
+  std::vector<std::string> args;
+  args.push_back("-s" + std::to_string(seed));
+  std::filesystem::path output_path =
+      std::filesystem::temp_directory_path() /
+      std::filesystem::path(
+          CreateNameForGeneratedFile(absl::MakeSpan(args), ".zstd", "fh"));
+  args.push_back("-p" + std::string(output_path));
+  if (btype != BlockType::RANDOM)
+    args.push_back("--block-type=" + std::to_string(btype));
+  if (btype == BlockType::RLE) args.push_back("--content-size");
+
+  XLS_ASSIGN_OR_RETURN(auto result, CallDecodecorpus(args));
+  auto raw_data = ReadFileAsRawData(output_path);
+  std::remove(output_path.c_str());
+  return raw_data;
+}
+
 }  // namespace xls::zstd
diff --git a/xls/modules/zstd/data_generator.h b/xls/modules/zstd/data_generator.h
@@ -35,7 +35,15 @@
 
 namespace xls::zstd {
 
+enum BlockType {
+  RAW,
+  RLE,
+  COMPRESSED,
+  RANDOM,
+};
+
 absl::StatusOr<std::vector<uint8_t>> GenerateFrameHeader(int seed, bool magic);
+absl::StatusOr<std::vector<uint8_t>> GenerateFrame(int seed, BlockType btype);
 
 }  // namespace xls::zstd
 

diff --git a/xls/modules/zstd/dec_demux.x b/xls/modules/zstd/dec_demux.x
@@ -109,6 +109,7 @@ pub proc DecoderDemux {
     )}
 
     next (tok: token, state: DecoderDemuxState) {
+        trace_fmt!("DecDemux: next: state: {:#x}", state);
         let (tok, data) = recv_if(tok, input_r, !state.last_packet.last, ZERO_DATA);
         let (send_raw, send_rle, send_cmp, new_state) = match state.status {
             DecoderDemuxStatus::IDLE =>
@@ -166,7 +167,7 @@ pub proc DecoderDemux {
         };
         let tok = send_if(tok, rle_s, send_rle, rle_data);
         let tok = send_if(tok, cmp_s, send_cmp, data_to_send);
-        if (new_state.send_data == new_state.byte_to_pass) {
+        let end_state = if (new_state.send_data == new_state.byte_to_pass) {
             let next_id = if (state.last_packet.last && state.last_packet.last_block) {
                 u32: 0
             } else {
@@ -181,7 +182,10 @@ pub proc DecoderDemux {
             }
         } else {
             new_state
-        }
+        };
+        trace_fmt!("DecDemux: next: end_state: {:#x}", end_state);
+
+        end_state
     }
 }
 

diff --git a/xls/modules/zstd/dec_mux.x b/xls/modules/zstd/dec_mux.x
@@ -62,6 +62,7 @@ pub proc DecoderMux {
     ) {(raw_r, rle_r, cmp_r, output_s)}
 
     next (tok: token, state: DecoderMuxState) {
+        trace_fmt!("DecMux: next: state: {:#x}", state);
         let (tok, raw_data, raw_data_valid) = recv_if_non_blocking(
             tok, raw_r, !state.raw_data_valid, zero!<ExtendedBlockDataPacket>());
         let state = if (raw_data_valid) {
@@ -159,6 +160,8 @@ pub proc DecoderMux {
         if (do_send) {
             trace_fmt!("sent {:#x}", data_to_send);
         } else {()};
+        trace_fmt!("DecMux: next: end_state: {:#x}", state);
+
         state
     }
 }

diff --git a/xls/modules/zstd/rle_block_dec.x b/xls/modules/zstd/rle_block_dec.x
@@ -186,6 +186,7 @@ proc BatchPacker {
     init { (BatchState { prev_last: true, ..ZERO_BATCH_STATE }) }
 
     next(tok: token, state: BatchState) {
+        trace_fmt!("BatchPacker: next: state: {:#x}", state);
         let (tok, decoded_data) = recv(tok, rle_data_r);
 
         let symbols_in_batch = state.symbols_in_batch;
@@ -218,13 +219,16 @@ proc BatchPacker {
         let new_symbols_in_batch =
             if do_send_batch { BlockPacketLength:0 } else { symbols_in_batch };
         let new_batch = if do_send_batch { BlockData:0 } else { batch };
-        BatchState {
+        let end_state = BatchState {
             batch: new_batch,
             symbols_in_batch: new_symbols_in_batch,
             prev_last: decoded_data.last,
             prev_last_block: sync_data.last_block,
             prev_id: sync_data.id
-        }
+        };
+        trace_fmt!("BatchPacker: next: end_state: {:#x}", end_state);
+
+        end_state
     }
 }