From 12d1758bf4767c531eed0cf29ccbb1cc4f1213a2 Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 28 Jun 2024 14:51:17 +0200 Subject: [PATCH] - commit the generated data into the reop, because this facilitates running the tests in CI (since generating the data requires to equip the test runners with pyspark, which at least mother prefers not to) - the data is minimized to 5MB, by setting the scale factor 10x smaller, and from expected_results/ only keeping last/ this motivates rewriting some testcases that used expected_results/q08 into expected_results/last (which is the same) - removed one testcase, which tests if we get an error when parquet is not loaded (in MotherDuck, when we run iceberg in so-called "remote_only" mode, this will actually not raise an error) it was kind of a trivial test, so I hope this is OK.. --- Makefile | 6 +- .../metadata/v1.metadata.json | 217 +++++++ .../metadata/v2.metadata.json | 246 ++++++++ .../metadata/v3.metadata.json | 272 +++++++++ .../metadata/v4.metadata.json | 301 ++++++++++ .../metadata/v5.metadata.json | 324 ++++++++++ .../metadata/v6.metadata.json | 353 +++++++++++ .../metadata/v7.metadata.json | 445 ++++++++++++++ .../metadata/v8.metadata.json | 474 +++++++++++++++ .../metadata/v9.metadata.json | 561 ++++++++++++++++++ .../metadata/version-hint.text | 1 + .../metadata/v1.metadata.json | 139 +++++ .../metadata/v2.metadata.json | 169 ++++++ .../metadata/v3.metadata.json | 196 ++++++ .../metadata/v4.metadata.json | 226 +++++++ .../metadata/v5.metadata.json | 256 ++++++++ .../metadata/v6.metadata.json | 280 +++++++++ .../metadata/v7.metadata.json | 367 ++++++++++++ .../metadata/v8.metadata.json | 397 +++++++++++++ .../metadata/v9.metadata.json | 484 +++++++++++++++ .../metadata/version-hint.text | 1 + test/sql/iceberg_scan.test | 10 +- ...iceberg_scan_generated_data_0_01.test_slow | 12 +- .../iceberg_scan_generated_data_1.test_slow | 6 +- 24 files changed, 5722 insertions(+), 21 deletions(-) create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json create mode 100644 data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json create mode 100644 data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text diff --git a/Makefile b/Makefile index a488f1e..e9f0cfb 100644 --- a/Makefile +++ b/Makefile @@ -9,11 +9,11 @@ include extension-ci-tools/makefiles/duckdb_extension.Makefile # Custom makefile targets data: data_clean - python3 scripts/test_data_generator/generate_iceberg.py 0.01 data/iceberg/generated_spec1_0_01 1 - python3 scripts/test_data_generator/generate_iceberg.py 0.01 data/iceberg/generated_spec2_0_01 2 + python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec1_0_01 1 + python3 scripts/test_data_generator/generate_iceberg.py 0.001 data/iceberg/generated_spec2_0_01 2 data_large: data data_clean python3 scripts/test_data_generator/generate_iceberg.py 1 data/iceberg/generated_spec2_1 2 data_clean: - rm -rf data/iceberg/generated_* \ No newline at end of file + rm -rf data/iceberg/generated_* diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..1bd45a1 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,217 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577635792, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 955407744644249037, + "refs" : { + "main" : { + "snapshot-id" : 955407744644249037, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..b8bf95d --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,246 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577636708, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 8676758148642122301, + "refs" : { + "main" : { + "snapshot-id" : 8676758148642122301, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..ff125f2 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,272 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577637255, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 3557969894486994669, + "refs" : { + "main" : { + "snapshot-id" : 3557969894486994669, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..f186394 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,301 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577637664, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4366004082418470871, + "refs" : { + "main" : { + "snapshot-id" : 4366004082418470871, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..1217ec7 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,324 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577637990, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 7962977952076077932, + "refs" : { + "main" : { + "snapshot-id" : 7962977952076077932, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7962977952076077932, + "parent-snapshot-id" : 4366004082418470871, + "timestamp-ms" : 1719577637990, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7962977952076077932-1-ea6a8513-3bec-4596-8fd6-d84e2c268adf.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + }, { + "timestamp-ms" : 1719577637990, + "snapshot-id" : 7962977952076077932 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577637664, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..604c55f --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,353 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577638405, + "last-column-id" : 15, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4038755184896960077, + "refs" : { + "main" : { + "snapshot-id" : 4038755184896960077, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7962977952076077932, + "parent-snapshot-id" : 4366004082418470871, + "timestamp-ms" : 1719577637990, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7962977952076077932-1-ea6a8513-3bec-4596-8fd6-d84e2c268adf.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4038755184896960077, + "parent-snapshot-id" : 7962977952076077932, + "timestamp-ms" : 1719577638405, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4038755184896960077-1-ebb9cda2-3f47-4bb7-85ee-dcd54b9b0eb0.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + }, { + "timestamp-ms" : 1719577637990, + "snapshot-id" : 7962977952076077932 + }, { + "timestamp-ms" : 1719577638405, + "snapshot-id" : 4038755184896960077 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577637664, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577637990, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..b07316e --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,445 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577638566, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 4038755184896960077, + "refs" : { + "main" : { + "snapshot-id" : 4038755184896960077, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7962977952076077932, + "parent-snapshot-id" : 4366004082418470871, + "timestamp-ms" : 1719577637990, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7962977952076077932-1-ea6a8513-3bec-4596-8fd6-d84e2c268adf.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4038755184896960077, + "parent-snapshot-id" : 7962977952076077932, + "timestamp-ms" : 1719577638405, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4038755184896960077-1-ebb9cda2-3f47-4bb7-85ee-dcd54b9b0eb0.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + }, { + "timestamp-ms" : 1719577637990, + "snapshot-id" : 7962977952076077932 + }, { + "timestamp-ms" : 1719577638405, + "snapshot-id" : 4038755184896960077 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577637664, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577637990, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577638405, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..448f60a --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,474 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577638934, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 25064456467094441, + "refs" : { + "main" : { + "snapshot-id" : 25064456467094441, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7962977952076077932, + "parent-snapshot-id" : 4366004082418470871, + "timestamp-ms" : 1719577637990, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7962977952076077932-1-ea6a8513-3bec-4596-8fd6-d84e2c268adf.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4038755184896960077, + "parent-snapshot-id" : 7962977952076077932, + "timestamp-ms" : 1719577638405, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4038755184896960077-1-ebb9cda2-3f47-4bb7-85ee-dcd54b9b0eb0.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 25064456467094441, + "parent-snapshot-id" : 4038755184896960077, + "timestamp-ms" : 1719577638934, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "401521", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "401521", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-25064456467094441-1-617ea192-42d6-4bd8-adcb-0c26f8bcf219.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + }, { + "timestamp-ms" : 1719577637990, + "snapshot-id" : 7962977952076077932 + }, { + "timestamp-ms" : 1719577638405, + "snapshot-id" : 4038755184896960077 + }, { + "timestamp-ms" : 1719577638934, + "snapshot-id" : 25064456467094441 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577637664, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577637990, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577638405, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719577638566, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..218bd39 --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,561 @@ +{ + "format-version" : 1, + "table-uuid" : "f1ef3dc3-e9ef-4cd4-a881-c0fe9bbe4a5b", + "location" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table", + "last-updated-ms" : 1719577639105, + "last-column-id" : 16, + "schema" : { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + }, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 25064456467094441, + "refs" : { + "main" : { + "snapshot-id" : 25064456467094441, + "type" : "branch" + } + }, + "snapshots" : [ { + "snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577635792, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440910", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-955407744644249037-1-435eebe7-026b-4d5c-a2bc-a92c5fc40db1.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 8676758148642122301, + "parent-snapshot-id" : 955407744644249037, + "timestamp-ms" : 1719577636708, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6005", + "deleted-records" : "6005", + "added-files-size" : "340179", + "removed-files-size" : "440910", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "340179", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-8676758148642122301-1-551c7633-6148-4f62-9709-5def8d7153f4.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 3557969894486994669, + "parent-snapshot-id" : 8676758148642122301, + "timestamp-ms" : 1719577637255, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133338", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "473517", + "total-data-files" : "2", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-3557969894486994669-1-7769771b-b112-410b-9bed-0d749003ce71.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4366004082418470871, + "parent-snapshot-id" : 3557969894486994669, + "timestamp-ms" : 1719577637664, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "2", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "473517", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4366004082418470871-1-5d249d79-052a-4730-b214-4ec9be0d9728.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 7962977952076077932, + "parent-snapshot-id" : 4366004082418470871, + "timestamp-ms" : 1719577637990, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "changed-partition-count" : "0", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-7962977952076077932-1-ea6a8513-3bec-4596-8fd6-d84e2c268adf.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 4038755184896960077, + "parent-snapshot-id" : 7962977952076077932, + "timestamp-ms" : 1719577638405, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "399697", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "399697", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-4038755184896960077-1-ebb9cda2-3f47-4bb7-85ee-dcd54b9b0eb0.avro", + "schema-id" : 0 + }, { + "snapshot-id" : 25064456467094441, + "parent-snapshot-id" : 4038755184896960077, + "timestamp-ms" : 1719577638934, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577633178", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "7690", + "deleted-records" : "7690", + "added-files-size" : "401521", + "removed-files-size" : "399697", + "changed-partition-count" : "1", + "total-records" : "7690", + "total-files-size" : "401521", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/snap-25064456467094441-1-617ea192-42d6-4bd8-adcb-0c26f8bcf219.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577635792, + "snapshot-id" : 955407744644249037 + }, { + "timestamp-ms" : 1719577636708, + "snapshot-id" : 8676758148642122301 + }, { + "timestamp-ms" : 1719577637255, + "snapshot-id" : 3557969894486994669 + }, { + "timestamp-ms" : 1719577637664, + "snapshot-id" : 4366004082418470871 + }, { + "timestamp-ms" : 1719577637990, + "snapshot-id" : 7962977952076077932 + }, { + "timestamp-ms" : 1719577638405, + "snapshot-id" : 4038755184896960077 + }, { + "timestamp-ms" : 1719577638934, + "snapshot-id" : 25064456467094441 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577635792, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577636708, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577637255, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577637664, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577637990, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577638405, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719577638566, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1719577638934, + "metadata-file" : "data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec1_0_01/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json new file mode 100644 index 0000000..23764f7 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json @@ -0,0 +1,139 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 1, + "last-updated-ms" : 1719577643231, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 6282776777977664610, + "refs" : { + "main" : { + "snapshot-id" : 6282776777977664610, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + } ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json new file mode 100644 index 0000000..ae659ad --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json @@ -0,0 +1,169 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 2, + "last-updated-ms" : 1719577643950, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 7042070068400506628, + "refs" : { + "main" : { + "snapshot-id" : 7042070068400506628, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json new file mode 100644 index 0000000..76712a2 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json @@ -0,0 +1,196 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 3, + "last-updated-ms" : 1719577644763, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 5748721454628515239, + "refs" : { + "main" : { + "snapshot-id" : 5748721454628515239, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json new file mode 100644 index 0000000..2091ec0 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json @@ -0,0 +1,226 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 4, + "last-updated-ms" : 1719577645353, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 1711628578728651211, + "refs" : { + "main" : { + "snapshot-id" : 1711628578728651211, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json new file mode 100644 index 0000000..b2aee01 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json @@ -0,0 +1,256 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 5, + "last-updated-ms" : 1719577646077, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 1263004814000970671, + "refs" : { + "main" : { + "snapshot-id" : 1263004814000970671, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 1263004814000970671, + "parent-snapshot-id" : 1711628578728651211, + "timestamp-ms" : 1719577646077, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333730", + "removed-files-size" : "388603", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1263004814000970671-1-8b5240fc-77b5-457c-85be-40c145103568.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + }, { + "timestamp-ms" : 1719577646077, + "snapshot-id" : 1263004814000970671 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577645353, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json new file mode 100644 index 0000000..e2dd823 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json @@ -0,0 +1,280 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1719577646439, + "last-column-id" : 15, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 1116022577254091560, + "refs" : { + "main" : { + "snapshot-id" : 1116022577254091560, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 1263004814000970671, + "parent-snapshot-id" : 1711628578728651211, + "timestamp-ms" : 1719577646077, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333730", + "removed-files-size" : "388603", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1263004814000970671-1-8b5240fc-77b5-457c-85be-40c145103568.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 1116022577254091560, + "parent-snapshot-id" : 1263004814000970671, + "timestamp-ms" : 1719577646439, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719577640656", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1116022577254091560-1-e8871b49-d83f-4ff6-9524-90762c78ca5f.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + }, { + "timestamp-ms" : 1719577646077, + "snapshot-id" : 1263004814000970671 + }, { + "timestamp-ms" : 1719577646439, + "snapshot-id" : 1116022577254091560 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577645353, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577646077, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json new file mode 100644 index 0000000..4eb2f75 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json @@ -0,0 +1,367 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 6, + "last-updated-ms" : 1719577646716, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 1116022577254091560, + "refs" : { + "main" : { + "snapshot-id" : 1116022577254091560, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 1263004814000970671, + "parent-snapshot-id" : 1711628578728651211, + "timestamp-ms" : 1719577646077, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333730", + "removed-files-size" : "388603", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1263004814000970671-1-8b5240fc-77b5-457c-85be-40c145103568.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 1116022577254091560, + "parent-snapshot-id" : 1263004814000970671, + "timestamp-ms" : 1719577646439, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719577640656", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1116022577254091560-1-e8871b49-d83f-4ff6-9524-90762c78ca5f.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + }, { + "timestamp-ms" : 1719577646077, + "snapshot-id" : 1263004814000970671 + }, { + "timestamp-ms" : 1719577646439, + "snapshot-id" : 1116022577254091560 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577645353, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577646077, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577646439, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json new file mode 100644 index 0000000..45b76f8 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json @@ -0,0 +1,397 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1719577647210, + "last-column-id" : 16, + "current-schema-id" : 1, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 8646071970507518580, + "refs" : { + "main" : { + "snapshot-id" : 8646071970507518580, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 1263004814000970671, + "parent-snapshot-id" : 1711628578728651211, + "timestamp-ms" : 1719577646077, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333730", + "removed-files-size" : "388603", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1263004814000970671-1-8b5240fc-77b5-457c-85be-40c145103568.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 1116022577254091560, + "parent-snapshot-id" : 1263004814000970671, + "timestamp-ms" : 1719577646439, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719577640656", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1116022577254091560-1-e8871b49-d83f-4ff6-9524-90762c78ca5f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 8646071970507518580, + "parent-snapshot-id" : 1116022577254091560, + "timestamp-ms" : 1719577647210, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "685", + "added-files-size" : "51613", + "added-position-deletes" : "685", + "changed-partition-count" : "1", + "total-records" : "18044", + "total-files-size" : "1095894", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "11452", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-8646071970507518580-1-99fa164f-0590-437e-b59a-da1f20bcc6ec.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + }, { + "timestamp-ms" : 1719577646077, + "snapshot-id" : 1263004814000970671 + }, { + "timestamp-ms" : 1719577646439, + "snapshot-id" : 1116022577254091560 + }, { + "timestamp-ms" : 1719577647210, + "snapshot-id" : 8646071970507518580 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577645353, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577646077, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577646439, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719577646716, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json new file mode 100644 index 0000000..7abffb8 --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v9.metadata.json @@ -0,0 +1,484 @@ +{ + "format-version" : 2, + "table-uuid" : "15011b3b-c52d-4644-9093-64d864a0425d", + "location" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table", + "last-sequence-number" : 7, + "last-updated-ms" : 1719577647447, + "last-column-id" : 16, + "current-schema-id" : 2, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + } ] + }, { + "type" : "struct", + "schema-id" : 1, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "int" + } ] + }, { + "type" : "struct", + "schema-id" : 2, + "fields" : [ { + "id" : 1, + "name" : "l_orderkey_bool", + "required" : false, + "type" : "boolean" + }, { + "id" : 2, + "name" : "l_partkey_int", + "required" : false, + "type" : "int" + }, { + "id" : 3, + "name" : "l_suppkey_long", + "required" : false, + "type" : "long" + }, { + "id" : 4, + "name" : "l_extendedprice_float", + "required" : false, + "type" : "float" + }, { + "id" : 5, + "name" : "l_extendedprice_double", + "required" : false, + "type" : "double" + }, { + "id" : 6, + "name" : "l_extendedprice_dec9_2", + "required" : false, + "type" : "decimal(9, 2)" + }, { + "id" : 7, + "name" : "l_extendedprice_dec18_6", + "required" : false, + "type" : "decimal(18, 6)" + }, { + "id" : 8, + "name" : "l_extendedprice_dec38_10", + "required" : false, + "type" : "decimal(38, 10)" + }, { + "id" : 9, + "name" : "l_shipdate_date", + "required" : false, + "type" : "date" + }, { + "id" : 10, + "name" : "l_partkey_time", + "required" : false, + "type" : "int" + }, { + "id" : 11, + "name" : "l_commitdate_timestamp", + "required" : false, + "type" : "timestamp" + }, { + "id" : 12, + "name" : "l_commitdate_timestamp_tz", + "required" : false, + "type" : "timestamptz" + }, { + "id" : 13, + "name" : "l_comment_string", + "required" : false, + "type" : "string" + }, { + "id" : 14, + "name" : "uuid", + "required" : false, + "type" : "string" + }, { + "id" : 15, + "name" : "l_comment_blob", + "required" : false, + "type" : "binary" + }, { + "id" : 16, + "name" : "schema_evol_added_col_1", + "required" : false, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "peter", + "write.parquet.compression-codec" : "zstd", + "write.update.mode" : "merge-on-read" + }, + "current-snapshot-id" : 8646071970507518580, + "refs" : { + "main" : { + "snapshot-id" : 8646071970507518580, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643231, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "6005", + "added-files-size" : "440825", + "changed-partition-count" : "1", + "total-records" : "6005", + "total-files-size" : "440825", + "total-data-files" : "1", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-6282776777977664610-1-6ce7cc58-e5aa-4044-9a8e-f04039a9a334.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 7042070068400506628, + "parent-snapshot-id" : 6282776777977664610, + "timestamp-ms" : 1719577643950, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "3077", + "added-files-size" : "114769", + "added-position-deletes" : "3077", + "changed-partition-count" : "1", + "total-records" : "9082", + "total-files-size" : "555594", + "total-data-files" : "2", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-7042070068400506628-1-7570406a-99a1-49f5-af0c-80958424e9c7.avro", + "schema-id" : 0 + }, { + "sequence-number" : 3, + "snapshot-id" : 5748721454628515239, + "parent-snapshot-id" : 7042070068400506628, + "timestamp-ms" : 1719577644763, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-records" : "1685", + "added-files-size" : "133302", + "changed-partition-count" : "1", + "total-records" : "10767", + "total-files-size" : "688896", + "total-data-files" : "3", + "total-delete-files" : "1", + "total-position-deletes" : "3077", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-5748721454628515239-1-8113b514-3a00-4335-9624-471ece66bbad.avro", + "schema-id" : 0 + }, { + "sequence-number" : 4, + "snapshot-id" : 1711628578728651211, + "parent-snapshot-id" : 5748721454628515239, + "timestamp-ms" : 1719577645353, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "7690", + "added-files-size" : "410258", + "added-position-deletes" : "7690", + "changed-partition-count" : "1", + "total-records" : "18457", + "total-files-size" : "1099154", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1711628578728651211-1-cbe43f94-1f76-424c-a5cf-399018b65c79.avro", + "schema-id" : 0 + }, { + "sequence-number" : 5, + "snapshot-id" : 1263004814000970671, + "parent-snapshot-id" : 1711628578728651211, + "timestamp-ms" : 1719577646077, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "deleted-data-files" : "1", + "added-records" : "6592", + "deleted-records" : "7690", + "added-files-size" : "333730", + "removed-files-size" : "388603", + "changed-partition-count" : "1", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1263004814000970671-1-8b5240fc-77b5-457c-85be-40c145103568.avro", + "schema-id" : 0 + }, { + "sequence-number" : 6, + "snapshot-id" : 1116022577254091560, + "parent-snapshot-id" : 1263004814000970671, + "timestamp-ms" : 1719577646439, + "summary" : { + "operation" : "delete", + "spark.app.id" : "local-1719577640656", + "changed-partition-count" : "0", + "total-records" : "17359", + "total-files-size" : "1044281", + "total-data-files" : "4", + "total-delete-files" : "2", + "total-position-deletes" : "10767", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-1116022577254091560-1-e8871b49-d83f-4ff6-9524-90762c78ca5f.avro", + "schema-id" : 0 + }, { + "sequence-number" : 7, + "snapshot-id" : 8646071970507518580, + "parent-snapshot-id" : 1116022577254091560, + "timestamp-ms" : 1719577647210, + "summary" : { + "operation" : "overwrite", + "spark.app.id" : "local-1719577640656", + "added-data-files" : "1", + "added-position-delete-files" : "1", + "added-delete-files" : "1", + "added-records" : "685", + "added-files-size" : "51613", + "added-position-deletes" : "685", + "changed-partition-count" : "1", + "total-records" : "18044", + "total-files-size" : "1095894", + "total-data-files" : "5", + "total-delete-files" : "3", + "total-position-deletes" : "11452", + "total-equality-deletes" : "0" + }, + "manifest-list" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/snap-8646071970507518580-1-99fa164f-0590-437e-b59a-da1f20bcc6ec.avro", + "schema-id" : 1 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1719577643231, + "snapshot-id" : 6282776777977664610 + }, { + "timestamp-ms" : 1719577643950, + "snapshot-id" : 7042070068400506628 + }, { + "timestamp-ms" : 1719577644763, + "snapshot-id" : 5748721454628515239 + }, { + "timestamp-ms" : 1719577645353, + "snapshot-id" : 1711628578728651211 + }, { + "timestamp-ms" : 1719577646077, + "snapshot-id" : 1263004814000970671 + }, { + "timestamp-ms" : 1719577646439, + "snapshot-id" : 1116022577254091560 + }, { + "timestamp-ms" : 1719577647210, + "snapshot-id" : 8646071970507518580 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1719577643231, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1719577643950, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v2.metadata.json" + }, { + "timestamp-ms" : 1719577644763, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v3.metadata.json" + }, { + "timestamp-ms" : 1719577645353, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v4.metadata.json" + }, { + "timestamp-ms" : 1719577646077, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v5.metadata.json" + }, { + "timestamp-ms" : 1719577646439, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json" + }, { + "timestamp-ms" : 1719577646716, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v7.metadata.json" + }, { + "timestamp-ms" : 1719577647210, + "metadata-file" : "data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v8.metadata.json" + } ] +} \ No newline at end of file diff --git a/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text new file mode 100644 index 0000000..f11c82a --- /dev/null +++ b/data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/version-hint.text @@ -0,0 +1 @@ +9 \ No newline at end of file diff --git a/test/sql/iceberg_scan.test b/test/sql/iceberg_scan.test index e505d1c..7847b53 100644 --- a/test/sql/iceberg_scan.test +++ b/test/sql/iceberg_scan.test @@ -8,16 +8,10 @@ SELECT * FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg'); ---- Catalog Error -require iceberg - -# parquet is required for scanning this parquet-backed iceberg table -statement error -SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE); ----- -Catalog Error - require parquet +require iceberg + ### Scanning latest snapshot query I SELECT count(*) FROM ICEBERG_SCAN('data/iceberg/lineitem_iceberg', ALLOW_MOVED_PATHS=TRUE); diff --git a/test/sql/iceberg_scan_generated_data_0_01.test_slow b/test/sql/iceberg_scan_generated_data_0_01.test_slow index 9285d34..5fe7c50 100644 --- a/test/sql/iceberg_scan_generated_data_0_01.test_slow +++ b/test/sql/iceberg_scan_generated_data_0_01.test_slow @@ -6,8 +6,6 @@ require parquet require iceberg -require-env DUCKDB_ICEBERG_HAVE_TEST_DATA - ### Iceberg spec v1 # Check count matches @@ -26,7 +24,7 @@ DESCRIBE SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_i ---- query I nosort q1-schema -DESCRIBE SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec1_0_01/expected_results/q08/data/*.parquet') ORDER BY uuid; +DESCRIBE SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec1_0_01/expected_results/last/data/*.parquet') ORDER BY uuid; ---- # Check data is identical, sorting by uuid to guarantee unique order @@ -39,7 +37,7 @@ SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec1_0_01/pyspark_iceberg_ta ---- query I nosort q1-data -SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec1_0_01/expected_results/q08/data/*.parquet') ORDER BY uuid; +SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec1_0_01/expected_results/last/data/*.parquet') ORDER BY uuid; ---- # Confirm the type matches that of the iceberg schema @@ -72,7 +70,7 @@ DESCRIBE SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_i ---- query I nosort q2-schema -DESCRIBE SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec2_0_01/expected_results/q08/data/*.parquet') ORDER BY uuid; +DESCRIBE SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec2_0_01/expected_results/last/data/*.parquet') ORDER BY uuid; ---- # Check data is identical, sorting by uuid to guarantee unique order @@ -86,7 +84,7 @@ SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_ta ---- query I nosort q2-data -SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec2_0_01/expected_results/q08/data/*.parquet') ORDER BY uuid; +SELECT * FROM PARQUET_SCAN('data/iceberg/generated_spec2_0_01/expected_results/last/data/*.parquet') ORDER BY uuid; ---- ### Test schema evolution @@ -107,4 +105,4 @@ schema_evol_added_col_1 INTEGER YES NULL NULL NULL statement error DESCRIBE SELECT schema_evol_added_col_1 FROM ICEBERG_SCAN('data/iceberg/generated_spec2_0_01/pyspark_iceberg_table/metadata/v6.metadata.json') ORDER BY uuid; ---- -Binder Error \ No newline at end of file +Binder Error diff --git a/test/sql/iceberg_scan_generated_data_1.test_slow b/test/sql/iceberg_scan_generated_data_1.test_slow index 550ece5..a5856fd 100644 --- a/test/sql/iceberg_scan_generated_data_1.test_slow +++ b/test/sql/iceberg_scan_generated_data_1.test_slow @@ -20,7 +20,7 @@ SELECT COUNT(*) FROM ICEBERG_SCAN('data/iceberg/generated_1/pyspark_iceberg_tabl ---- query I nosort q1 -SELECT COUNT(*) FROM PARQUET_SCAN('data/iceberg/generated_1/expected_results/q08/data/*.parquet'); +SELECT COUNT(*) FROM PARQUET_SCAN('data/iceberg/generated_1/expected_results/last/data/*.parquet'); ---- query I nosort q2 @@ -37,5 +37,5 @@ SELECT * FROM ICEBERG_SCAN('data/iceberg/generated_1/pyspark_iceberg_table') WHE ---- query I nosort q3 -SELECT * FROM PARQUET_SCAN('data/iceberg/generated_1/expected_results/q08/data/*.parquet') WHERE uuid NOT NULL ORDER BY uuid; ----- \ No newline at end of file +SELECT * FROM PARQUET_SCAN('data/iceberg/generated_1/expected_results/last/data/*.parquet') WHERE uuid NOT NULL ORDER BY uuid; +----