diff --git a/integration-tests/transforms/0029-parquet-input-no-files.hpl b/integration-tests/transforms/0029-parquet-input-no-files.hpl new file mode 100644 index 00000000000..252f6660a8d --- /dev/null +++ b/integration-tests/transforms/0029-parquet-input-no-files.hpl @@ -0,0 +1,232 @@ + + + + + 0029-parquet-input-no-files + Y + + + + Normal + + + N + 1000 + 100 + - + 2024/12/18 10:21:38.301 + - + 2024/12/18 10:21:38.301 + + + + + + No filenames + Parquet File Input + Y + + + Parquet File Input + Detect empty stream + Y + + + Detect empty stream + Count + Y + + + Count + Output + Y + + + + Count + GroupBy + + Y + + 1 + + none + + + N + Y + ${java.io.tmpdir} + + + count + id + COUNT_ALL + + + N + + + N + grp + + + 544 + 80 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 416 + 80 + + + + No filenames + DataGrid + + Y + + 1 + + none + + + + + + + + + + -1 + -1 + N + filename + String + + + + + 112 + 80 + + + + Parquet File Input + ParquetFileInput + + Y + + 1 + + none + + + + + id + id + Integer + + + lastname + lastname + String + + + firstname + firstname + String + + + zip + zip + Integer + + + city + city + String + + + birthdate + birthdate + Integer + + + street + street + String + + + housenr + housenr + Integer + + + stateCode + stateCode + String + + + stateDesc + stateDesc + String + + + filename + + + 272 + 80 + + + + Output + Dummy + + Y + + 1 + + none + + + + + 672 + 80 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv b/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv new file mode 100644 index 00000000000..1466dc703c4 --- /dev/null +++ b/integration-tests/transforms/datasets/golden-parquet-input-no-files.csv @@ -0,0 +1,2 @@ +filename,id,lastname,firstname,zip,city,birthdate,street,housenr,stateCode,stateDesc,count +"",,,,,,,,,,," 0000000000" diff --git a/integration-tests/transforms/main-0029-parquet.hwf b/integration-tests/transforms/main-0029-parquet.hwf index f2aff8f5586..f6dcd9d6815 100644 --- a/integration-tests/transforms/main-0029-parquet.hwf +++ b/integration-tests/transforms/main-0029-parquet.hwf @@ -35,14 +35,14 @@ limitations under the License. SPECIAL - N - 0 - 0 - 60 + 1 12 + 60 + 0 0 + N + 0 1 - 1 N 128 80 @@ -53,24 +53,22 @@ limitations under the License. PIPELINE - ${PROJECT_HOME}/0029-parquet-output.hpl - N - N - N - N - N - - N N - Basic - N - Y + N + N N - local + N + ${PROJECT_HOME}/0029-parquet-output.hpl + Basic Y + N + local + N + N + Y N 272 80 @@ -88,6 +86,9 @@ limitations under the License. 0029-parquet-input-nulls UNIT + + 0029-parquet-input-no-files UNIT + N 640 @@ -99,24 +100,22 @@ limitations under the License. PIPELINE - ${PROJECT_HOME}/0029-parquet-output-nulls.hpl - N - N - N - N - N - - N N - Basic - N - Y + N + N N - local + N + ${PROJECT_HOME}/0029-parquet-output-nulls.hpl + Basic Y + N + local + N + N + Y N 448 80 diff --git a/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json b/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json new file mode 100644 index 00000000000..3c1f7f30f8c --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-parquet-input-no-files.json @@ -0,0 +1,104 @@ +{ + "base_filename": "golden-parquet-input-no-files.csv", + "name": "golden-parquet-input-no-files", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "filename", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "id", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "lastname", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "firstname", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "zip", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "city", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "birthdate", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "street", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 5, + "field_precision": 0, + "field_name": "housenr", + "field_format": "####0;-####0" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "stateCode", + "field_format": "" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_name": "stateDesc", + "field_format": "" + }, + { + "field_comment": "", + "field_length": 10, + "field_type": 5, + "field_precision": 0, + "field_name": "count", + "field_format": " 0000000000;-0000000000" + } + ], + "folder_name": "" +} \ No newline at end of file diff --git a/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json b/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json new file mode 100644 index 00000000000..26192e82b48 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0029-parquet-input-no-files UNIT.json @@ -0,0 +1,83 @@ +{ + "database_replacements": [], + "autoOpening": true, + "description": "", + "persist_filename": "", + "test_type": "UNIT_TEST", + "variableValues": [], + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "filename", + "data_set_field": "filename" + }, + { + "transform_field": "id", + "data_set_field": "id" + }, + { + "transform_field": "lastname", + "data_set_field": "lastname" + }, + { + "transform_field": "firstname", + "data_set_field": "firstname" + }, + { + "transform_field": "zip", + "data_set_field": "zip" + }, + { + "transform_field": "city", + "data_set_field": "city" + }, + { + "transform_field": "birthdate", + "data_set_field": "birthdate" + }, + { + "transform_field": "street", + "data_set_field": "street" + }, + { + "transform_field": "housenr", + "data_set_field": "housenr" + }, + { + "transform_field": "stateCode", + "data_set_field": "stateCode" + }, + { + "transform_field": "stateDesc", + "data_set_field": "stateDesc" + }, + { + "transform_field": "count", + "data_set_field": "count" + } + ], + "field_order": [ + "filename", + "id", + "lastname", + "firstname", + "zip", + "city", + "birthdate", + "street", + "housenr", + "stateCode", + "stateDesc", + "count" + ], + "data_set_name": "golden-parquet-input-no-files", + "transform_name": "Output" + } + ], + "input_data_sets": [], + "name": "0029-parquet-input-no-files UNIT", + "trans_test_tweaks": [], + "pipeline_filename": "./0029-parquet-input-no-files.hpl" +} \ No newline at end of file