From ec48a36dbf912b416664705dc92a87c7b5fb8b72 Mon Sep 17 00:00:00 2001 From: Shaharuk Shaikh <56402576+shaharuk-yb@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:52:33 +0530 Subject: [PATCH] added array data type support (#146) --- .../benchmarks/dataloader/Column.java | 28 ++++++++++ .../benchmarks/dataloader/DataGenerator.java | 3 +- .../dataloader/DataGeneratorLoader.java | 39 ++++++++----- .../featurebench/FeatureBenchLoader.java | 35 +++++++----- .../utils/RandomIntegerArrayGen.java | 15 +++-- .../utils/RandomLongArrayGen.java | 55 +++++++++++++++++++ .../perf-dataloader/array-mapping.properties | 32 +++++++++++ .../datatype-mapping.properties | 3 +- 8 files changed, 171 insertions(+), 39 deletions(-) create mode 100644 src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomLongArrayGen.java create mode 100644 src/main/resources/benchmarks/perf-dataloader/array-mapping.properties diff --git a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/Column.java b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/Column.java index 70100c27e..cd0fa5be9 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/Column.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/Column.java @@ -3,8 +3,10 @@ public class Column { private String columnName; private String dataType; + private String baseDataType; private Integer characterMaximumLength; private Boolean isIdentity; + private Boolean nullable; // Getters and Setters public String getColumnName() { @@ -31,6 +33,22 @@ public void setCharacterMaximumLength(Integer characterMaximumLength) { this.characterMaximumLength = characterMaximumLength; } + public String getBaseDataType() { + return baseDataType; + } + + public void setBaseDataType(String baseDataType) { + this.baseDataType = baseDataType; + } + + public Boolean getIdentity() { + return isIdentity; + } + + public void setIdentity(Boolean identity) { + isIdentity = identity; + } + public Boolean getIsIdentity() { return isIdentity; } @@ -39,13 +57,23 @@ public void setIsIdentity(Boolean isIdentity) { this.isIdentity = isIdentity; } + public Boolean getNullable() { + return nullable; + } + + public void setNullable(Boolean nullable) { + this.nullable = nullable; + } + @Override public String toString() { return "Column{" + "columnName='" + columnName + '\'' + ", dataType='" + dataType + '\'' + + ", baseDataType='" + baseDataType + '\'' + ", characterMaximumLength=" + characterMaximumLength + ", isIdentity=" + isIdentity + + ", nullable=" + nullable + '}'; } } diff --git a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGenerator.java b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGenerator.java index 3943c555f..de4844cf0 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGenerator.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGenerator.java @@ -34,7 +34,8 @@ protected Loader makeLoaderImpl() { // load properties file return new DataGeneratorLoader(this, getProperties("datatype-mapping.properties"), - getProperties("pk-mapping.properties"), getFkProperties()); + getProperties("pk-mapping.properties"), + getProperties("array-mapping.properties"), getFkProperties()); } @Override diff --git a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGeneratorLoader.java b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGeneratorLoader.java index c66098fbd..20d003f32 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGeneratorLoader.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/dataloader/DataGeneratorLoader.java @@ -27,16 +27,18 @@ public class DataGeneratorLoader extends Loader { private final Map properties; private final Map fkProperties; - + private final Map arrayProperties; private final Map pkProperties; private int minLevel = Integer.MAX_VALUE; public DataGeneratorLoader(DataGenerator benchmark, Map properties, - Map pkProperties, Map fkProperties) { + Map pkProperties, Map arrayProperties, + Map fkProperties) { super(benchmark); this.properties = properties; this.fkProperties = fkProperties; this.pkProperties = pkProperties; + this.arrayProperties = arrayProperties; } @Override @@ -173,9 +175,7 @@ public static String getTableSchemaIfTableExists(String tableName, Connection co public static List getTableSchema(String tableName, Connection conn, String tableSchema) { List tableSchemaList = new ArrayList<>(); - String query = "SELECT column_name, data_type, character_maximum_length, is_identity " + - "FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = ? and table_schema = ?"; - + String query = "SELECT c.column_name, c.data_type, CASE WHEN c.data_type = 'ARRAY' THEN (SELECT replace(t.typname, '_', '') FROM pg_type t JOIN pg_namespace ns ON t.typnamespace = ns.oid WHERE t.oid = (SELECT typarray FROM pg_type WHERE typname = replace(c.udt_name, '_', ''))) ELSE c.data_type END as base_data_type, c.character_maximum_length, c.is_identity, CASE WHEN c.is_nullable = 'YES' THEN TRUE END as nullable FROM INFORMATION_SCHEMA.COLUMNS c WHERE c.table_name = ? AND c.table_schema = ?"; try (PreparedStatement pstmt = conn.prepareStatement(query)) { pstmt.setString(1, tableName); pstmt.setString(2, tableSchema); @@ -188,6 +188,8 @@ public static List getTableSchema(String tableName, Connection conn, Str column.setDataType(rs.getString("data_type").replaceAll("\\s", "")); column.setCharacterMaximumLength((Integer) rs.getObject("character_maximum_length")); column.setIsIdentity("YES".equals(rs.getString("is_identity"))); + column.setBaseDataType(rs.getString("base_data_type")); + column.setNullable(rs.getBoolean("nullable")); tableSchemaList.add(column); } } catch (SQLException e) { @@ -429,18 +431,25 @@ public Map utilsMapping(List tableSchema, List< // take care of the rest of the keys for (Column col : tableSchema) { if (!columnToUtilMapping.containsKey(col.getColumnName())) { - // skip the columns having array data types - if (col.getDataType().equalsIgnoreCase("array")) - continue; PropertyMapping pm; // if column is one of the unique constraint columns, then use primary key util functions for it if (uniqueConstraintColumns.contains(col.getColumnName())) pm = pkProperties.get(col.getDataType()); - else - pm = properties.get(col.getDataType().toLowerCase()); + else { + + if (col.getDataType().equalsIgnoreCase("array")) { + // if it is array data type, find utility function with base data type + pm = arrayProperties.get(col.getBaseDataType().toLowerCase()); + // if suitable utility function is not present and the column is nullable, it can be skipped + if (pm == null && col.getNullable()) continue; + } else { + // if it is not array type, find utility function is properties. + pm = properties.get(col.getDataType().toLowerCase()); + } + } - if (pm == null) { - throw new RuntimeException(String.format("Cannot find suitable utility function for column " + + if (pm == null && !col.getNullable()) { + throw new RuntimeException(String.format("Cannot find suitable utility function for NOT NULL column " + "`%s` of datatype `%s`. Consider asking #perf team to add a utility function for given " + "data type", col.getColumnName(), col.getDataType())); } @@ -470,12 +479,12 @@ public Root generateMappingObject(String tableName, int rows, Map { if (fkColNames.contains(colName)) { - if (param instanceof Integer) + if (param instanceof Integer || param instanceof Long) col.params.add(param); else col.params.add(param.toString()); } else if (udColumns.containsKey(colName)) { - if (param instanceof Integer) + if (param instanceof Integer || param instanceof Long) col.params.add(param); else col.params.add(param.toString()); @@ -483,7 +492,7 @@ public Root generateMappingObject(String tableName, int rows, Map columnsDetails = this.columns.get(index); if (columnsDetails.containsKey("count")) { for (int i = 0; i < (int) columnsDetails.get("count"); i++) { - columnString.append(columnsDetails.get("name") + String.valueOf(i + 1)).append(","); - if (this.baseutils.get(index).getInstance().getClass().getName(). - toLowerCase().indexOf("json") >= 0) - valueString.append("?::JSON,"); - else { - valueString.append("?,"); - } + columnString.append(columnsDetails.get("name")).append(i + 1).append(","); + typeCastDataTypes(valueString, index); } } else { columnString.append(columnsDetails.get("name")).append(","); - if (this.baseutils.get(index).getInstance().getClass().getName(). - toLowerCase().indexOf("json") >= 0) - valueString.append("?::JSON,"); - else { - valueString.append("?,"); - } + typeCastDataTypes(valueString, index); } } @@ -240,6 +230,25 @@ public void load(Connection conn) throws SQLException { numberOfGeneratorFinished += 1; } + private void typeCastDataTypes(StringBuilder valueString, int index) { + String utilName = this.baseutils.get(index).getInstance().getClass().getName().toLowerCase(); + if (utilName.contains("array")) { + if(utilName.contains("integer")) + valueString.append("?::int[],"); + else if (utilName.contains("long")) + valueString.append("?::bigint[],"); + else if (utilName.contains("double")) + valueString.append("?::double[],"); + else if (utilName.contains("text")) + valueString.append("?::text[],"); + } + else if (utilName.contains("json")) + valueString.append("?::JSON,"); + else { + valueString.append("?,"); + } + } + @Override public void afterLoad() { if (numberOfGeneratorFinished != sizeOfLoadRule) return; diff --git a/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomIntegerArrayGen.java b/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomIntegerArrayGen.java index f138ccf5b..f26a88db6 100644 --- a/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomIntegerArrayGen.java +++ b/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomIntegerArrayGen.java @@ -9,9 +9,8 @@ public class RandomIntegerArrayGen implements BaseUtil { private List integerArray; - private int minValue; - private int maxValue; - + private Integer minValue; + private Integer maxValue; private int arraySize; public RandomIntegerArrayGen(List values) { @@ -22,8 +21,8 @@ public RandomIntegerArrayGen(List values) { this.arraySize = ((Number) values.get(0)).intValue(); if (arraySize <= 0) throw new RuntimeException("Please enter positive integer array length"); - this.minValue = ((Number) values.get(1)).intValue(); - this.maxValue = ((Number) values.get(2)).intValue(); + this.minValue = (Integer) values.get(1); + this.maxValue = (Integer) values.get(2); if (minValue > maxValue) throw new RuntimeException("Please enter correct bounds for max and min value"); } @@ -36,8 +35,8 @@ public RandomIntegerArrayGen(List values, int workerId, int totalWorkers this.arraySize = ((Number) values.get(0)).intValue(); if (arraySize <= 0) throw new RuntimeException("Please enter positive integer array length"); - this.minValue = ((Number) values.get(1)).intValue(); - this.maxValue = ((Number) values.get(2)).intValue(); + this.minValue = (Integer) values.get(1); + this.maxValue = (Integer) values.get(2); if (minValue > maxValue) throw new RuntimeException("Please enter correct bounds for max and min value"); } @@ -47,7 +46,7 @@ public Object run() throws ClassNotFoundException, InvocationTargetException, No Random random = new Random(); integerArray = new ArrayList<>(); for (int i = 0; i < arraySize; i++) { - int rd = random.nextInt((maxValue - minValue) + 1) + minValue; + Integer rd = random.nextInt((maxValue - minValue) + 1) + minValue; integerArray.add(rd); } return integerArray.toString().replaceFirst("\\[", "{").replace("]", "}"); diff --git a/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomLongArrayGen.java b/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomLongArrayGen.java new file mode 100644 index 000000000..678424ee3 --- /dev/null +++ b/src/main/java/com/oltpbenchmark/benchmarks/featurebench/utils/RandomLongArrayGen.java @@ -0,0 +1,55 @@ +package com.oltpbenchmark.benchmarks.featurebench.utils; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class RandomLongArrayGen implements BaseUtil { + + private List longArray; + + private Long minValue; + private Long maxValue; + private int arraySize; + + public RandomLongArrayGen(List values) { + if (values.size() != 3) { + throw new RuntimeException("Incorrect number of parameters for util function " + + this.getClass()); + } + this.arraySize = ((Number) values.get(0)).intValue(); + if (arraySize <= 0) + throw new RuntimeException("Please enter positive integer array length"); + this.minValue = ((Number) values.get(1)).longValue(); + this.maxValue = ((Number) values.get(2)).longValue(); + if (minValue > maxValue) + throw new RuntimeException("Please enter correct bounds for max and min value"); + } + + public RandomLongArrayGen(List values, int workerId, int totalWorkers) { + if (values.size() != 3) { + throw new RuntimeException("Incorrect number of parameters for util function " + + this.getClass()); + } + this.arraySize = ((Number) values.get(0)).intValue(); + if (arraySize <= 0) + throw new RuntimeException("Please enter positive integer array length"); + this.minValue = ((Number) values.get(1)).longValue(); + this.maxValue = ((Number) values.get(2)).longValue(); + if (minValue > maxValue) + throw new RuntimeException("Please enter correct bounds for max and min value"); + } + + @Override + public Object run() throws ClassNotFoundException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + Random random = new Random(); + longArray = new ArrayList<>(); + for (int i = 0; i < arraySize; i++) { + Long rd = random.nextLong((maxValue - minValue) + 1) + minValue; + longArray.add(rd); + } + return longArray.toString().replaceFirst("\\[", "{").replace("]", "}"); + } +} + diff --git a/src/main/resources/benchmarks/perf-dataloader/array-mapping.properties b/src/main/resources/benchmarks/perf-dataloader/array-mapping.properties new file mode 100644 index 000000000..faa4554f1 --- /dev/null +++ b/src/main/resources/benchmarks/perf-dataloader/array-mapping.properties @@ -0,0 +1,32 @@ +# arraySize, minLength,maxLength +int2=RandomIntegerArrayGen:3:1,1,32767 +int4=RandomIntegerArrayGen:3:1,1,2147483647 +int8=RandomLongArrayGen:3:1,1,9223372036854775807 +smallint=RandomIntegerArrayGen:3:1,1,32767 +integer=RandomIntegerArrayGen:3:1,1,2147483647 +bigint=RandomLongArrayGen:3:1,1,9223372036854775807 + +#int2=RandomIntegerArrayGen:3:2,-32768,32767 +#int4=RandomIntegerArrayGen:3:2,-2147483648,2147483647 +#int8=RandomLongArrayGen:3:2,-9223372036854775808,9223372036854775807 +#smallint=RandomIntegerArrayGen:3:2,-32768,32767 +#integer=RandomIntegerArrayGen:3:2,-2147483648,2147483647 +#bigint=RandomLongArrayGen:3:2,-9223372036854775808,9223372036854775807 + +#decimal +#numeric +#real +serial=RandomIntegerArrayGen:3:2,1,2147483647 +bigserial=RandomLongArrayGen:3:2,1,9223372036854775807 +#money +#char +#varchar +#date +#time +#timestamp +#timestamptz +#interval +#boolean +#json +#jsonb +text=RandomTextArrayGen:3:1,20,20 diff --git a/src/main/resources/benchmarks/perf-dataloader/datatype-mapping.properties b/src/main/resources/benchmarks/perf-dataloader/datatype-mapping.properties index 2f6194e54..f198ece63 100644 --- a/src/main/resources/benchmarks/perf-dataloader/datatype-mapping.properties +++ b/src/main/resources/benchmarks/perf-dataloader/datatype-mapping.properties @@ -26,8 +26,7 @@ text=RandomAString:2:1,1000 uuid=RandomUUID:0: -# arraySize, minLength,maxLength -array=RandomTextArrayGen:3:1,20,20 + timestampwithtimezone=RandomTimestampWithTimeZone:1:rows timestampwithouttimezone=RandomTimestampWithoutTimeZone:1:rows