diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java index 6ddc0dd77e39..4837e7e97f02 100644 --- a/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java +++ b/example/udf/src/main/java/org/apache/iotdb/udf/AggregateFunctionExample.java @@ -79,7 +79,7 @@ public void deserialize(byte[] bytes) { @Override public AggregateFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() != 1) { + if (arguments.getArgumentsSize() != 1) { throw new UDFArgumentNotValidException("Only one parameter is required."); } return new AggregateFunctionAnalysis.Builder() diff --git a/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java b/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java index 0777c076bc95..ff48d0b57a9a 100644 --- a/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java +++ b/example/udf/src/main/java/org/apache/iotdb/udf/ScalarFunctionExample.java @@ -51,7 +51,7 @@ public class ScalarFunctionExample implements ScalarFunction { @Override public ScalarFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() < 1) { + if (arguments.getArgumentsSize() < 1) { throw new UDFArgumentNotValidException("At least one parameter is required."); } return new ScalarFunctionAnalysis.Builder().outputDataType(Type.BOOLEAN).build(); diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java index 8f51cbc0775c..2f2527f81754 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/AllSum.java @@ -38,10 +38,10 @@ public class AllSum implements ScalarFunction { @Override public ScalarFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() < 1) { + if (arguments.getArgumentsSize() < 1) { throw new UDFArgumentNotValidException("At least one parameter is required."); } - for (int i = 0; i < arguments.getChildExpressionsSize(); i++) { + for (int i = 0; i < arguments.getArgumentsSize(); i++) { if (arguments.getDataType(i) != Type.INT32 && arguments.getDataType(i) != Type.INT64 && arguments.getDataType(i) != Type.FLOAT @@ -62,7 +62,7 @@ public void beforeStart(FunctionArguments arguments) throws UDFException { private Type inferOutputDataType(FunctionArguments arguments) { Set inputTypeSet = new HashSet<>(); - for (int i = 0; i < arguments.getChildExpressionsSize(); i++) { + for (int i = 0; i < arguments.getArgumentsSize(); i++) { inputTypeSet.add(arguments.getDataType(i)); } if (inputTypeSet.contains(Type.DOUBLE)) { diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java index eb0a92dd24a7..656a79a9c27c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/ContainNull.java @@ -31,7 +31,7 @@ public class ContainNull implements ScalarFunction { @Override public ScalarFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() < 1) { + if (arguments.getArgumentsSize() < 1) { throw new UDFArgumentNotValidException("At least one parameter is required."); } return new ScalarFunctionAnalysis.Builder().outputDataType(Type.BOOLEAN).build(); diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java index 18354c10b9bb..e6fb975830b0 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/DatePlus.java @@ -33,7 +33,7 @@ public class DatePlus implements ScalarFunction { @Override public ScalarFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() != 2) { + if (arguments.getArgumentsSize() != 2) { throw new UDFArgumentNotValidException("Only two parameter is required."); } if (arguments.getDataType(0) != Type.DATE) { diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java index 59cbf1013173..c2aa402d8283 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/FirstTwoSum.java @@ -70,7 +70,7 @@ public void deserialize(byte[] bytes) { @Override public AggregateFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() != 3) { + if (arguments.getArgumentsSize() != 3) { throw new UDFArgumentNotValidException("FirstTwoSum should accept three column as input"); } for (int i = 0; i < 2; i++) { diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java index 8f43f0119c13..94c45254920f 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyAvg.java @@ -63,7 +63,7 @@ public void deserialize(byte[] bytes) { @Override public AggregateFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() != 1) { + if (arguments.getArgumentsSize() != 1) { throw new UDFArgumentNotValidException("MyAvg only accepts one column as input"); } if (arguments.getDataType(0) != Type.INT32 diff --git a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java index 9a2ad17b92bb..e1dae4ff90df 100644 --- a/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java +++ b/integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/relational/MyCount.java @@ -58,7 +58,7 @@ public void deserialize(byte[] bytes) { @Override public AggregateFunctionAnalysis analyze(FunctionArguments arguments) throws UDFArgumentNotValidException { - if (arguments.getChildExpressionsSize() == 0) { + if (arguments.getArgumentsSize() == 0) { throw new UDFArgumentNotValidException("MyCount accepts at least one parameter"); } return new AggregateFunctionAnalysis.Builder().outputDataType(Type.INT64).build(); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java new file mode 100644 index 000000000000..7808098eca2c --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileTableGenerator.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.it.utils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeSet; +import java.util.stream.Collectors; + +public class TsFileTableGenerator implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(TsFileTableGenerator.class); + + private final File tsFile; + private final TsFileWriter writer; + private final Map> table2TimeSet; + private final Map> table2MeasurementSchema; + private final Map> table2ColumnCategory; + private Random random; + + public TsFileTableGenerator(final File tsFile) throws IOException { + this.tsFile = tsFile; + this.writer = new TsFileWriter(tsFile); + this.table2TimeSet = new HashMap<>(); + this.table2MeasurementSchema = new HashMap<>(); + this.table2ColumnCategory = new HashMap<>(); + this.random = new Random(); + } + + public void registerTable( + final String tableName, + final List columnSchemasList, + final List columnCategoryList) { + if (table2MeasurementSchema.containsKey(tableName)) { + LOGGER.warn("Table {} already exists", tableName); + return; + } + + writer.registerTableSchema(new TableSchema(tableName, columnSchemasList, columnCategoryList)); + table2TimeSet.put(tableName, new TreeSet<>()); + table2MeasurementSchema.put(tableName, columnSchemasList); + table2ColumnCategory.put(tableName, columnCategoryList); + } + + public void generateData( + final String tableName, final int number, final long timeGap, final boolean isAligned) + throws IOException, WriteProcessException { + final List schemas = table2MeasurementSchema.get(tableName); + final List columnNameList = + schemas.stream().map(IMeasurementSchema::getMeasurementName).collect(Collectors.toList()); + final List dataTypeList = + schemas.stream().map(IMeasurementSchema::getType).collect(Collectors.toList()); + final List columnCategoryList = table2ColumnCategory.get(tableName); + final TreeSet timeSet = table2TimeSet.get(tableName); + final Tablet tablet = new Tablet(tableName, columnNameList, dataTypeList, columnCategoryList); + final Object[] values = tablet.values; + final long sensorNum = schemas.size(); + long startTime = timeSet.isEmpty() ? 0L : timeSet.last(); + + for (long r = 0; r < number; r++) { + final int row = tablet.getRowSize(); + startTime += timeGap; + tablet.addTimestamp(row, startTime); + timeSet.add(startTime); + for (int i = 0; i < sensorNum; i++) { + generateDataPoint(values[i], row, schemas.get(i)); + } + // write + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + if (!isAligned) { + writer.writeTable(tablet); + } else { + writer.writeAligned(tablet); + } + tablet.reset(); + } + } + // write + if (tablet.getRowSize() != 0) { + if (!isAligned) { + writer.writeTable(tablet); + } else { + writer.writeAligned(tablet); + } + tablet.reset(); + } + + LOGGER.info("Write {} points into table {}", number, tableName); + } + + private void generateDataPoint(final Object obj, final int row, final IMeasurementSchema schema) { + switch (schema.getType()) { + case INT32: + generateINT32(obj, row); + break; + case DATE: + generateDATE(obj, row); + break; + case INT64: + case TIMESTAMP: + generateINT64(obj, row); + break; + case FLOAT: + generateFLOAT(obj, row); + break; + case DOUBLE: + generateDOUBLE(obj, row); + break; + case BOOLEAN: + generateBOOLEAN(obj, row); + break; + case TEXT: + case BLOB: + case STRING: + generateTEXT(obj, row); + break; + default: + LOGGER.error("Wrong data type {}.", schema.getType()); + } + } + + private void generateINT32(final Object obj, final int row) { + final int[] ints = (int[]) obj; + ints[row] = random.nextInt(); + } + + private void generateDATE(final Object obj, final int row) { + final LocalDate[] dates = (LocalDate[]) obj; + dates[row] = + LocalDate.of(1000 + random.nextInt(9000), 1 + random.nextInt(12), 1 + random.nextInt(28)); + } + + private void generateINT64(final Object obj, final int row) { + final long[] longs = (long[]) obj; + longs[row] = random.nextLong(); + } + + private void generateFLOAT(final Object obj, final int row) { + final float[] floats = (float[]) obj; + floats[row] = random.nextFloat(); + } + + private void generateDOUBLE(final Object obj, final int row) { + final double[] doubles = (double[]) obj; + doubles[row] = random.nextDouble(); + } + + private void generateBOOLEAN(final Object obj, final int row) { + final boolean[] booleans = (boolean[]) obj; + booleans[row] = random.nextBoolean(); + } + + private void generateTEXT(final Object obj, final int row) { + final Binary[] binaries = (Binary[]) obj; + binaries[row] = + new Binary(String.format("test point %d", random.nextInt()), TSFileConfig.STRING_CHARSET); + } + + public long getTotalNumber() { + return table2TimeSet.entrySet().stream() + .mapToInt( + entry -> entry.getValue().size() * table2MeasurementSchema.get(entry.getKey()).size()) + .sum(); + } + + @Override + public void close() throws Exception { + writer.close(); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index cf4d22c6aca4..fe31f97b4511 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -24,14 +24,17 @@ import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileGenerator; +import org.apache.iotdb.it.utils.TsFileTableGenerator; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.jdbc.IoTDBSQLException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; @@ -902,7 +905,7 @@ public void testLoadLocally() throws Exception { } @Test - public void testLoadWithConvertOnTypeMismatch() throws Exception { + public void testLoadWithConvertOnTypeMismatchForTreeModel() throws Exception { List> measurementSchemas = generateMeasurementSchemasForDataTypeConvertion(); @@ -918,7 +921,7 @@ public void testLoadWithConvertOnTypeMismatch() throws Exception { try (final TsFileGenerator generator = new TsFileGenerator(file)) { generator.registerTimeseries(SchemaConfig.DEVICE_0, schemaList2); - generator.generateData(SchemaConfig.DEVICE_0, 100, PARTITION_INTERVAL / 10_000, false); + generator.generateData(SchemaConfig.DEVICE_0, 10000, PARTITION_INTERVAL / 10_000, false); writtenPoint = generator.getTotalNumber(); } @@ -969,7 +972,80 @@ public void testLoadWithConvertOnTypeMismatch() throws Exception { return pairs; } + @Test + public void testLoadWithConvertOnTypeMismatchForTableModel() throws Exception { + final int lineCount = 10000; + + List> measurementSchemas = + generateMeasurementSchemasForDataTypeConvertion(); + List columnCategories = + generateTabletColumnCategory(0, measurementSchemas.size()); + + final File file = new File(tmpDir, "1-0-0-0.tsfile"); + + List schemaList1 = + measurementSchemas.stream().map(pair -> pair.left).collect(Collectors.toList()); + List schemaList2 = + measurementSchemas.stream().map(pair -> pair.right).collect(Collectors.toList()); + + try (final TsFileTableGenerator generator = new TsFileTableGenerator(file)) { + generator.registerTable(SchemaConfig.TABLE_0, schemaList2, columnCategories); + + generator.generateData(SchemaConfig.TABLE_0, lineCount, PARTITION_INTERVAL / 10_000, false); + } + + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute(String.format("create database if not exists %s", SchemaConfig.DATABASE_0)); + statement.execute(String.format("use %s", SchemaConfig.DATABASE_0)); + statement.execute(convert2TableSQL(SchemaConfig.TABLE_0, schemaList1, columnCategories)); + statement.execute(String.format("load '%s'", file.getAbsolutePath())); + try (final ResultSet resultSet = + statement.executeQuery(String.format("select count(*) from %s", SchemaConfig.TABLE_0))) { + if (resultSet.next()) { + Assert.assertEquals(lineCount, resultSet.getLong(1)); + } else { + Assert.fail("This ResultSet is empty."); + } + } + } + } + + private List generateTabletColumnCategory(int tagNum, int filedNum) { + List columnTypes = new ArrayList<>(tagNum + filedNum); + for (int i = 0; i < tagNum; i++) { + columnTypes.add(Tablet.ColumnCategory.TAG); + } + for (int i = 0; i < filedNum; i++) { + columnTypes.add(Tablet.ColumnCategory.FIELD); + } + return columnTypes; + } + + private String convert2TableSQL( + final String tableName, + final List schemaList, + final List columnCategoryList) { + List columns = new ArrayList<>(); + for (int i = 0; i < schemaList.size(); i++) { + final MeasurementSchema measurement = schemaList.get(i); + columns.add( + String.format( + "%s %s %s", + measurement.getMeasurementName(), + measurement.getType(), + columnCategoryList.get(i).name())); + } + String tableCreation = + String.format("create table %s(%s)", tableName, String.join(", ", columns)); + LOGGER.info("schema execute: {}", tableCreation); + return tableCreation; + } + private static class SchemaConfig { + private static final String DATABASE_0 = "root"; + private static final String TABLE_0 = "test"; private static final String STORAGE_GROUP_0 = "root.sg.test_0"; private static final String STORAGE_GROUP_1 = "root.sg.test_1"; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index 66bae32c5b10..a2f5c65cb73e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -26,7 +26,6 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.Assert; import org.junit.Test; @@ -106,7 +105,7 @@ public void testBasicAlterPipe() throws Exception { } // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java index cdcc41ad53fe..03e52a982e61 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java @@ -266,11 +266,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -281,10 +284,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java index a81b02f7dd73..1665f9d2ce2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java @@ -65,7 +65,8 @@ public void testBasicAlterPipe() throws Exception { long lastCreationTime; try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -96,7 +97,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -114,7 +116,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -149,7 +152,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -183,7 +187,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -225,7 +230,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -257,7 +263,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -289,7 +296,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -323,7 +331,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -354,7 +363,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java index cd144bd4b86c..bd0a94419c14 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java @@ -28,22 +28,28 @@ import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.write.record.Tablet; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2TableModel.class}) public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelTestIT { + @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -117,8 +123,6 @@ public void testSinkTabletFormat() throws Exception { testSinkFormat("tablet"); } - // table model not support - @Ignore @Test public void testSinkTsFileFormat() throws Exception { testSinkFormat("tsfile"); @@ -179,7 +183,7 @@ private void testSinkFormat(final String format) throws Exception { Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - TableModelUtils.insertData("test", "test", 100, 150, senderEnv, true); + TableModelUtils.insertData("test", "test", 50, 150, senderEnv, true); if (!TestUtils.tryExecuteNonQueriesWithRetry( senderEnv, @@ -187,6 +191,8 @@ private void testSinkFormat(final String format) throws Exception { return; } + TableModelUtils.assertCountData("test", "test", 150, receiverEnv); + TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.**", @@ -218,6 +224,9 @@ private void testSinkFormat(final String format) throws Exception { } TableModelUtils.insertData("test", "test", 150, 200, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 200, 250, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 250, 300, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 300, 350, senderEnv, true); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -227,7 +236,7 @@ private void testSinkFormat(final String format) throws Exception { new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), handleFailure); - TableModelUtils.assertCountData("test", "test", 150, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test", 350, receiverEnv); } } @@ -294,4 +303,414 @@ public void testWriteBackSink() throws Exception { TableModelUtils.assertCountData("test1", "test", 100, receiverEnv, handleFailure); } } + + @Test + public void testSinkTsFileFormat2() throws Exception { + doTest(this::insertTablet1); + } + + @Test + public void testSinkTsFileFormat3() throws Exception { + doTest(this::insertTablet2); + } + + @Test + public void testSinkTsFileFormat4() throws Exception { + doTest(this::insertTablet3); + } + + @Test + public void testSinkTsFileFormat5() throws Exception { + doTest(this::insertTablet4); + } + + @Test + public void testSinkTsFileFormat6() throws Exception { + doTest(this::insertTablet5); + } + + @Test + public void testSinkTsFileFormat7() throws Exception { + doTest(this::insertTablet6); + } + + @Test + public void testSinkTsFileFormat8() throws Exception { + doTest(this::insertTablet7); + } + + @Test + public void testSinkTsFileFormat9() throws Exception { + doTest(this::insertTablet8); + } + + @Test + public void testSinkTsFileFormat10() throws Exception { + doTest(this::insertTablet9); + } + + private void doTest(BiConsumer>, Map>> consumer) + throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; + + Map> testResult = new HashMap<>(); + Map> test1Result = new HashMap<>(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + for (int i = 0; i < 5; i++) { + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test0"); + TableModelUtils.createDataBaseAndTable(senderEnv, "test" + i, "test1"); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (1, 1)", "flush"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.realtime.mode", "forced-log"); + extractorAttributes.put("capture.table", "true"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("extractor.database-name", "test.*"); + extractorAttributes.put("extractor.table-name", "test.*"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.format", "tsfile"); + connectorAttributes.put("connector.realtime-first", "true"); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)) + .getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList("insert into root.vehicle.d0(time, s1) values (2, 1)", "flush"))) { + return; + } + + consumer.accept(testResult, test1Result); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet(new HashSet<>(Arrays.asList("1,1.0,", "2,1.0,"))), + handleFailure); + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "insert into root.vehicle.d0(time, s1) values (4, 1)", + "insert into root.vehicle.d0(time, s1) values (3, 1), (0, 1)", + "flush"))) { + return; + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.**", + "Time,root.vehicle.d0.s1,", + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), + handleFailure); + } + + for (Map.Entry> entry : testResult.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData( + "test0", entry.getKey(), set.size(), receiverEnv, handleFailure); + TableModelUtils.assertData("test0", entry.getKey(), set, receiverEnv, handleFailure); + } + + for (Map.Entry> entry : test1Result.entrySet()) { + final Set set = new HashSet<>(); + entry + .getValue() + .forEach( + tablet -> { + set.addAll(TableModelUtils.generateExpectedResults(tablet)); + }); + TableModelUtils.assertCountData( + "test1", entry.getKey(), set.size(), receiverEnv, handleFailure); + TableModelUtils.assertData("test1", entry.getKey(), set, receiverEnv, handleFailure); + } + } + + private void insertTablet1( + final Map> testResult, final Map> test1Result) { + + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 10; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 0, 10, false, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet2( + final Map> testResult, final Map> test1Result) { + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 10; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 10, false, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet3( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + final String dataBaseName = "test" + j % 2; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, + deviceIDStartIndex, + deviceIDEndIndex, + deviceIDStartIndex, + deviceIDEndIndex, + false, + true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet4( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, + deviceIDStartIndex, + deviceIDEndIndex, + deviceIDStartIndex, + deviceIDEndIndex, + false, + false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet5( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + for (int j = 0; j < 25; j++) { + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + final String dataBaseName = "test" + j % 2; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, 0, 10, deviceIDStartIndex, deviceIDEndIndex, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet6( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } + + private void insertTablet7( + final Map> testResult, final Map> test1Result) { + + final Random random = new Random(); + int deviceIDStartIndex = 0; + int deviceIDEndIndex = 100; + deviceIDStartIndex = random.nextInt(1 << 16) - 10; + deviceIDEndIndex = deviceIDStartIndex + 10; + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, deviceIDStartIndex, deviceIDEndIndex, 100, 110, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet8( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + int deviceIDStartIndex = random.nextInt(1 << 16); + int deviceIDEndIndex = deviceIDStartIndex + 10; + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTablet( + tableName, 100, 110, deviceIDStartIndex, deviceIDEndIndex, false, true); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + deviceIDStartIndex += 2; + deviceIDEndIndex += 2; + } + } + + private void insertTablet9( + final Map> testResult, final Map> test1Result) { + final Random random = new Random(); + for (int j = 0; j < 25; j++) { + final String dataBaseName = "test" + j % 2; + for (int i = 0; i < 5; i++) { + final String tableName = "test" + i; + Tablet tablet = + TableModelUtils.generateTabletDeviceIDAllIsNull(tableName, 100, 110, 10, false); + TableModelUtils.insertTablet(dataBaseName, tablet, senderEnv); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + Map> map = j % 2 == 0 ? testResult : test1Result; + map.computeIfAbsent(tableName, k -> new ArrayList<>()).add(tablet); + } + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java new file mode 100644 index 000000000000..f6573728cf29 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelTestIT { + + @Test + public void testDoubleLivingInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.table'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'forwarding-pipe-requests'='true'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + // combination of org.apache.iotdb.pipe.it.tablemodel.IoTDBPipeLifeCycleIT.testDoubleLiving and + // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving + @Test + public void testBasicDoubleLiving() { + boolean insertResult; + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + }; + + // insertion on sender + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on sender + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p2", senderDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on receiver + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 400, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + + // restart cluster + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + // insertion on receiver + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + for (int i = 400; i < 500; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 500, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); + } + + @Test + public void testDoubleLivingIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java new file mode 100644 index 000000000000..57a3be38e217 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { + + @Test + public void testWritePipeIsolation() throws Exception { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Start pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(true)) + .getCode()); + + // Stop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + + // Alter pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setIsTableModel(true)) + .getCode()); + + // Drop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(false)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(true)).getCode()); + } + } + + @Test + public void testReadPipeIsolation() { + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } + + @Test + public void testCaptureTreeAndTableIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Drop pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } + + @Test + public void testCaptureCornerCases() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe but capture table data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe but capture tree data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Create pipe with capture.tree and capture.table set to false + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java index c801cd25c622..aae3626ee531 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java @@ -69,7 +69,7 @@ private void testInsertNullValueTemplate( TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); if (insertType == InsertType.SESSION_INSERT_TABLET) { - TableModelUtils.insertDataByTablet("test", "test", 0, 200, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 0, 200, senderEnv, true); } else if (insertType == InsertType.SQL_INSERT) { TableModelUtils.insertData("test", "test", 0, 200, senderEnv, true); } @@ -96,7 +96,7 @@ private void testInsertNullValueTemplate( } if (insertType == InsertType.SESSION_INSERT_TABLET) { - TableModelUtils.insertDataByTablet("test", "test", 200, 400, senderEnv, true); + TableModelUtils.insertTablet("test", "test", 200, 400, senderEnv, true); } else if (insertType == InsertType.SQL_INSERT) { TableModelUtils.insertData("test", "test", 200, 400, senderEnv, true); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java index 5c5cbcd7b950..4c80f0ae6731 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java @@ -303,11 +303,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -318,10 +321,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java index 24b1c92563b6..09658a583b08 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java @@ -89,7 +89,8 @@ public void testValidPipeName() throws Exception { } } - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; for (final String pipeName : expectedPipeNames) { Assert.assertTrue( showPipeResult.stream() @@ -106,7 +107,7 @@ public void testValidPipeName() throws Exception { } } - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult = client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -124,7 +125,7 @@ public void testRevertParameterOrder() { String.format( "create pipe p1" + " with source ( " - + "'capture.table'='test'," + + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," + "'mode.streaming'='true'," @@ -177,7 +178,8 @@ public void testRevertStageOrder() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -249,7 +251,8 @@ public void testMissingStage() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -315,7 +318,8 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } @@ -522,7 +526,8 @@ public void testBrackets() throws Exception { } catch (Exception ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -540,6 +545,7 @@ public void testShowPipeWithWrongPipeName() throws Exception { final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); @@ -578,10 +584,12 @@ public void testShowPipeWithWrongPipeName() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(3, showPipeResult.size()); - showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1")).pipeInfoList; + showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true).setPipeName("p1")).pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -589,7 +597,9 @@ public void testShowPipeWithWrongPipeName() throws Exception { // Show all pipes whose connector is also used by p1. // p1 and p2 share the same connector parameters, so they have the same connector. showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setPipeName("p1").setWhereClause(true)) + .pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -678,7 +688,8 @@ public void testInclusionPattern() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java index 4145fc9a5d2a..b3e40bb8a10d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java @@ -56,12 +56,13 @@ import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2TableModel.class}) public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelTestIT { - private static final int generateDataSize = 100; + private static final int generateDataSize = 1000; @Test public void insertTablet() { @@ -90,8 +91,8 @@ private SessionDataSet query( param.append(schema.getMeasurementName()); param.append(','); } - sql = sql + param.substring(0, param.length() - 1); - sql = sql + " from " + tableName + " ORDER BY time ASC"; + + sql = sql + param + "time from " + tableName + " ORDER BY time ASC"; session.executeNonQueryStatement("use test"); return session.executeQueryStatement(sql); } @@ -106,7 +107,8 @@ private void prepareTypeConversionTest( createDatabaseAndTable(measurementSchemas, false, tablet.getColumnTypes(), receiverEnv); try (ITableSession senderSession = senderEnv.getTableSessionConnection(); ITableSession receiverSession = receiverEnv.getTableSessionConnection()) { - + senderSession.executeNonQueryStatement("use test"); + receiverSession.executeNonQueryStatement("use test"); if (isTsFile) { // Send TsFile data to receiver executeDataWriteOperation.accept(senderSession, receiverSession, tablet); @@ -121,7 +123,7 @@ private void prepareTypeConversionTest( } // Verify receiver data - long timeoutSeconds = 30; + long timeoutSeconds = 600; List> expectedValues = generateTabletResultSetForTable(tablet, measurementSchemas); await() @@ -140,10 +142,10 @@ private void prepareTypeConversionTest( fail(e.getMessage()); } }); - tablet.reset(); } catch (Exception e) { fail(e.getMessage()); } + tablet.reset(); } private void createDatabaseAndTable( @@ -174,16 +176,15 @@ private void createDataPipe(boolean isTSFile) { String sql = String.format( "create pipe test" - + " with source ('source'='iotdb-source','realtime.mode'='%s','realtime.enable'='%s','history.enable'='%s')" + + " with source ('source'='iotdb-source','realtime.mode'='%s')" + " with processor ('processor'='do-nothing-processor')" + " with sink ('node-urls'='%s:%s','batch.enable'='false','sink.format'='%s')", isTSFile ? "file" : "forced-log", - !isTSFile, - isTSFile, receiverEnv.getIP(), receiverEnv.getPort(), isTSFile ? "tsfile" : "tablet"); - TestUtils.tryExecuteNonQueriesWithRetry(senderEnv, Collections.singletonList(sql)); + TestUtils.tryExecuteNonQueriesWithRetry( + null, BaseEnv.TABLE_SQL_DIALECT, senderEnv, Collections.singletonList(sql)); } private void validateResultSet( @@ -193,11 +194,13 @@ private void validateResultSet( while (dataSet.hasNext()) { RowRecord record = dataSet.next(); List fields = record.getFields(); - - assertEquals(record.getTimestamp(), timestamps[index]); List rowValues = values.get(index++); for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); + if (field.getDataType() == null) { + assertNull(rowValues.get(i)); + continue; + } switch (field.getDataType()) { case INT64: case TIMESTAMP: @@ -207,11 +210,9 @@ private void validateResultSet( assertEquals(field.getDateV(), rowValues.get(i)); break; case BLOB: - assertEquals(field.getBinaryV(), rowValues.get(i)); - break; case TEXT: case STRING: - assertEquals(field.getStringValue(), rowValues.get(i)); + assertEquals(field.getBinaryV(), rowValues.get(i)); break; case INT32: assertEquals(field.getIntV(), (int) rowValues.get(i)); @@ -283,13 +284,11 @@ private void createTestDataForTimestamp(Tablet tablet, int j) { } } - private long[] createTestDataForTimestamp() { + private void createTestDataForTimeColumn(Tablet tablet) { long time = new Date().getTime(); - long[] data = new long[generateDataSize]; - for (int i = 0; i < data.length; i++) { - data[i] = time++; + for (int i = 0; i < generateDataSize; i++) { + tablet.addTimestamp(i, time++); } - return data; } private void createTestDataForDate(Tablet tablet, int j) { @@ -346,19 +345,35 @@ private List> generateTabletResultSetForTable( switch (sourceType) { case INT64: case TIMESTAMP: - value = ValueConverter.convert(sourceType, targetType, ((long[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((long[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case INT32: - value = ValueConverter.convert(sourceType, targetType, ((int[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((int[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case DOUBLE: - value = ValueConverter.convert(sourceType, targetType, ((double[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((double[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case FLOAT: - value = ValueConverter.convert(sourceType, targetType, ((float[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((float[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case DATE: @@ -366,24 +381,39 @@ private List> generateTabletResultSetForTable( ValueConverter.convert( sourceType, targetType, - DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); + tablet.bitMaps[j].isMarked(i) + ? null + : DateUtils.parseDateExpressionToInt(((LocalDate[]) values[j])[i])); insertRecord.add(convert(value, targetType)); break; case TEXT: case STRING: - value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case BLOB: - value = ValueConverter.convert(sourceType, targetType, ((Binary[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((Binary[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; case BOOLEAN: - value = ValueConverter.convert(sourceType, targetType, ((boolean[]) values[j])[i]); + value = + ValueConverter.convert( + sourceType, + targetType, + tablet.bitMaps[j].isMarked(i) ? null : ((boolean[]) values[j])[i]); insertRecord.add(convert(value, targetType)); break; } } + insertRecord.add(tablet.timestamps[i]); insertRecords.add(insertRecord); } @@ -391,12 +421,15 @@ private List> generateTabletResultSetForTable( } private Object convert(Object value, TSDataType targetType) { + if (value == null) { + return null; + } switch (targetType) { case DATE: return DateUtils.parseIntToLocalDate((Integer) value); case TEXT: case STRING: - return new String(((Binary) value).getValues(), TSFileConfig.STRING_CHARSET); + return value; } return value; } @@ -417,7 +450,7 @@ private Tablet generateTabletAndMeasurementSchema( columnTypes, generateDataSize); tablet.initBitMaps(); - tablet.timestamps = createTestDataForTimestamp(); + createTestDataForTimeColumn(tablet); for (int i = 0; i < pairs.size(); i++) { MeasurementSchema schema = pairs.get(i).left; switch (schema.getType()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java index 9ac09b2199a2..d4f689187bfb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java @@ -200,13 +200,15 @@ public void testReceiverNotLoadWhenIdColumnMismatch() throws Exception { } Set expectedResSet = new java.util.HashSet<>(); - expectedResSet.add("1970-01-01T00:00:00.002Z,d3,d4,blue2,20,"); - expectedResSet.add("1970-01-01T00:00:00.001Z,d3,d4,red2,10,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,d3,d4,blue2,20,null,null,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,d3,d4,red2,10,null,null,null,null,"); + expectedResSet.add("1970-01-01T00:00:00.002Z,null,null,null,null,d1,d2,blue,2,"); + expectedResSet.add("1970-01-01T00:00:00.001Z,null,null,null,null,d1,d2,red,1,"); // make sure data are not transferred TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from t1", - "time,tag3,tag4,s3,s4,", + "time,tag3,tag4,s3,s4,tag1,tag2,s1,s2,", expectedResSet, "db", handleFailure); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java index 0d74e2e89201..a3c8c3757420 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java @@ -37,6 +37,8 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -64,21 +66,22 @@ */ public class TableModelUtils { - public static void createDataBaseAndTable(BaseEnv baseEnv, String table, String database) { + public static void createDataBaseAndTable( + final BaseEnv baseEnv, final String table, final String database) { try (Connection connection = baseEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("create database if not exists " + database); statement.execute("use " + database); statement.execute( - "CREATE TABLE " - + table - + "(s0 string tag, s1 int64 field, s2 float field, s3 string field, s4 timestamp field, s5 int32 field, s6 double field, s7 date field, s8 text field )"); + String.format( + "CREATE TABLE %s(s0 string tag, s1 string tag, s2 string tag, s3 string tag,s4 int64 field, s5 float field, s6 string field, s7 timestamp field, s8 int32 field, s9 double field, s10 date field, s11 text field )", + table)); } catch (Exception e) { fail(e.getMessage()); } } - public static void createDataBase(BaseEnv baseEnv, String database) { + public static void createDataBase(final BaseEnv baseEnv, final String database) { try (Connection connection = baseEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { statement.execute("create database if not exists " + database); @@ -88,101 +91,118 @@ public static void createDataBase(BaseEnv baseEnv, String database) { } public static boolean insertData( - String dataBaseName, String tableName, int start, int end, BaseEnv baseEnv) { + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv) { List list = new ArrayList<>(end - start + 1); for (int i = start; i < end; ++i) { list.add( String.format( - "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, time) values ('t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", - tableName, i, i, i, i, i, i, i, getDateStr(i), i, i)); + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); } list.add("flush"); - if (!TestUtils.tryExecuteNonQueriesWithRetry( - dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list)) { - return false; - } - return true; + return TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); } public static boolean insertData( - String dataBaseName, - String tableName, - int start, - int end, - BaseEnv baseEnv, - boolean allowNullValue) { + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final boolean allowNullValue) { List list = new ArrayList<>(end - start + 1); - Object[] values = new Object[9]; + Object[] values = new Object[12]; Random random = new Random(); - // s0 string, s1 int64, s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + // s0 string, s1 string, s2 string, s3 string, s4 int64, s5 float, s6 string s7 timestamp, s8 + // int32, s9 double, s10 date, s11 text for (int i = start; i < end; ++i) { Arrays.fill(values, i); values[0] = String.format("'t%s'", i); - values[2] = String.format("%s.0", i); - values[3] = String.format("%s", i); - values[6] = String.format("%s.0", i); - values[7] = String.format("'%s'", getDateStr(i)); - values[8] = String.format("'%s'", i); + values[1] = String.format("'t%s'", i); + values[2] = String.format("'t%s'", i); + values[3] = String.format("'t%s'", i); + values[4] = String.format("%s", i); + values[5] = String.format("%s.0", i); + values[6] = String.format("%s", i); + values[7] = String.format("%s", i); + values[8] = String.format("%s", i); + values[9] = String.format("%s.0", i); + values[10] = String.format("'%s'", getDateStr(i)); + values[11] = String.format("'%s'", i); if (allowNullValue) { values[random.nextInt(9)] = "null"; } list.add( String.format( - "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", - tableName, values[0], values[1], values[2], values[3], values[4], values[5], - values[6], values[7], values[8], i)); - } - if (!TestUtils.tryExecuteNonQueriesWithRetry( - dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list)) { - return false; + "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8,s9, s10, s11, time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + tableName, + values[0], + values[1], + values[2], + values[3], + values[4], + values[5], + values[6], + values[7], + values[8], + values[9], + values[10], + values[11], + i)); } - return true; + return TestUtils.tryExecuteNonQueriesWithRetry( + dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); } public static boolean insertDataNotThrowError( - String dataBaseName, String tableName, int start, int end, BaseEnv baseEnv) { + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv) { List list = new ArrayList<>(end - start + 1); for (int i = start; i < end; ++i) { list.add( String.format( - "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, time) values ('t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", - tableName, i, i, i, i, i, i, i, getDateStr(i), i, i)); + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); } return TestUtils.tryExecuteNonQueriesWithRetry( dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list); } public static boolean insertData( - String dataBaseName, - String tableName, - int start, - int end, - BaseEnv baseEnv, - DataNodeWrapper wrapper) { + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final DataNodeWrapper wrapper) { List list = new ArrayList<>(end - start + 1); for (int i = start; i < end; ++i) { list.add( String.format( - "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, time) values ('t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", - tableName, i, i, i, i, i, i, i, getDateStr(i), i, i)); + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i)); } list.add("flush"); - if (!TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( - baseEnv, wrapper, list, dataBaseName, BaseEnv.TABLE_SQL_DIALECT)) { - return false; - } - - return true; + return TestUtils.tryExecuteNonQueriesOnSpecifiedDataNodeWithRetry( + baseEnv, wrapper, list, dataBaseName, BaseEnv.TABLE_SQL_DIALECT); } - public static boolean insertDataByTablet( - String dataBaseName, - String tableName, - int start, - int end, - BaseEnv baseEnv, - boolean allowNullValue) { - final Tablet tablet = generateTablet(tableName, start, end, allowNullValue); + public static boolean insertTablet( + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv, + final boolean allowNullValue) { + final Tablet tablet = generateTablet(tableName, start, end, allowNullValue, true); ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(1); try (final ITableSession session = tableSessionPool.getSession()) { session.executeNonQueryStatement("use " + dataBaseName); @@ -195,8 +215,26 @@ public static boolean insertDataByTablet( } } + public static boolean insertTablet( + final String dataBaseName, final Tablet tablet, final BaseEnv baseEnv) { + try (ITableSessionPool tableSessionPool = baseEnv.getTableSessionPool(20); + final ITableSession session = tableSessionPool.getSession()) { + session.executeNonQueryStatement("use " + dataBaseName); + session.insert(tablet); + session.executeNonQueryStatement("flush"); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + public static void deleteData( - String dataBaseName, String tableName, int start, int end, BaseEnv baseEnv) { + final String dataBaseName, + final String tableName, + final int start, + final int end, + final BaseEnv baseEnv) { List list = new ArrayList<>(end - start + 1); list.add( String.format("delete from %s where time >= %s and time <= %s", tableName, start, end)); @@ -206,19 +244,21 @@ public static void deleteData( } } - public static Set generateExpectedResults(int start, int end) { + // s0 string, s1 string, s2 string, s3 string, s4 int64, s5 float, s6 string s7 timestamp, s8 + // int32, s9 double, s10 date, s11 text + public static Set generateExpectedResults(final int start, final int end) { Set expectedResSet = new HashSet<>(); for (int i = start; i < end; ++i) { final String time = RpcUtils.formatDatetime("default", "ms", i, ZoneOffset.UTC); expectedResSet.add( String.format( - "t%d,%d,%d.0,%d,%s,%d,%d.0,%s,%s,%s,", - i, i, i, i, time, i, i, getDateStr(i), i, time)); + "t%s,t%s,t%s,t%s,%s,%s.0,%s,%s,%d,%d.0,%s,%s,%s,", + i, i, i, i, i, i, i, time, i, i, getDateStr(i), i, time)); } return expectedResSet; } - public static Set generateExpectedResults(Tablet tablet) { + public static Set generateExpectedResults(final Tablet tablet) { Set expectedResSet = new HashSet<>(); List schemas = tablet.getSchemas(); for (int i = 0; i < tablet.getRowSize(); i++) { @@ -260,12 +300,15 @@ public static Set generateExpectedResults(Tablet tablet) { case FLOAT: stringBuffer.append(((float[]) tablet.values[j])[i]); stringBuffer.append(","); + break; case INT32: stringBuffer.append(((int[]) tablet.values[j])[i]); stringBuffer.append(","); + break; case INT64: stringBuffer.append(((long[]) tablet.values[j])[i]); stringBuffer.append(","); + break; } } String time = RpcUtils.formatDatetime("default", "ms", tablet.timestamps[i], ZoneOffset.UTC); @@ -278,19 +321,23 @@ public static Set generateExpectedResults(Tablet tablet) { } public static String generateHeaderResults() { - return "s0,s3,s2,s1,s4,s5,s6,s7,s8,time,"; + return "s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time,"; } - public static String getQuerySql(String table) { - return "select s0,s3,s2,s1,s4,s5,s6,s7,s8,time from " + table; + public static String getQuerySql(final String table) { + return "select s0,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10,s11,time from " + table; } - public static String getQueryCountSql(String table) { + public static String getQueryCountSql(final String table) { return "select count(*) from " + table; } public static void assertData( - String database, String table, int start, int end, BaseEnv baseEnv) { + final String database, + final String table, + final int start, + final int end, + final BaseEnv baseEnv) { TestUtils.assertDataEventuallyOnEnv( baseEnv, TableModelUtils.getQuerySql(table), @@ -300,12 +347,27 @@ public static void assertData( } public static void assertData( - String database, - String table, - int start, - int end, - BaseEnv baseEnv, - Consumer handleFailure) { + final String database, + final String table, + final Set expectedResults, + final BaseEnv baseEnv, + final Consumer handleFailure) { + TestUtils.assertDataEventuallyOnEnv( + baseEnv, + TableModelUtils.getQuerySql(table), + TableModelUtils.generateHeaderResults(), + expectedResults, + database, + handleFailure); + } + + public static void assertData( + final String database, + final String table, + final int start, + final int end, + final BaseEnv baseEnv, + final Consumer handleFailure) { TestUtils.assertDataEventuallyOnEnv( baseEnv, TableModelUtils.getQuerySql(table), @@ -315,7 +377,8 @@ public static void assertData( handleFailure); } - public static void assertData(String database, String table, Tablet tablet, BaseEnv baseEnv) { + public static void assertData( + final String database, final String table, final Tablet tablet, final BaseEnv baseEnv) { TestUtils.assertDataEventuallyOnEnv( baseEnv, TableModelUtils.getQuerySql(table), @@ -324,18 +387,23 @@ public static void assertData(String database, String table, Tablet tablet, Base database); } - public static boolean hasDataBase(String database, BaseEnv baseEnv) { + public static boolean hasDataBase(final String database, final BaseEnv baseEnv) { TestUtils.assertDataEventuallyOnEnv(baseEnv, "", "", Collections.emptySet(), database); return true; } - public static void assertCountData(String database, String table, int count, BaseEnv baseEnv) { + public static void assertCountData( + final String database, final String table, final int count, final BaseEnv baseEnv) { TestUtils.assertDataEventuallyOnEnv( baseEnv, getQueryCountSql(table), "_col0,", Collections.singleton(count + ","), database); } public static void assertCountData( - String database, String table, int count, BaseEnv baseEnv, Consumer handleFailure) { + final String database, + final String table, + final int count, + final BaseEnv baseEnv, + final Consumer handleFailure) { TestUtils.executeNonQueryWithRetry(baseEnv, "flush"); TestUtils.assertDataEventuallyOnEnv( baseEnv, @@ -346,7 +414,7 @@ public static void assertCountData( handleFailure); } - public static String getDateStr(int value) { + public static String getDateStr(final int value) { Date date = new Date(value); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); try { @@ -356,7 +424,7 @@ public static String getDateStr(int value) { } } - public static LocalDate getDate(int value) { + public static LocalDate getDate(final int value) { Date date = new Date(value); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); try { @@ -368,20 +436,30 @@ public static LocalDate getDate(int value) { } public static Tablet generateTablet( - String tableName, int start, int end, boolean allowNullValue) { + final String tableName, + final int start, + final int end, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); - schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); - schemaList.add(new MeasurementSchema("s2", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); - schemaList.add(new MeasurementSchema("s4", TSDataType.TIMESTAMP)); - schemaList.add(new MeasurementSchema("s5", TSDataType.INT32)); - schemaList.add(new MeasurementSchema("s6", TSDataType.DOUBLE)); - schemaList.add(new MeasurementSchema("s7", TSDataType.DATE)); - schemaList.add(new MeasurementSchema("s8", TSDataType.TEXT)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); final List columnTypes = Arrays.asList( + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, Tablet.ColumnCategory.TAG, Tablet.ColumnCategory.FIELD, Tablet.ColumnCategory.FIELD, @@ -400,30 +478,315 @@ public static Tablet generateTablet( end - start); tablet.initBitMaps(); Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; - // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text for (long row = 0; row < end - start; row++) { - int randomNumber = allowNullValue ? random.nextInt(9) : 9; + int randomNumber = allowNullValue ? random.nextInt(12) : 12; long value = start + row; int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, value); tablet.addValue( - "s0", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); - tablet.addValue("s1", rowIndex, value); - tablet.addValue("s2", rowIndex, (value * 1.0f)); + "s0", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); tablet.addValue( - "s3", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + "s2", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", rowIndex, new Binary(String.format("t%s", value).getBytes(StandardCharsets.UTF_8))); tablet.addValue("s4", rowIndex, value); - tablet.addValue("s5", rowIndex, (int) value); - tablet.addValue("s6", rowIndex, value * 0.1); - tablet.addValue("s7", rowIndex, getDate((int) value)); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); tablet.addValue( - "s8", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); - if (randomNumber < 9) { + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 11) { tablet.addValue("s" + randomNumber, rowIndex, null); } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); } return tablet; } + + public static Tablet generateTablet( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int start, + final int end, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * (end - start)); + tablet.initBitMaps(); + final Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + for (long row = start; row < end; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, row); + tablet.addValue( + "s0", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s2", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, row); + tablet.addValue("s5", rowIndex, (row * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, row); + tablet.addValue("s8", rowIndex, (int) row); + tablet.addValue("s9", rowIndex, row * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) row)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static Tablet generateTablet( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int deviceDataSize, + final boolean allowNullValue, + final boolean allowNullDeviceColumn) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * deviceDataSize); + tablet.initBitMaps(); + final Random random = new Random(); + int nullDeviceIndex = allowNullDeviceColumn ? random.nextInt(4) : 4; + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + long value = random.nextInt(1 << 16); + for (long row = 0; row < deviceDataSize; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + value += random.nextInt(100); + tablet.addTimestamp(rowIndex, value); + tablet.addValue( + "s0", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s1", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s2", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue( + "s3", + rowIndex, + new Binary(String.format("t%s", deviceIndex).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + if (nullDeviceIndex < 4) { + tablet.addValue("s" + nullDeviceIndex, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static Tablet generateTabletDeviceIDAllIsNull( + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int deviceDataSize, + final boolean allowNullValue) { + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s2", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s5", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s6", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s7", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s8", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s9", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s10", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s11", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + (deviceEndIndex - deviceStartIndex) * deviceDataSize); + tablet.initBitMaps(); + final Random random = new Random(); + + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; deviceIndex++) { + // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + long value = random.nextInt(1 << 16); + for (long row = 0; row < deviceDataSize; row++) { + int randomNumber = allowNullValue ? random.nextInt(12) : 12; + int rowIndex = tablet.getRowSize(); + value += random.nextInt(100); + tablet.addTimestamp(rowIndex, value); + tablet.addValue("s0", rowIndex, null); + tablet.addValue("s1", rowIndex, null); + tablet.addValue("s2", rowIndex, null); + tablet.addValue("s3", rowIndex, null); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (value * 1.0f)); + tablet.addValue( + "s6", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s7", rowIndex, value); + tablet.addValue("s8", rowIndex, (int) value); + tablet.addValue("s9", rowIndex, value * 0.1); + tablet.addValue("s10", rowIndex, getDate((int) value)); + tablet.addValue( + "s11", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + if (randomNumber < 12) { + tablet.addValue("s" + randomNumber, rowIndex, null); + } + tablet.setRowSize(rowIndex + 1); + } + } + + return tablet; + } + + public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) { + try (final Connection connection = baseEnv.getConnection(sqlDialect); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } catch (final SQLException e) { + fail(e.getMessage()); + } + return 0; + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index b69a557b41aa..99dfe05db99b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -593,7 +593,7 @@ public void testDeleteTable() throws SQLException { try (ResultSet ignored = statement.executeQuery("SELECT * FROM vehicle" + testNum)) { fail("Exception expected"); } catch (SQLException e) { - assertEquals("701: Table 'test.vehicle12' does not exist", e.getMessage()); + assertEquals("550: Table 'test.vehicle12' does not exist.", e.getMessage()); } statement.execute( diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index 71b1eb4a8c0f..e751ab6423f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -464,7 +464,7 @@ public void testInsertMultiRowWithNull() throws SQLException { st1.execute("insert into wt14(time, s1, s2) values(100, null, 1), (101, null, 2)"); fail(); } catch (SQLException e) { - assertEquals("507: Table wt14 does not exist", e.getMessage()); + assertEquals("550: Table 'test.wt14' does not exist.", e.getMessage()); } try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT)) { try (Statement st2 = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java index c221dc401670..bf9e7f89db0c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/alignbydevice/IoTDBAlignByDeviceWithTemplateTableIT.java @@ -187,7 +187,7 @@ public void selectMeasurementNoFilterTest() { statement.executeQuery("SELECT s3,s1,s_null FROM table1 order by device_id")) { fail("should throw exception to indicate that s_null doesn't exist"); } catch (SQLException e) { - assertEquals("701: Column 's_null' cannot be resolved", e.getMessage()); + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); } } } catch (SQLException e) { @@ -277,7 +277,7 @@ public void selectWildcardWithFilterOrderByTimeTest() { statement.executeQuery("SELECT * FROM table1 WHERE s_null > 1 order by device_id")) { fail("should throw exception to indicate that s_null doesn't exist"); } catch (SQLException e) { - assertEquals("701: Column 's_null' cannot be resolved", e.getMessage()); + assertEquals("616: Column 's_null' cannot be resolved", e.getMessage()); } } } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java index 35e20f7edec1..f9018b65d98b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBFormatFunctionTableIT.java @@ -15,11 +15,17 @@ package org.apache.iotdb.relational.it.query.old.builtinfunction.scalar; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.Statement; @@ -27,6 +33,9 @@ import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +@Ignore +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) public class IoTDBFormatFunctionTableIT { private static final String DATABASE_NAME = "db"; @@ -36,12 +45,12 @@ public class IoTDBFormatFunctionTableIT { // normal cases "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, - "CREATE TABLE number_table(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT)", - "CREATE TABLE string_table(device_id STRING ID, s1 STRING MEASUREMENT, s2 TEXT MEASUREMENT)", - "CREATE TABLE boolean_table(device_id STRING ID, s1 BOOLEAN MEASUREMENT)", - "CREATE TABLE timestamp_table(device_id STRING ID, s1 TIMESTAMP MEASUREMENT)", - "CREATE TABLE date_table(device_id STRING ID, s1 DATE MEASUREMENT)", - "CREATE TABLE null_table(device_id STRING ID, s1 INT32 MEASUREMENT)", + "CREATE TABLE number_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD)", + "CREATE TABLE string_table(device_id STRING TAG, s1 STRING FIELD, s2 TEXT FIELD)", + "CREATE TABLE boolean_table(device_id STRING TAG, s1 BOOLEAN FIELD)", + "CREATE TABLE timestamp_table(device_id STRING TAG, s1 TIMESTAMP FIELD)", + "CREATE TABLE date_table(device_id STRING TAG, s1 DATE FIELD)", + "CREATE TABLE null_table(device_id STRING TAG, s1 INT32 FIELD)", // data for number series "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (10, 'd1', 1000000, 1000000, 3.1415926, 3.1415926)", "INSERT INTO number_table(time, device_id, s1, s2, s3, s4) VALUES (20, 'd1', 1, 1, 1234567.25, 1234567.25)", diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java index de0dbea13d9b..27d14c0e1fcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNoSelectExpressionAfterAnalyzedTableIT.java @@ -64,7 +64,7 @@ public void testAlignByDevice() { String[] retArray = new String[] {}; tableAssertTestFail( "select s2 from sg where s1>0 order by device", - "701: Column 's2' cannot be resolved", + "616: Column 's2' cannot be resolved", DATABASE_NAME); // TODO After Aggregation supported @@ -81,14 +81,14 @@ public void testAlignByDevice() { tableAssertTestFail( "select s1, s2 from sg where s1>0 order by device", - "701: Column 's2' cannot be resolved", + "616: Column 's2' cannot be resolved", DATABASE_NAME); } @Test public void testAlignByTime() { tableAssertTestFail( - "select s2 from sg where s1>0", "701: Column 's2' cannot be resolved", DATABASE_NAME); + "select s2 from sg where s1>0", "616: Column 's2' cannot be resolved", DATABASE_NAME); /*tableResultSetEqualTest("select count(s2) from sg where s1>0", expectedHeader, retArray,DATABASE_NAME);*/ } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java deleted file mode 100644 index a7ba0315d52b..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBNullValueFillTableIT.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.query.old.query; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; -import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; -import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualWithDescOrderTest; - -@Ignore -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBNullValueFillTableIT { - private static final String DATABASE_NAME = "test"; - - /** - * [root.sg1.d1 (aligned)] - * - *

Time, s1[INT32], s2[INT64], s3[FLOAT], s4[DOUBLE], s5[BOOLEAN], s6[TEXT]
- * 1, null, 1, null, 1.0, null, t1
- * 2, 2, 2, 2.0, 2.0, true, t2
- * 3, 3, null, 3.0, null, false, null
- * 4, null, 4, null, 4.0, null, t4
- * 5, 5, 5, 5.0, 5.0, false, t5
- * 6, null, 6, null, 6.0, false, null
- * 7, null, null, null, null, null, null
- * 8, 8, 8, 8.0, 8.0, true, t8
- * 9, 9, null, 9.0, null, true, null - * - *

[root.sg1.d2 (non-aligned)] - * - *

Time, s1[INT32], s2[INT64], s3[FLOAT], s4[DOUBLE], s5[BOOLEAN], s6[TEXT]
- * 1, 1, null, 1.0, null, true, null
- * 2, 2, 2, 2.0, 2.0, true, t2
- * 3, null, 3, null, 3.0, null, t3
- * 4, 4, null, 4.0, null, false, null
- * 5, 5, 5, 5.0, 5.0, false, t5
- * 6, 6, null, 6.0, null, false, null
- * 7, null, null, null, null, null, null
- * 8, 8, 8, 8.0, 8.0, true, t8
- * 9, null, 9, null, 9.0, null, t9 - */ - private static final String[] sqls = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "CREATE TABLE testNullFilter(device STRING ID, s1 INT32 MEASUREMENT, s2 BOOLEAN MEASUREMENT, s3 DOUBLE MEASUREMENT)", - "create aligned timeseries root.sg1.d1(s1 INT32, s2 INT64, s3 FLOAT, s4 DOUBLE, s5 BOOLEAN, s6 TEXT)", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(1, 1, 1.0, 't1')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(2, 2, 2, 2.0, 2.0, true, 't2')", - "insert into root.sg1.d1(time, s1, s3, s5) aligned values(3, 3, 3.0, false)", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(4, 4, 4.0, 't4')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(5, 5, 5, 5.0, 5.0, false, 't5')", - "insert into root.sg1.d1(time, s2, s4, s6) aligned values(6, 6, 6.0, 't6')", - "insert into root.sg1.d1(time, s1, s2, s3, s4, s5, s6) aligned values(8, 8, 8, 8.0, 8.0, true, 't8')", - "insert into root.sg1.d1(time, s1, s3, s5) aligned values(9, 9, 9.0, true)", - "create timeseries root.sg1.d2.s1 INT32", - "create timeseries root.sg1.d2.s2 INT64", - "create timeseries root.sg1.d2.s3 FLOAT", - "create timeseries root.sg1.d2.s4 DOUBLE", - "create timeseries root.sg1.d2.s5 BOOLEAN", - "create timeseries root.sg1.d2.s6 TEXT", - "insert into root.sg1.d2(time, s1, s3, s5) values(1, 1, 1.0, true)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(2, 2, 2, 2.0, 2.0, true, 't2')", - "insert into root.sg1.d2(time, s2, s4, s6) values(3, 3, 3.0, 't3')", - "insert into root.sg1.d2(time, s1, s3, s5) values(4, 4, 4.0, false)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(5, 5, 5, 5.0, 5.0, false, 't5')", - "insert into root.sg1.d2(time, s1, s3, s5) values(6, 6, 6.0, false)", - "insert into root.sg1.d2(time, s1, s2, s3, s4, s5, s6) values(8, 8, 8, 8.0, 8.0, true, 't8')", - "insert into root.sg1.d2(time, s2, s4, s6) values(9, 9, 9.0, 't9')" - }; - - private final String[] expectedHeader = - new String[] { - "Time", - "root.sg1.d1.s1", - "root.sg1.d1.s2", - "root.sg1.d1.s3", - "root.sg1.d1.s4", - "root.sg1.d1.s5", - "root.sg1.d1.s6" - }; - - private final String[] expectedAlignByDeviceHeader = - new String[] {"Time", "Device", "s1", "s2", "s3", "s4", "s5", "s6"}; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - prepareData(sqls); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void previousFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,2,3.0,2.0,false,t2,", - "4,3,4,3.0,4.0,false,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,5,6,5.0,6.0,false,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,8,9.0,8.0,true,t8," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(previous)", expectedHeader, retArray); - } - - @Test - public void previousDescFillTest() { - String[] retArray = - new String[] { - "9,9,null,9.0,null,true,null,", - "8,8,8,8.0,8.0,true,t8,", - "6,8,6,8.0,6.0,true,t6,", - "5,5,5,5.0,5.0,false,t5,", - "4,5,4,5.0,4.0,false,t4,", - "3,3,4,3.0,4.0,false,t4,", - "2,2,2,2.0,2.0,true,t2,", - "1,2,1,2.0,1.0,true,t1," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(previous) order by time desc", - expectedHeader, - retArray); - } - - @Test - public void previousFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "1,root.sg1.d1,null,1,null,1.0,null,t1,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d1,3,2,3.0,2.0,false,t2,", - "4,root.sg1.d1,3,4,3.0,4.0,false,t4,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d1,5,6,5.0,6.0,false,t6,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d1,9,8,9.0,8.0,true,t8,", - "1,root.sg1.d2,1,8,1.0,8.0,true,t8,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d2,2,3,2.0,3.0,true,t3,", - "4,root.sg1.d2,4,3,4.0,3.0,false,t3,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d2,6,5,6.0,5.0,false,t5,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d2,8,9,8.0,9.0,true,t9," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(previous) align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void previousDescFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "9,root.sg1.d1,9,null,9.0,null,true,null,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "6,root.sg1.d1,8,6,8.0,6.0,true,t6,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "4,root.sg1.d1,5,4,5.0,4.0,false,t4,", - "3,root.sg1.d1,3,4,3.0,4.0,false,t4,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "1,root.sg1.d1,2,1,2.0,1.0,true,t1,", - "9,root.sg1.d2,2,9,2.0,9.0,true,t9,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "6,root.sg1.d2,6,8,6.0,8.0,false,t8,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "4,root.sg1.d2,4,5,4.0,5.0,false,t5,", - "3,root.sg1.d2,4,3,4.0,3.0,false,t3,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "1,root.sg1.d2,1,2,1.0,2.0,true,t2," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(previous) order by device,time desc align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void linearFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,3,3.0,3.0,false,null,", - "4,4,4,4.0,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,6,6,6.0,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,null," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(linear)", expectedHeader, retArray); - } - - @Test - public void linearFillAlignByDeviceTest() { - String[] retArray = - new String[] { - "1,root.sg1.d1,null,1,null,1.0,null,t1,", - "2,root.sg1.d1,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d1,3,3,3.0,3.0,false,null,", - "4,root.sg1.d1,4,4,4.0,4.0,null,t4,", - "5,root.sg1.d1,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d1,6,6,6.0,6.0,null,t6,", - "8,root.sg1.d1,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d1,9,9,9.0,9.0,true,null,", - "1,root.sg1.d2,1,1,1.0,1.0,true,null,", - "2,root.sg1.d2,2,2,2.0,2.0,true,t2,", - "3,root.sg1.d2,3,3,3.0,3.0,null,t3,", - "4,root.sg1.d2,4,4,4.0,4.0,false,null,", - "5,root.sg1.d2,5,5,5.0,5.0,false,t5,", - "6,root.sg1.d2,6,6,6.0,6.0,false,null,", - "8,root.sg1.d2,8,8,8.0,8.0,true,t8,", - "9,root.sg1.d2,null,9,null,9.0,null,t9," - }; - resultSetEqualTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.* fill(linear) align by device", - expectedAlignByDeviceHeader, - retArray); - } - - @Test - public void intFillTest() { - String[] retArray = - new String[] { - "1,1000,1,1000.0,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,1000,3.0,1000.0,false,1000,", - "4,1000,4,1000.0,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,1000,6,1000.0,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,1000,9.0,1000.0,true,1000," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(1000)", expectedHeader, retArray); - } - - @Test - public void floatFillTest() { - String[] retArray = - new String[] { - "1,null,1,3.14,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,3.14,false,3.14,", - "4,null,4,3.14,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,3.14,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,3.14,true,3.14," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(3.14)", expectedHeader, retArray); - } - - @Test - public void booleanFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,true,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,null,false,true,", - "4,null,4,null,4.0,true,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,null,6.0,true,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,true," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill(true)", expectedHeader, retArray); - } - - @Test - public void textFillTest() { - String[] retArray = - new String[] { - "1,null,1,null,1.0,null,t1,", - "2,2,2,2.0,2.0,true,t2,", - "3,3,null,3.0,null,false,t0,", - "4,null,4,null,4.0,null,t4,", - "5,5,5,5.0,5.0,false,t5,", - "6,null,6,null,6.0,null,t6,", - "8,8,8,8.0,8.0,true,t8,", - "9,9,null,9.0,null,true,t0," - }; - resultSetEqualWithDescOrderTest( - "select s1, s2, s3, s4, s5, s6 from root.sg1.d1 fill('t0')", expectedHeader, retArray); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java index 31a8cb21aa21..a7ad4a07be95 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBResultSetTableIT.java @@ -144,7 +144,7 @@ public void columnTypeTest() { @Test public void emptyQueryTest1() { - tableAssertTestFail("select * from sg1", "701: Table 'test.sg1' does not exist", DATABASE_NAME); + tableAssertTestFail("select * from sg1", "550: Table 'test.sg1' does not exist", DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java index aee30ec1de2b..e17f873e116b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBSelectSchemaTableIT.java @@ -27,7 +27,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -49,7 +48,7 @@ public class IoTDBSelectSchemaTableIT { new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, - "CREATE TABLE sg(device STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 DOUBLE MEASUREMENT)", + "CREATE TABLE sg(device STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 DOUBLE FIELD)", "insert into sg(time, device, s1, s2, s3) values (1, 'd1', 1, 2, 3.0)" }; @@ -64,7 +63,6 @@ public static void tearDown() throws Exception { EnvFactory.getEnv().cleanClusterEnvironment(); } - @Ignore // TODO After sin supported @Test public void testSchemaExpression() { String[] expressions = { @@ -80,18 +78,9 @@ public void testSchemaExpression() { "sin(s1)+s1", "((s1+1)*2-1)%2+1.5+s2" }; - String[] completeExpressions = { - "s1+s2", - "-s1+s2", - "-(s1+s3)", - "not(s1>s2)", - "-(-s1)", - "(s1+s2)*s3", - "-2+s1", - "not true or s1>0", - "-(-1)+s1", - "sin(s1)+s1", - "((s1+1)*2-1)%2+1.5+s2", + String[] columnNames = { + "_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8", "_col9", "_col10", + "_col11", }; try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { @@ -115,8 +104,7 @@ public void testSchemaExpression() { assertEquals(1 + expressions.length, columnCount); for (int i = 0; i < expressions.length; ++i) { - assertEquals( - completeExpressions[i], resultSet.getMetaData().getColumnName(i + 2).replace(" ", "")); + assertEquals(columnNames[i], resultSet.getMetaData().getColumnName(i + 2).replace(" ", "")); } } catch (SQLException throwable) { fail(throwable.getMessage()); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index 7566703ca48e..21c53f12d07c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -4051,10 +4051,6 @@ public void modeTest() { @Test public void exceptionTest() { - tableAssertTestFail( - "select s1 from table1 where s2 in (select s2 from table1)", - "Only TableSubquery is supported now", - DATABASE_NAME); tableAssertTestFail( "select avg() from table1", "701: Aggregate functions [avg] should only have one argument", @@ -4159,6 +4155,14 @@ public void simpleTest() { expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,"}; + tableResultSetEqualTest( + "select distinct s1 < 0 from table1 where s1 is not null", + expectedHeader, + retArray, + DATABASE_NAME); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java index 466b0535b52f..0e6e2124f328 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/SubqueryDataSetUtils.java @@ -20,9 +20,9 @@ package org.apache.iotdb.relational.it.query.recent.subquery; public class SubqueryDataSetUtils { - protected static final String DATABASE_NAME = "subqueryTest"; - protected static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; - protected static final String[] CREATE_SQLS = + public static final String DATABASE_NAME = "subqueryTest"; + public static final String[] NUMERIC_MEASUREMENTS = new String[] {"s1", "s2", "s3", "s4"}; + public static final String[] CREATE_SQLS = new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, @@ -104,6 +104,12 @@ public class SubqueryDataSetUtils { + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", "INSERT INTO table2(time,device_id,s1,s2,s3,s4,s5) " + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + // table3 + "CREATE TABLE table3(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 TEXT FIELD, s7 STRING FIELD, s8 BLOB FIELD, s9 TIMESTAMP FIELD, s10 DATE FIELD)", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d01',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d01',40,40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", + "INSERT INTO table3(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:13:30.000+00:00,'d_null',30,30,30.0,30.0,true,'shanghai_huangpu_red_A_d01_30','shanghai_huangpu_red_A_d01_30',X'cafebabe30',2024-09-24T06:13:00.000+00:00,'2024-09-23')", + "INSERT INTO table3(time,device_id,s2,s3,s4,s5,s6,s7,s8,s9,s10) values (2024-09-24T06:14:30.000+00:00,'d_null',40,40.0,40.0,false,'shanghai_huangpu_red_A_d01_40','shanghai_huangpu_red_A_d01_40',X'cafebabe40',2024-09-24T06:14:00.000+00:00,'2024-09-24')", "FLUSH", "CLEAR ATTRIBUTE CACHE", }; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java new file mode 100644 index 000000000000..ad26fb555909 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedInPredicateSubqueryIT.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.CREATE_SQLS; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.DATABASE_NAME; +import static org.apache.iotdb.relational.it.query.recent.subquery.SubqueryDataSetUtils.NUMERIC_MEASUREMENTS; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBUncorrelatedInPredicateSubqueryIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setSortBufferSize(128 * 1024); + EnvFactory.getEnv().getConfig().getCommonConfig().setMaxTsBlockSizeInByte(4 * 1024); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(CREATE_SQLS); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInPredicateSubqueryInWhereClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: where s in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,", "40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery) + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd01')"; + retArray = new String[] {"50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery returns empty set + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s not in (subquery), subquery returns empty set. Should return all rows + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s not in (SELECT (%s) FROM table3 WHERE device_id = 'd_empty')"; + retArray = new String[] {"30,", "40,", "50,", "60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT min(%s) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"30,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains scalar subquery. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and %s in (SELECT (%s) from table3 WHERE device_id = 'd01' and (%s) > (select avg(%s) from table3 where device_id = 'd01'))"; + retArray = new String[] {"40,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format( + sql, measurement, measurement, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), subquery contains expression. + sql = + "SELECT cast(%s AS INT32) as %s FROM table1 WHERE device_id = 'd01' and cast(%s AS INT32) in (SELECT cast(((%s) + 30) AS INT32) from table3 WHERE device_id = 'd01')"; + retArray = new String[] {"60,", "70,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + expectedHeader = new String[] {measurement}; + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: where s in (subquery), s contains null value + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {"30,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: where s not in (subquery), s contains null value, the resutl should be empty + sql = "SELECT s1 FROM table3 WHERE device_id = 'd_null' and s1 not in (SELECT s1 from table3)"; + expectedHeader = new String[] {"s1"}; + retArray = new String[] {}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryInHavingClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: having s in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery) + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 30 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd01') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s in (subquery), subquery returns empty set + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 in (SELECT cast(s1 as INT64) from table3 where device_id = 'd010')"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = new String[] {}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + + // Test case: having s not in (subquery), subquery returns empty set, should return all rows + sql = + "SELECT device_id, count(*) from table1 group by device_id having count(*) + 25 not in (SELECT cast(s1 as INT64) from table3 where device_id = 'd11') and count(*) > 3"; + expectedHeader = new String[] {"device_id", "_col1"}; + retArray = + new String[] { + "d01,5,", "d03,5,", "d05,5,", "d07,5,", "d09,5,", "d11,5,", "d13,5,", "d15,5," + }; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement, measurement, measurement), + expectedHeader, + retArray, + DATABASE_NAME); + } + } + + @Test + public void testInPredicateSubqueryInSelectClause() { + String sql; + String[] expectedHeader; + String[] retArray; + + // Test case: select s in (subquery) + sql = + "SELECT %s in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "true,", "false,", "false,", "false,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s not in (subquery) + sql = + "SELECT %s not in (SELECT (%s) from table3 WHERE device_id = 'd01') from table1 where device_id = 'd01'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "false,", "true,", "true,", "true,"}; + for (String measurement : NUMERIC_MEASUREMENTS) { + tableResultSetEqualTest( + String.format(sql, measurement, measurement), expectedHeader, retArray, DATABASE_NAME); + } + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"true,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. The result should also be null when + // s is null. + sql = "SELECT s1 not in (SELECT s1 from table3) from table3 where device_id = 'd_null'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // Test case: select s in (subquery), s contains null value. 36 not in(30, 40, null) returns + // null + sql = "SELECT s1 not in (SELECT s1 from table3) from table1 where device_id = 'd02'"; + expectedHeader = new String[] {"_col0"}; + retArray = new String[] {"null,", "false,", "null,"}; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void testInPredicateSubqueryLegalityCheck() { + // Legality check: Multiple parentheses around subquery, this behaves the same as Trino + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 not in ((select s1 from table3 where device_id = 'd01'))", + "701: Scalar sub-query has returned multiple rows.", + DATABASE_NAME); + + // Legality check: Join key type mismatch.(left key is int and right key is double) + tableAssertTestFail( + "select s1 from table1 where device_id = 'd01' and s1 in (select s1 + 30.0 from table3 where device_id = 'd01')", + "701: Join key type mismatch", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select s1, s2 in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: Row Type is not supported for now. + tableAssertTestFail( + "select (s1, s2) in (select (s1, s2) from table1) from table1", + "701: Subquery must return only one column for now. Row Type is not supported for now.", + DATABASE_NAME); + + // Legality check: subquery can not be parsed(without parentheses) + tableAssertTestFail( + "select s1 from table1 where s1 in select s1 from table1", + "mismatched input", + DATABASE_NAME); + + // Legality check: subquery can not be parsed + tableAssertTestFail( + "select s1 from table1 where s1 in (select s1 from)", "mismatched input", DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java rename to integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java index 1d93e36197e8..d2bce2e0d4ae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/IoTDBUncorrelatedScalarSubqueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/subquery/uncorrelated/IoTDBUncorrelatedScalarSubqueryIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.relational.it.query.recent.subquery; +package org.apache.iotdb.relational.it.query.recent.subquery.uncorrelated; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -262,7 +262,7 @@ public void testScalarSubqueryAfterComparisonLegalityCheck() { // Legality check: subquery returns multiple rows (should fail) tableAssertTestFail( "select s1 from table1 where s1 = (select s1 from table1)", - "301: Scalar sub-query has returned multiple rows.", + "701: Scalar sub-query has returned multiple rows.", DATABASE_NAME); // Legality check: subquery can not be parsed diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 4b3738a07d16..c98421e27725 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -406,6 +406,47 @@ public void testInformationSchema() throws SQLException { "datanode_id,INT32,ATTRIBUTE,", "elapsed_time,FLOAT,ATTRIBUTE,", "statement,STRING,ATTRIBUTE,"))); + + // Test table query + statement.execute("create database test"); + statement.execute("create table test.test (a tag, b attribute, c int32)"); + + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from databases"), + "database,ttl(ms),schema_replication_factor,data_replication_factor,time_partition_interval,schema_region_group_num,data_region_group_num,", + new HashSet<>( + Arrays.asList( + "information_schema,INF,null,null,null,null,null,", + "test,INF,1,1,604800000,0,0,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("show devices from tables where status = 'USING'"), + "database,table_name,ttl(ms),status,", + new HashSet<>( + Arrays.asList( + "information_schema,databases,INF,USING,", + "information_schema,tables,INF,USING,", + "information_schema,columns,INF,USING,", + "information_schema,queries,INF,USING,", + "test,test,INF,USING,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from tables where status = 'USING'"), + "count(devices),", + Collections.singleton("5,")); + TestUtils.assertResultSetEqual( + statement.executeQuery( + "select * from columns where table_name = 'queries' or database = 'test'"), + "database,table_name,column_name,datatype,category,status,", + new HashSet<>( + Arrays.asList( + "information_schema,queries,query_id,STRING,TAG,USING,", + "information_schema,queries,start_time,TIMESTAMP,ATTRIBUTE,USING,", + "information_schema,queries,datanode_id,INT32,ATTRIBUTE,USING,", + "information_schema,queries,elapsed_time,FLOAT,ATTRIBUTE,USING,", + "information_schema,queries,statement,STRING,ATTRIBUTE,USING,", + "test,test,time,TIMESTAMP,TIME,USING,", + "test,test,a,STRING,TAG,USING,", + "test,test,b,STRING,ATTRIBUTE,USING,", + "test,test,c,INT32,FIELD,USING,"))); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java index 4652c78fec3b..578d807a6dde 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java @@ -153,14 +153,14 @@ public void testDevice() throws SQLException { statement.executeQuery("show devices from table2"); fail("Show devices shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { statement.executeQuery("count devices from table2"); fail("Count devices shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { @@ -176,7 +176,7 @@ public void testDevice() throws SQLException { statement.executeQuery("count devices from table0 where a = 1"); fail("Count devices shall fail for non-exist column"); } catch (final Exception e) { - assertEquals("701: Column 'a' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'a' cannot be resolved", e.getMessage()); } // Test fully qualified name @@ -193,7 +193,7 @@ public void testDevice() throws SQLException { statement.execute("update table2 set model = '1'"); fail("Update shall fail for non-exist table"); } catch (final Exception e) { - assertEquals("701: Table 'test.table2' does not exist.", e.getMessage()); + assertEquals("550: Table 'test.table2' does not exist.", e.getMessage()); } try { @@ -214,7 +214,7 @@ public void testDevice() throws SQLException { statement.execute("update table0 set col = '1'"); fail("Update shall fail for non-exist column"); } catch (final Exception e) { - assertEquals("701: Column 'col' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'col' cannot be resolved", e.getMessage()); } try { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index f730a21c22b5..d27e298f8776 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -422,14 +422,14 @@ public void testManageTable() { statement.executeQuery("select color from table2"); fail(); } catch (final SQLException e) { - assertEquals("701: Column 'color' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'color' cannot be resolved", e.getMessage()); } try { statement.executeQuery("select speed from table2"); fail(); } catch (final SQLException e) { - assertEquals("701: Column 'speed' cannot be resolved", e.getMessage()); + assertEquals("616: Column 'speed' cannot be resolved", e.getMessage()); } try { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index b2d45d554668..1f70341892b3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -23,8 +23,6 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -222,7 +220,6 @@ private static void insertRelationalTabletPerformanceTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -334,7 +331,6 @@ public void insertRelationalSqlTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void partialInsertSQLTest() throws IoTDBConnectionException, StatementExecutionException { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { // disable auto-creation only for this test @@ -365,7 +361,6 @@ public void partialInsertSQLTest() throws IoTDBConnectionException, StatementExe } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void partialInsertRelationalTabletTest() throws IoTDBConnectionException, StatementExecutionException { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { @@ -510,7 +505,6 @@ public void partialInsertRelationalTabletTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -587,7 +581,6 @@ public void insertRelationalTabletTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletWithCacheLeaderTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -669,7 +662,6 @@ public void insertRelationalTabletWithCacheLeaderTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateNontagColumnTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -744,7 +736,6 @@ public void autoCreateNontagColumnTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTableTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -803,7 +794,6 @@ public void autoCreateTableTest() throws IoTDBConnectionException, StatementExec } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTagColumnTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -882,7 +872,6 @@ public void autoCreateTagColumnTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoAdjustTagTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -950,7 +939,6 @@ public void autoAdjustTagTest() throws IoTDBConnectionException, StatementExecut } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlWithoutDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1008,7 +996,6 @@ public void insertRelationalSqlWithoutDBTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalSqlAnotherDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1057,7 +1044,6 @@ public void insertRelationalSqlAnotherDBTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertNonExistTableTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1070,7 +1056,7 @@ public void insertNonExistTableTest() 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db1.table13' does not exist.", e.getMessage()); } try { @@ -1080,13 +1066,12 @@ public void insertNonExistTableTest() 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db2.table13' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertNonExistDBTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { session.executeNonQueryStatement("USE \"db1\""); @@ -1098,13 +1083,12 @@ public void insertNonExistDBTest() throws IoTDBConnectionException, StatementExe 0, "tag:" + 0, "attr:" + 0, 0 * 1.0)); fail("Exception expected"); } catch (StatementExecutionException e) { - assertEquals("507: Table table13 does not exist", e.getMessage()); + assertEquals("550: Table 'db3.table13' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertWithoutMeasurementTest() throws IoTDBConnectionException, StatementExecutionException { try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { @@ -1356,7 +1340,6 @@ private Object genValue(TSDataType dataType, int i) { } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalTabletWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 14; @@ -1380,7 +1363,6 @@ public void insertRelationalTabletWithAutoCastTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void deleteTableAndWriteDifferentTypeTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 15; @@ -1410,7 +1392,6 @@ public void deleteTableAndWriteDifferentTypeTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void dropTableOfTheSameNameTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 16; @@ -1441,13 +1422,12 @@ public void dropTableOfTheSameNameTest() session.executeQueryStatement("select * from db2.table" + testNum + " order by time"); fail("expected exception"); } catch (StatementExecutionException e) { - assertEquals("701: Table 'db2.table16' does not exist", e.getMessage()); + assertEquals("550: Table 'db2.table16' does not exist.", e.getMessage()); } } } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void insertRelationalRowWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 17; @@ -1471,7 +1451,6 @@ public void insertRelationalRowWithAutoCastTest() } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTagColumnTest2() throws IoTDBConnectionException, StatementExecutionException { int testNum = 18; diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java index ea65984db0f9..4b63c5a4f2f6 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/parameter/FunctionArguments.java @@ -29,31 +29,31 @@ * implementation. It contains the data types of the child expressions, system attributes, etc. */ public class FunctionArguments { - private final List childExpressionDataTypes; + private final List argumentTypes; private final Map systemAttributes; public FunctionArguments( List childExpressionDataTypes, Map systemAttributes) { - this.childExpressionDataTypes = childExpressionDataTypes; + this.argumentTypes = childExpressionDataTypes; this.systemAttributes = systemAttributes; } /** - * Get the data types of the input children expressions. + * Get the data types of the arguments. * - * @return a list of data types of the input children expressions + * @return a list of data types of the arguments */ - public List getChildExpressionDataTypes() { - return childExpressionDataTypes; + public List getArgumentTypes() { + return argumentTypes; } /** - * Get the number of the input children expressions. + * Get the number of the arguments. * - * @return the number of the input children expressions + * @return the number of the arguments */ - public int getChildExpressionsSize() { - return childExpressionDataTypes.size(); + public int getArgumentsSize() { + return argumentTypes.size(); } /** @@ -63,7 +63,7 @@ public int getChildExpressionsSize() { * @return the data type of the input child expression at the specified index */ public Type getDataType(int index) { - return childExpressionDataTypes.get(index); + return argumentTypes.get(index); } /** diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java index 0ec121f96992..8f2b3aaa860a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java @@ -20,6 +20,7 @@ package org.apache.iotdb.session.subscription.payload; import org.apache.iotdb.rpc.subscription.exception.SubscriptionIncompatibleHandlerException; +import org.apache.iotdb.session.util.RetryUtils; import java.io.File; import java.io.IOException; @@ -56,8 +57,11 @@ public synchronized Path getPath() { */ public synchronized Path deleteFile() throws IOException { final Path sourcePath = getPath(); - Files.delete(sourcePath); - return sourcePath; + return RetryUtils.retryOnException( + () -> { + Files.delete(sourcePath); + return sourcePath; + }); } /** @@ -66,7 +70,7 @@ public synchronized Path deleteFile() throws IOException { * @throws IOException if an I/O error occurs */ public synchronized Path moveFile(final String target) throws IOException { - return this.moveFile(Paths.get(target)); + return RetryUtils.retryOnException(() -> this.moveFile(Paths.get(target))); } /** @@ -78,7 +82,8 @@ public synchronized Path moveFile(final Path target) throws IOException { if (!Files.exists(target.getParent())) { Files.createDirectories(target.getParent()); } - return Files.move(getPath(), target, StandardCopyOption.REPLACE_EXISTING); + return RetryUtils.retryOnException( + () -> Files.move(getPath(), target, StandardCopyOption.REPLACE_EXISTING)); } /** @@ -87,7 +92,7 @@ public synchronized Path moveFile(final Path target) throws IOException { * @throws IOException if an I/O error occurs */ public synchronized Path copyFile(final String target) throws IOException { - return this.copyFile(Paths.get(target)); + return RetryUtils.retryOnException(() -> this.copyFile(Paths.get(target))); } /** @@ -99,8 +104,13 @@ public synchronized Path copyFile(final Path target) throws IOException { if (!Files.exists(target.getParent())) { Files.createDirectories(target.getParent()); } - return Files.copy( - getPath(), target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + return RetryUtils.retryOnException( + () -> + Files.copy( + getPath(), + target, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES)); } @Override diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java new file mode 100644 index 000000000000..bd767d270b53 --- /dev/null +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/RetryUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.session.util; + +public class RetryUtils { + + public interface CallableWithException { + T call() throws E; + } + + public static final int MAX_RETRIES = 3; + + public static T retryOnException( + final CallableWithException callable) throws E { + int attempt = 0; + while (true) { + try { + return callable.call(); + } catch (Exception e) { + attempt++; + if (attempt >= MAX_RETRIES) { + throw e; + } + } + } + } + + private RetryUtils() { + // utility class + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 5d3bdcc6710e..15236010654d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -171,6 +171,8 @@ public enum ConfigPhysicalPlanType { PreDeleteColumn((short) 860), CommitDeleteColumn((short) 861), DescTable((short) 862), + ShowTable4InformationSchema((short) 863), + DescTable4InformationSchema((short) 864), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java new file mode 100644 index 000000000000..27b7bcad87c2 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/DescTable4InformationSchemaPlan.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class DescTable4InformationSchemaPlan extends ConfigPhysicalReadPlan { + public DescTable4InformationSchemaPlan() { + super(ConfigPhysicalPlanType.DescTable4InformationSchema); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java new file mode 100644 index 000000000000..4dc452330436 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTable4InformationSchemaPlan.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +public class ShowTable4InformationSchemaPlan extends ConfigPhysicalReadPlan { + + public ShowTable4InformationSchemaPlan() { + super(ConfigPhysicalPlanType.ShowTable4InformationSchema); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 020690f24651..20cd76e4a41d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -94,6 +94,13 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { } } + public PipeTableResp filter( + final Boolean whereClause, final String pipeName, final boolean isTableModel) { + final PipeTableResp resp = filter(whereClause, pipeName); + resp.allPipeMeta.removeIf(meta -> !meta.visibleUnder(isTableModel)); + return resp; + } + public TGetAllPipeInfoResp convertToTGetAllPipeInfoResp() throws IOException { final List pipeInformationByteBuffers = new ArrayList<>(); for (final PipeMeta pipeMeta : allPipeMeta) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTable4InformationSchemaResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTable4InformationSchemaResp.java new file mode 100644 index 000000000000..5fc5fe851df1 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/DescTable4InformationSchemaResp.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.response.table; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TTableColumnInfo; +import org.apache.iotdb.consensus.common.DataSet; + +import java.util.Map; + +public class DescTable4InformationSchemaResp implements DataSet { + private final TSStatus status; + private final Map> tableColumnInfoMap; + + public DescTable4InformationSchemaResp( + final TSStatus status, final Map> tableColumnInfoMap) { + this.status = status; + this.tableColumnInfoMap = tableColumnInfoMap; + } + + public TDescTable4InformationSchemaResp convertToTDescTable4InformationSchemaResp() { + return new TDescTable4InformationSchemaResp(status).setTableColumnInfoMap(tableColumnInfoMap); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/ShowTable4InformationSchemaResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/ShowTable4InformationSchemaResp.java new file mode 100644 index 000000000000..a0cbe756d5a0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/ShowTable4InformationSchemaResp.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.response.table; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; +import org.apache.iotdb.consensus.common.DataSet; + +import java.util.List; +import java.util.Map; + +public class ShowTable4InformationSchemaResp implements DataSet { + private final TSStatus status; + private final Map> databaseTableInfoMap; + + public ShowTable4InformationSchemaResp( + final TSStatus status, final Map> databaseTableInfoMap) { + this.status = status; + this.databaseTableInfoMap = databaseTableInfoMap; + } + + public TShowTable4InformationSchemaResp convertToTShowTable4InformationSchemaResp() { + return new TShowTable4InformationSchemaResp(status) + .setDatabaseTableInfoMap(databaseTableInfoMap); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 432784737c71..6f6d625768c4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -163,6 +163,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; @@ -218,12 +219,15 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; @@ -2191,18 +2195,18 @@ public TSStatus alterPipe(TAlterPipeReq req) { } @Override - public TSStatus startPipe(String pipeName) { + public TSStatus startPipe(TStartPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().startPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().startPipe(req) : status; } @Override - public TSStatus stopPipe(String pipeName) { + public TSStatus stopPipe(TStopPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().stopPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().stopPipe(req) : status; } @@ -2650,6 +2654,14 @@ public TShowTableResp showTables(final String database, final boolean isDetails) : new TShowTableResp(status); } + @Override + public TShowTable4InformationSchemaResp showTables4InformationSchema() { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? clusterSchemaManager.showTables4InformationSchema() + : new TShowTable4InformationSchemaResp(status); + } + @Override public TDescTableResp describeTable( final String database, final String tableName, final boolean isDetails) { @@ -2659,6 +2671,14 @@ public TDescTableResp describeTable( : new TDescTableResp(status); } + @Override + public TDescTable4InformationSchemaResp describeTable4InformationSchema() { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? clusterSchemaManager.describeTables4InformationSchema() + : new TDescTable4InformationSchemaResp(status); + } + @Override public TFetchTableResp fetchTables(final Map> fetchTableMap) { final TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index cae5fe9c956e..82dae052ef23 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -87,6 +87,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; @@ -141,10 +142,13 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; @@ -692,34 +696,39 @@ TDataPartitionTableResp getOrCreateDataPartition( * Alter Pipe. * * @param req Info about Pipe - * @return TSStatus + * @return {@link TSStatusCode#SUCCESS_STATUS} if altered the pipe successfully, {@link + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus alterPipe(TAlterPipeReq req); /** * Start Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if started the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus startPipe(String pipeName); + TSStatus startPipe(TStartPipeReq req); /** * Stop Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if stopped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus stopPipe(String pipeName); + TSStatus stopPipe(TStopPipeReq req); /** * Drop Pipe. * * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if dropped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus dropPipe(TDropPipeReq req); @@ -843,8 +852,12 @@ TDataPartitionTableResp getOrCreateDataPartition( TShowTableResp showTables(final String database, final boolean isDetails); + TShowTable4InformationSchemaResp showTables4InformationSchema(); + TDescTableResp describeTable( final String database, final String tableName, final boolean isDetails); + TDescTable4InformationSchemaResp describeTable4InformationSchema(); + TFetchTableResp fetchTables(final Map> fetchTableMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 7de39849e599..4aaf3ab46c3f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -31,6 +31,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -116,7 +118,7 @@ public boolean isLocked() { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus createPipe(TCreatePipeReq req) { - TSStatus status = null; + final TSStatus status; if (req.getPipeName().startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().createConsensusPipe(req); } else { @@ -130,6 +132,17 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { + final String pipeName = req.getPipeName(); + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to alter pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } final TSStatus status = configManager.getProcedureManager().alterPipe(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to alter pipe {}. Result status: {}.", req.getPipeName(), status); @@ -138,8 +151,8 @@ public TSStatus alterPipe(TAlterPipeReq req) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus startPipe(String pipeName) { - TSStatus status = null; + private TSStatus startPipe(String pipeName) { + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().startConsensusPipe(pipeName); } else { @@ -152,9 +165,21 @@ public TSStatus startPipe(String pipeName) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus stopPipe(String pipeName) { + public TSStatus startPipe(TStartPipeReq req) { + final String pipeName = req.getPipeName(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to start pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return startPipe(pipeName); + } + + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + private TSStatus stopPipe(String pipeName) { final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); - TSStatus status = null; + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().stopConsensusPipe(pipeName); } else { @@ -176,11 +201,32 @@ public TSStatus stopPipe(String pipeName) { return status; } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + public TSStatus stopPipe(TStopPipeReq req) { + final String pipeName = req.getPipeName(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to stop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return stopPipe(pipeName); + } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName); - TSStatus status = null; + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().dropConsensusPipe(pipeName); } else { @@ -189,23 +235,13 @@ public TSStatus dropPipe(TDropPipeReq req) { if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to drop pipe {}. Result status: {}.", pipeName, status); } - - final boolean isSetIfExistsCondition = - req.isSetIfExistsCondition() && req.isIfExistsCondition(); - // If the `IF EXISTS` condition is not set and the pipe does not exist before the delete - // operation, return an error status indicating that the pipe does not exist. - return isPipeExistedBeforeDrop || isSetIfExistsCondition - ? status - : RpcUtils.getStatus( - TSStatusCode.PIPE_NOT_EXIST_ERROR, - String.format( - "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + return status; } public TShowPipeResp showPipes(final TShowPipeReq req) { try { return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName) + .filter(req.whereClause, req.pipeName, req.isTableModel) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 353d97995b0b..9a6c06c26733 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -40,8 +40,10 @@ import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.read.table.DescTable4InformationSchemaPlan; import org.apache.iotdb.confignode.consensus.request.read.table.DescTablePlan; import org.apache.iotdb.confignode.consensus.request.read.table.FetchTablePlan; +import org.apache.iotdb.confignode.consensus.request.read.table.ShowTable4InformationSchemaPlan; import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetAllSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetAllTemplateSetInfoPlan; @@ -67,8 +69,10 @@ import org.apache.iotdb.confignode.consensus.response.database.CountDatabaseResp; import org.apache.iotdb.confignode.consensus.response.database.DatabaseSchemaResp; import org.apache.iotdb.confignode.consensus.response.partition.PathInfoResp; +import org.apache.iotdb.confignode.consensus.response.table.DescTable4InformationSchemaResp; import org.apache.iotdb.confignode.consensus.response.table.DescTableResp; import org.apache.iotdb.confignode.consensus.response.table.FetchTableResp; +import org.apache.iotdb.confignode.consensus.response.table.ShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.consensus.response.table.ShowTableResp; import org.apache.iotdb.confignode.consensus.response.template.AllTemplateSetInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; @@ -82,12 +86,14 @@ import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp; import org.apache.iotdb.confignode.rpc.thrift.TGetTemplateResp; import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.schemaengine.template.Template; @@ -1092,6 +1098,19 @@ public TShowTableResp showTables(final String database, final boolean isDetails) } } + public TShowTable4InformationSchemaResp showTables4InformationSchema() { + try { + return ((ShowTable4InformationSchemaResp) + configManager.getConsensusManager().read(new ShowTable4InformationSchemaPlan())) + .convertToTShowTable4InformationSchemaResp(); + } catch (final ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return new TShowTable4InformationSchemaResp(res); + } + } + public TDescTableResp describeTable( final String database, final String tableName, final boolean isDetails) { try { @@ -1108,6 +1127,19 @@ public TDescTableResp describeTable( } } + public TDescTable4InformationSchemaResp describeTables4InformationSchema() { + try { + return ((DescTable4InformationSchemaResp) + configManager.getConsensusManager().read(new DescTable4InformationSchemaPlan())) + .convertToTDescTable4InformationSchemaResp(); + } catch (final ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return new TDescTable4InformationSchemaResp(res); + } + } + public TFetchTableResp fetchTables(final Map> fetchTableMap) { try { return ((FetchTableResp) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 712d8e3bf4c5..07a75a0720bd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -322,10 +322,14 @@ public DataSet executeQueryPlan(final ConfigPhysicalReadPlan req) return clusterSchemaInfo.getTemplateSetInfo((GetTemplateSetInfoPlan) req); case ShowTable: return clusterSchemaInfo.showTables((ShowTablePlan) req); + case ShowTable4InformationSchema: + return clusterSchemaInfo.showTables4InformationSchema(); case FetchTable: return clusterSchemaInfo.fetchTables((FetchTablePlan) req); case DescTable: return clusterSchemaInfo.descTable((DescTablePlan) req); + case DescTable4InformationSchema: + return clusterSchemaInfo.descTable4InformationSchema(); case GetTriggerTable: return triggerInfo.getTriggerTable((GetTriggerTablePlan) req); case GetTriggerLocation: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 372209bfd741..8fe6659dc0c8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -187,23 +187,19 @@ private boolean checkBeforeCreatePipeInternal(final TCreatePipeReq createPipeReq throw new PipeException(exceptionMessage); } - public boolean checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) + public void checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) throws PipeException { acquireReadLock(); try { - return checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); + checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); } finally { releaseReadLock(); } } - private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) + private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) throws PipeException { if (!isPipeExisted(alterPipeRequest.getPipeName())) { - if (alterPipeRequest.isSetIfExistsCondition() && alterPipeRequest.isIfExistsCondition()) { - return false; - } - final String exceptionMessage = String.format( "Failed to alter pipe %s, %s", alterPipeRequest.getPipeName(), PIPE_NOT_EXIST_MSG); @@ -267,8 +263,6 @@ private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq .getAttribute()); } } - - return true; } public void checkBeforeStartPipe(final String pipeName) throws PipeException { @@ -351,6 +345,15 @@ public boolean isPipeExisted(final String pipeName) { } } + public boolean isPipeExisted(final String pipeName, final boolean isTableModel) { + acquireReadLock(); + try { + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel); + } finally { + releaseReadLock(); + } + } + private PipeStatus getPipeStatus(final String pipeName) { acquireReadLock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 069c5d99400f..debf75106994 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -70,14 +71,17 @@ import org.apache.iotdb.confignode.consensus.response.database.CountDatabaseResp; import org.apache.iotdb.confignode.consensus.response.database.DatabaseSchemaResp; import org.apache.iotdb.confignode.consensus.response.partition.PathInfoResp; +import org.apache.iotdb.confignode.consensus.response.table.DescTable4InformationSchemaResp; import org.apache.iotdb.confignode.consensus.response.table.DescTableResp; import org.apache.iotdb.confignode.consensus.response.table.FetchTableResp; +import org.apache.iotdb.confignode.consensus.response.table.ShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.consensus.response.table.ShowTableResp; import org.apache.iotdb.confignode.consensus.response.template.AllTemplateSetInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateSetInfoResp; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TTableColumnInfo; import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -1200,6 +1204,34 @@ public ShowTableResp showTables(final ShowTablePlan plan) { } } + public ShowTable4InformationSchemaResp showTables4InformationSchema() { + databaseReadWriteLock.readLock().lock(); + try { + return new ShowTable4InformationSchemaResp( + StatusUtils.OK, + tableModelMTree.getAllTables().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().stream() + .map( + pair -> { + final TTableInfo info = + new TTableInfo( + pair.getLeft().getTableName(), + pair.getLeft() + .getPropValue(TTL_PROPERTY) + .orElse(TTL_INFINITE)); + info.setState(pair.getRight().ordinal()); + return info; + }) + .collect(Collectors.toList())))); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + public FetchTableResp fetchTables(final FetchTablePlan plan) { databaseReadWriteLock.readLock().lock(); try { @@ -1241,6 +1273,51 @@ public DescTableResp descTable(final DescTablePlan plan) { } } + public DescTable4InformationSchemaResp descTable4InformationSchema() { + databaseReadWriteLock.readLock().lock(); + try { + return new DescTable4InformationSchemaResp( + StatusUtils.OK, + tableModelMTree.getAllDatabasePaths(true).stream() + .collect( + Collectors.toMap( + databasePath -> PathUtils.unQualifyDatabaseName(databasePath.getFullPath()), + databasePath -> { + try { + return tableModelMTree + .getAllTablesUnderSpecificDatabase(databasePath) + .stream() + .map( + pair -> { + try { + return tableModelMTree.getTableSchemaDetails( + databasePath, pair.getLeft().getTableName()); + } catch (final MetadataException ignore) { + // Table path must exist because the "getTableSchemaDetails()" + // is called in databaseReadWriteLock.readLock(). + } + return new Pair>(null, null); + }) + .collect( + Collectors.toMap( + pair -> pair.getLeft().getTableName(), + pair -> + new TTableColumnInfo() + .setTableInfo( + TsTableInternalRPCUtil.serializeSingleTsTable( + pair.getLeft())) + .setPreDeletedColumns(pair.getRight()))); + } catch (final MetadataException ignore) { + // Database path must exist because the "getAllDatabasePaths()" is called + // in databaseReadWriteLock.readLock(). + } + return Collections.emptyMap(); + }))); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + public Map> getAllUsingTables() { databaseReadWriteLock.readLock().lock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index f02a854f48c5..ff91998a8874 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -23,6 +23,9 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.exception.table.ColumnNotExistsException; +import org.apache.iotdb.commons.exception.table.TableAlreadyExistsException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; @@ -42,9 +45,6 @@ import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.exception.metadata.table.ColumnNotExistsException; -import org.apache.iotdb.db.exception.metadata.table.TableAlreadyExistsException; -import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.DatabaseCollector; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MNodeAboveDBCollector; @@ -752,6 +752,22 @@ public Map getSpecificTablesUnderSpecificDatabase( return result; } + public Map>> getAllTables() { + return getAllDatabasePaths(true).stream() + .collect( + Collectors.toMap( + databasePath -> PathUtils.unQualifyDatabaseName(databasePath.getFullPath()), + databasePath -> { + try { + return getAllTablesUnderSpecificDatabase(databasePath); + } catch (final MetadataException ignore) { + // Database path must exist because the "getAllDatabasePaths()" is called in + // databaseReadWriteLock.readLock(). + } + return Collections.emptyList(); + })); + } + public Map> getAllUsingTables() { return getAllDatabasePaths(true).stream() .collect( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java index 8b6aceb9c300..ba7d40abc53f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java @@ -96,9 +96,7 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx // We should execute checkBeforeAlterPipe before checking the pipe plugin. This method will // update the alterPipeRequest based on the alterPipeRequest and existing pipe metadata. - if (!pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest)) { - return false; - } + pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest); final PipeManager pipeManager = env.getConfigManager().getPipeManager(); pipeManager diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 5d3de28a26ac..219b68813e6b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -128,6 +128,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; @@ -192,12 +193,15 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTTLResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1111,12 +1115,22 @@ public TSStatus alterPipe(TAlterPipeReq req) { @Override public TSStatus startPipe(String pipeName) { - return configManager.startPipe(pipeName); + return configManager.startPipe(new TStartPipeReq().setPipeName(pipeName)); + } + + @Override + public TSStatus startPipeExtended(TStartPipeReq req) { + return configManager.startPipe(req); } @Override public TSStatus stopPipe(String pipeName) { - return configManager.stopPipe(pipeName); + return configManager.stopPipe(new TStopPipeReq().setPipeName(pipeName)); + } + + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) { + return configManager.stopPipe(req); } @Override @@ -1311,12 +1325,22 @@ public TShowTableResp showTables(final String database, final boolean isDetails) return configManager.showTables(database, isDetails); } + @Override + public TShowTable4InformationSchemaResp showTables4InformationSchema() { + return configManager.showTables4InformationSchema(); + } + @Override public TDescTableResp describeTable( final String database, final String tableName, final boolean isDetails) { return configManager.describeTable(database, tableName, isDetails); } + @Override + public TDescTable4InformationSchemaResp descTables4InformationSchema() { + return configManager.describeTable4InformationSchema(); + } + @Override public TFetchTableResp fetchTables(final Map> fetchTableMap) { return configManager.fetchTables(fetchTableMap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 08ec1372910d..ccf2adb4ca97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1064,7 +1064,7 @@ public class IoTDBConfig { private int driverTaskExecutionTimeSliceInMs = 200; /** Maximum size of wal buffer used in IoTConsensus. Unit: byte */ - private long throttleThreshold = 50 * 1024 * 1024 * 1024L; + private long throttleThreshold = 200 * 1024 * 1024 * 1024L; /** Maximum wait time of write cache in IoTConsensus. Unit: ms */ private long cacheWindowTimeInMs = 10 * 1000L; @@ -1144,7 +1144,7 @@ public class IoTDBConfig { private int maxSizePerBatch = 16 * 1024 * 1024; private int maxPendingBatchesNum = 5; private double maxMemoryRatioForQueue = 0.6; - private long regionMigrationSpeedLimitBytesPerSecond = 32 * 1024 * 1024L; + private long regionMigrationSpeedLimitBytesPerSecond = 48 * 1024 * 1024L; // IoTConsensusV2 Config private int iotConsensusV2PipelineSize = 5; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index c10d5d7339a4..daab55117d19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -445,6 +445,7 @@ public TSStatus visitActivateTemplate( public TSStatus visitBatchActivateTemplate( final BatchActivateTemplateNode node, final ISchemaRegion schemaRegion) { final List statusList = new ArrayList<>(); + final List alreadyActivatedDeviceList = new ArrayList<>(); for (final Map.Entry> entry : node.getTemplateActivationMap().entrySet()) { final Template template = @@ -455,10 +456,20 @@ public TSStatus visitBatchActivateTemplate( entry.getKey(), entry.getValue().right, entry.getValue().left), template); } catch (final MetadataException e) { - logger.error(e.getMessage(), e); - statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + if (e.getErrorCode() == TSStatusCode.TEMPLATE_IS_IN_USE.getStatusCode()) { + alreadyActivatedDeviceList.add(entry.getKey()); + } else { + logger.error(e.getMessage(), e); + statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + } } } + if (!alreadyActivatedDeviceList.isEmpty()) { + final TemplateIsInUseException e = + new TemplateIsInUseException(alreadyActivatedDeviceList.toString()); + logger.error(e.getMessage(), e); + statusList.add(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); + } return statusList.isEmpty() ? RpcUtils.SUCCESS_STATUS : RpcUtils.getStatus(statusList); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java similarity index 82% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java index 78c6d8d4d5cc..a92fdbfce4f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeException.java @@ -17,18 +17,18 @@ * under the License. */ -package org.apache.iotdb.db.exception; +package org.apache.iotdb.db.exception.load; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.rpc.TSStatusCode; -public class VerifyMetadataException extends IoTDBException { +public class LoadAnalyzeException extends IoTDBException { - public VerifyMetadataException(String message) { + public LoadAnalyzeException(String message) { super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); } - public VerifyMetadataException(String message, int errorCode) { + public LoadAnalyzeException(String message, int errorCode) { super(message, errorCode); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java new file mode 100644 index 000000000000..a34ecd12c167 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTableColumnDisorderException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception.load; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.rpc.TSStatusCode; + +public class LoadAnalyzeTableColumnDisorderException extends SemanticException { + + public LoadAnalyzeTableColumnDisorderException(String message) { + super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java similarity index 83% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java index 209fb83bcdbe..b057069546b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/VerifyMetadataTypeMismatchException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/load/LoadAnalyzeTypeMismatchException.java @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.iotdb.db.exception; +package org.apache.iotdb.db.exception.load; import org.apache.iotdb.rpc.TSStatusCode; -public class VerifyMetadataTypeMismatchException extends VerifyMetadataException { +public class LoadAnalyzeTypeMismatchException extends LoadAnalyzeException { - public VerifyMetadataTypeMismatchException(String message) { + public LoadAnalyzeTypeMismatchException(String message) { super(message, TSStatusCode.VERIFY_METADATA_ERROR.getStatusCode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java index 02a82e598c98..123228160726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -19,114 +19,54 @@ package org.apache.iotdb.db.pipe.connector.payload.evolvable.batch; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; -import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.builder.PipeTableModeTsFileBuilder; +import org.apache.iotdb.db.pipe.connector.util.builder.PipeTreeModelTsFileBuilder; +import org.apache.iotdb.db.pipe.connector.util.builder.PipeTsFileBuilder; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTableModelTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; -import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager; -import org.apache.iotdb.db.storageengine.rescon.disk.strategy.DirectoryStrategyType; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; -import org.apache.iotdb.pipe.api.exception.PipeException; -import org.apache.commons.io.FileUtils; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; public class PipeTabletEventTsFileBatch extends PipeTabletEventBatch { private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventTsFileBatch.class); - private static final AtomicReference FOLDER_MANAGER = new AtomicReference<>(); private static final AtomicLong BATCH_ID_GENERATOR = new AtomicLong(0); private final AtomicLong currentBatchId = new AtomicLong(BATCH_ID_GENERATOR.incrementAndGet()); - private final File batchFileBaseDir; - - private static final String TS_FILE_PREFIX = "tb"; // tb means tablet batch - private final AtomicLong tsFileIdGenerator = new AtomicLong(0); private final long maxSizeInBytes; - private final Map, Double> pipeName2WeightMap = new HashMap<>(); - - private final List tabletList = new ArrayList<>(); - private final List isTabletAlignedList = new ArrayList<>(); + private final PipeTsFileBuilder treeModeTsFileBuilder; + private final PipeTsFileBuilder tableModeTsFileBuilder; - @SuppressWarnings("java:S3077") - private volatile TsFileWriter fileWriter; + private final Map, Double> pipeName2WeightMap = new HashMap<>(); public PipeTabletEventTsFileBatch(final int maxDelayInMs, final long requestMaxBatchSizeInBytes) { super(maxDelayInMs); this.maxSizeInBytes = requestMaxBatchSizeInBytes; - try { - this.batchFileBaseDir = getNextBaseDir(); - } catch (final Exception e) { - throw new PipeException( - String.format("Failed to create file dir for batch: %s", e.getMessage())); - } - } - - private File getNextBaseDir() throws DiskSpaceInsufficientException { - if (FOLDER_MANAGER.get() == null) { - synchronized (FOLDER_MANAGER) { - if (FOLDER_MANAGER.get() == null) { - FOLDER_MANAGER.set( - new FolderManager( - Arrays.stream(IoTDBDescriptor.getInstance().getConfig().getPipeReceiverFileDirs()) - .map(fileDir -> fileDir + File.separator + ".batch") - .collect(Collectors.toList()), - DirectoryStrategyType.SEQUENCE_STRATEGY)); - } - } - } - final File baseDir = - new File(FOLDER_MANAGER.get().getNextFolder(), Long.toString(currentBatchId.get())); - if (baseDir.exists()) { - FileUtils.deleteQuietly(baseDir); - } - if (!baseDir.exists() && !baseDir.mkdirs()) { - LOGGER.warn( - "Batch id = {}: Failed to create batch file dir {}.", - currentBatchId.get(), - baseDir.getPath()); - throw new PipeException( - String.format( - "Failed to create batch file dir %s. (Batch id = %s)", - baseDir.getPath(), currentBatchId.get())); - } - LOGGER.info( - "Batch id = {}: Create batch dir successfully, batch file dir = {}.", - currentBatchId.get(), - baseDir.getPath()); - return baseDir; + final AtomicLong tsFileIdGenerator = new AtomicLong(0); + treeModeTsFileBuilder = new PipeTreeModelTsFileBuilder(currentBatchId, tsFileIdGenerator); + tableModeTsFileBuilder = new PipeTableModeTsFileBuilder(currentBatchId, tsFileIdGenerator); } @Override @@ -134,18 +74,28 @@ protected boolean constructBatch(final TabletInsertionEvent event) { if (event instanceof PipeInsertNodeTabletInsertionEvent) { final PipeInsertNodeTabletInsertionEvent insertNodeTabletInsertionEvent = (PipeInsertNodeTabletInsertionEvent) event; - // TODO: for table model insertion, we need to get the database name + final boolean isTableModel = insertNodeTabletInsertionEvent.isTableModelEvent(); final List tablets = insertNodeTabletInsertionEvent.convertToTablets(); for (int i = 0; i < tablets.size(); ++i) { final Tablet tablet = tablets.get(i); if (tablet.getRowSize() == 0) { continue; } - bufferTablet( - insertNodeTabletInsertionEvent.getPipeName(), - insertNodeTabletInsertionEvent.getCreationTime(), - tablet, - insertNodeTabletInsertionEvent.isAligned(i)); + if (isTableModel) { + // table Model + bufferTableModelTablet( + insertNodeTabletInsertionEvent.getPipeName(), + insertNodeTabletInsertionEvent.getCreationTime(), + tablet, + insertNodeTabletInsertionEvent.getTableModelDatabaseName()); + } else { + // tree Model + bufferTreeModelTablet( + insertNodeTabletInsertionEvent.getPipeName(), + insertNodeTabletInsertionEvent.getCreationTime(), + tablet, + insertNodeTabletInsertionEvent.isAligned(i)); + } } } else if (event instanceof PipeRawTabletInsertionEvent) { final PipeRawTabletInsertionEvent rawTabletInsertionEvent = @@ -154,11 +104,21 @@ protected boolean constructBatch(final TabletInsertionEvent event) { if (tablet.getRowSize() == 0) { return true; } - bufferTablet( - rawTabletInsertionEvent.getPipeName(), - rawTabletInsertionEvent.getCreationTime(), - tablet, - rawTabletInsertionEvent.isAligned()); + if (rawTabletInsertionEvent.isTableModelEvent()) { + // table Model + bufferTableModelTablet( + rawTabletInsertionEvent.getPipeName(), + rawTabletInsertionEvent.getCreationTime(), + tablet, + rawTabletInsertionEvent.getTableModelDatabaseName()); + } else { + // tree Model + bufferTreeModelTablet( + rawTabletInsertionEvent.getPipeName(), + rawTabletInsertionEvent.getCreationTime(), + tablet, + rawTabletInsertionEvent.isAligned()); + } } else { LOGGER.warn( "Batch id = {}: Unsupported event {} type {} when constructing tsfile batch", @@ -169,12 +129,25 @@ protected boolean constructBatch(final TabletInsertionEvent event) { return true; } - private void bufferTablet( + private void bufferTreeModelTablet( final String pipeName, final long creationTime, final Tablet tablet, final boolean isAligned) { - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + + totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet); + + pipeName2WeightMap.compute( + new Pair<>(pipeName, creationTime), + (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1); + + treeModeTsFileBuilder.bufferTreeModelTablet(tablet, isAligned); + } + + private void bufferTableModelTablet( + final String pipeName, final long creationTime, final Tablet tablet, final String dataBase) { + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp(); totalBufferSize += PipeMemoryWeightUtil.calculateTabletSizeInBytes(tablet); @@ -182,8 +155,7 @@ private void bufferTablet( new Pair<>(pipeName, creationTime), (pipe, weight) -> Objects.nonNull(weight) ? ++weight : 1); - tabletList.add(tablet); - isTabletAlignedList.add(isAligned); + tableModeTsFileBuilder.bufferTableModelTablet(dataBase, tablet); } public Map, Double> deepCopyPipe2WeightMap() { @@ -195,193 +167,28 @@ public Map, Double> deepCopyPipe2WeightMap() { return new HashMap<>(pipeName2WeightMap); } - public synchronized List sealTsFiles() throws IOException, WriteProcessException { - return isClosed ? Collections.emptyList() : writeTabletsToTsFiles(); - } - - private List writeTabletsToTsFiles() throws IOException, WriteProcessException { - final Map> device2Tablets = new HashMap<>(); - final Map device2Aligned = new HashMap<>(); - - // Sort the tablets by device id - for (int i = 0, size = tabletList.size(); i < size; ++i) { - final Tablet tablet = tabletList.get(i); - final String deviceId = tablet.getDeviceId(); - device2Tablets.computeIfAbsent(deviceId, k -> new ArrayList<>()).add(tablet); - device2Aligned.put(deviceId, isTabletAlignedList.get(i)); + /** + * Converts a Tablet to a TSFile and returns the generated TSFile along with its corresponding + * database name. + * + * @return a list of pairs containing the database name and the generated TSFile + * @throws IOException if an I/O error occurs during the conversion process + * @throws WriteProcessException if an error occurs during the write process + */ + public synchronized List> sealTsFiles() + throws IOException, WriteProcessException { + if (isClosed) { + return Collections.emptyList(); } - // Sort the tablets by start time in each device - for (final List tablets : device2Tablets.values()) { - tablets.sort( - // Each tablet has at least one timestamp - Comparator.comparingLong(tablet -> tablet.timestamps[0])); + final List> list = new ArrayList<>(); + if (!treeModeTsFileBuilder.isEmpty()) { + list.addAll(treeModeTsFileBuilder.convertTabletToTsFileWithDBInfo()); } - - // Sort the devices by device id - final List devices = new ArrayList<>(device2Tablets.keySet()); - devices.sort(Comparator.naturalOrder()); - - // Replace ArrayList with LinkedList to improve performance - final LinkedHashMap> device2TabletsLinkedList = - new LinkedHashMap<>(); - for (final String device : devices) { - device2TabletsLinkedList.put(device, new LinkedList<>(device2Tablets.get(device))); - } - - // Help GC - devices.clear(); - device2Tablets.clear(); - - // Write the tablets to the tsfile device by device, and the tablets - // in the same device are written in order of start time. Tablets in - // the same device should not be written if their time ranges overlap. - // If overlapped, we try to write the tablets whose device id is not - // the same as the previous one. For the tablets not written in the - // previous round, we write them in a new tsfile. - final List sealedFiles = new ArrayList<>(); - - // Try making the tsfile size as large as possible - while (!device2TabletsLinkedList.isEmpty()) { - if (Objects.isNull(fileWriter)) { - fileWriter = - new TsFileWriter( - new File( - batchFileBaseDir, - TS_FILE_PREFIX - + "_" - + IoTDBDescriptor.getInstance().getConfig().getDataNodeId() - + "_" - + currentBatchId.get() - + "_" - + tsFileIdGenerator.getAndIncrement() - + TsFileConstant.TSFILE_SUFFIX)); - } - - try { - tryBestToWriteTabletsIntoOneFile(device2TabletsLinkedList, device2Aligned); - } catch (final Exception e) { - LOGGER.warn( - "Batch id = {}: Failed to write tablets into tsfile, because {}", - currentBatchId.get(), - e.getMessage(), - e); - - try { - fileWriter.close(); - } catch (final Exception closeException) { - LOGGER.warn( - "Batch id = {}: Failed to close the tsfile {} after failed to write tablets into, because {}", - currentBatchId.get(), - fileWriter.getIOWriter().getFile().getPath(), - closeException.getMessage(), - closeException); - } finally { - // Add current writing file to the list and delete the file - sealedFiles.add(fileWriter.getIOWriter().getFile()); - } - - for (final File sealedFile : sealedFiles) { - final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile); - LOGGER.warn( - "Batch id = {}: {} delete the tsfile {} after failed to write tablets into {}. {}", - currentBatchId.get(), - deleteSuccess ? "Successfully" : "Failed to", - sealedFile.getPath(), - fileWriter.getIOWriter().getFile().getPath(), - deleteSuccess ? "" : "Maybe the tsfile needs to be deleted manually."); - } - sealedFiles.clear(); - - fileWriter = null; - - throw e; - } - - fileWriter.close(); - final File sealedFile = fileWriter.getIOWriter().getFile(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Batch id = {}: Seal tsfile {} successfully.", - currentBatchId.get(), - sealedFile.getPath()); - } - sealedFiles.add(sealedFile); - fileWriter = null; - } - - return sealedFiles; - } - - private void tryBestToWriteTabletsIntoOneFile( - final LinkedHashMap> device2TabletsLinkedList, - final Map device2Aligned) - throws IOException, WriteProcessException { - final Iterator>> iterator = - device2TabletsLinkedList.entrySet().iterator(); - - while (iterator.hasNext()) { - final Map.Entry> entry = iterator.next(); - final String deviceId = entry.getKey(); - final LinkedList tablets = entry.getValue(); - - final List tabletsToWrite = new ArrayList<>(); - - Tablet lastTablet = null; - while (!tablets.isEmpty()) { - final Tablet tablet = tablets.peekFirst(); - if (Objects.isNull(lastTablet) - // lastTablet.rowSize is not 0 - || lastTablet.timestamps[lastTablet.getRowSize() - 1] < tablet.timestamps[0]) { - tabletsToWrite.add(tablet); - lastTablet = tablet; - tablets.pollFirst(); - } else { - break; - } - } - - if (tablets.isEmpty()) { - iterator.remove(); - } - - final boolean isAligned = device2Aligned.get(deviceId); - if (isAligned) { - final Map> deviceId2MeasurementSchemas = new HashMap<>(); - tabletsToWrite.forEach( - tablet -> - deviceId2MeasurementSchemas.compute( - tablet.getDeviceId(), - (k, v) -> { - if (Objects.isNull(v)) { - return new ArrayList<>(tablet.getSchemas()); - } - v.addAll(tablet.getSchemas()); - return v; - })); - for (final Entry> deviceIdWithMeasurementSchemas : - deviceId2MeasurementSchemas.entrySet()) { - fileWriter.registerAlignedTimeseries( - new Path(deviceIdWithMeasurementSchemas.getKey()), - deviceIdWithMeasurementSchemas.getValue()); - } - for (final Tablet tablet : tabletsToWrite) { - fileWriter.writeAligned(tablet); - } - } else { - for (final Tablet tablet : tabletsToWrite) { - for (final IMeasurementSchema schema : tablet.getSchemas()) { - try { - fileWriter.registerTimeseries(new Path(tablet.getDeviceId()), schema); - } catch (final WriteProcessException ignore) { - // Do nothing if the timeSeries has been registered - } - } - - fileWriter.writeTree(tablet); - } - } + if (!tableModeTsFileBuilder.isEmpty()) { + list.addAll(tableModeTsFileBuilder.convertTabletToTsFileWithDBInfo()); } + return list; } @Override @@ -394,13 +201,8 @@ public synchronized void onSuccess() { super.onSuccess(); pipeName2WeightMap.clear(); - - tabletList.clear(); - isTabletAlignedList.clear(); - - // We don't need to delete the tsFile here, because the tsFile - // will be deleted after the file is transferred. - fileWriter = null; + tableModeTsFileBuilder.onSuccess(); + treeModeTsFileBuilder.onSuccess(); } @Override @@ -409,33 +211,7 @@ public synchronized void close() { pipeName2WeightMap.clear(); - tabletList.clear(); - isTabletAlignedList.clear(); - - if (Objects.nonNull(fileWriter)) { - try { - fileWriter.close(); - } catch (final Exception e) { - LOGGER.info( - "Batch id = {}: Failed to close the tsfile {} when trying to close batch, because {}", - currentBatchId.get(), - fileWriter.getIOWriter().getFile().getPath(), - e.getMessage(), - e); - } - - try { - FileUtils.delete(fileWriter.getIOWriter().getFile()); - } catch (final Exception e) { - LOGGER.info( - "Batch id = {}: Failed to delete the tsfile {} when trying to close batch, because {}", - currentBatchId.get(), - fileWriter.getIOWriter().getFile().getPath(), - e.getMessage(), - e); - } - - fileWriter = null; - } + tableModeTsFileBuilder.close(); + treeModeTsFileBuilder.close(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java index 7c5ef90299d0..bf27b697698c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -23,7 +23,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.IoTDBConnectorRequestVersion; import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -60,7 +60,7 @@ public boolean getIsAligned() { } public InsertTabletStatement constructStatement() { - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); try { if (isTabletEmpty(tablet)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReqV2.java index c0ae65e0628e..fd435ded99ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferTabletRawReqV2.java @@ -23,7 +23,8 @@ import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.IoTDBConnectorRequestVersion; import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTableModelTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter; import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -57,7 +58,11 @@ public String getDataBaseName() { @Override public InsertTabletStatement constructStatement() { - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + if (Objects.isNull(dataBaseName)) { + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + } else { + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByTimestampIfNecessary(); + } try { if (isTabletEmpty(tablet)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaNameSpace.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaNameSpace.java index 270188005ce4..834a14f94df8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaNameSpace.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/opcua/OpcUaNameSpace.java @@ -22,7 +22,8 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException; import org.apache.iotdb.commons.utils.PathUtils; -import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTableModelTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.iotdb.pipe.api.event.Event; @@ -116,7 +117,7 @@ private void transferTabletForClientServerModel(final Tablet tablet, final boole final List schemas = tablet.getSchemas(); final List newSchemas = new ArrayList<>(); if (!isTableModel) { - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); final List timestamps = new ArrayList<>(); final List values = new ArrayList<>(); @@ -137,6 +138,8 @@ private void transferTabletForClientServerModel(final Tablet tablet, final boole transferTabletRowForClientServerModel( tablet.getDeviceId().split("\\."), newSchemas, timestamps, values); } else { + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByTimestampIfNecessary(); + final List columnIndexes = new ArrayList<>(); for (int i = 0; i < schemas.size(); ++i) { if (tablet.getColumnTypes().get(i) == Tablet.ColumnCategory.FIELD) { @@ -144,6 +147,7 @@ private void transferTabletForClientServerModel(final Tablet tablet, final boole newSchemas.add(schemas.get(i)); } } + for (int i = 0; i < tablet.getRowSize(); ++i) { final Object[] segments = tablet.getDeviceID(i).getSegments(); final String[] folderSegments = new String[segments.length + 2]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/request/PipeConsensusDeleteNodeReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/request/PipeConsensusDeleteNodeReq.java index c8bc33bb52ac..f0e71db4f8e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/request/PipeConsensusDeleteNodeReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/pipeconsensus/payload/request/PipeConsensusDeleteNodeReq.java @@ -27,7 +27,6 @@ import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDeleteDataNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.tsfile.utils.PublicBAOS; import org.slf4j.Logger; @@ -85,7 +84,8 @@ public static PipeConsensusDeleteNodeReq fromTPipeConsensusTransferReq( TPipeConsensusTransferReq transferReq) { final PipeConsensusDeleteNodeReq deleteNodeReq = new PipeConsensusDeleteNodeReq(); - deleteNodeReq.deleteDataNode = (DeleteDataNode) PlanNodeType.deserialize(transferReq.body); + deleteNodeReq.deleteDataNode = + (AbstractDeleteDataNode) PlanNodeType.deserialize(transferReq.body); deleteNodeReq.version = transferReq.version; deleteNodeReq.type = transferReq.type; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java index 4eb24322e83c..1dfebff416c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java @@ -187,13 +187,13 @@ private void transferInBatchWithoutCheck( new PipeTransferTabletBatchEventHandler((PipeTabletEventPlainBatch) batch, this)); } else if (batch instanceof PipeTabletEventTsFileBatch) { final PipeTabletEventTsFileBatch tsFileBatch = (PipeTabletEventTsFileBatch) batch; - final List sealedFiles = tsFileBatch.sealTsFiles(); + final List> dbTsFilePairs = tsFileBatch.sealTsFiles(); final Map, Double> pipe2WeightMap = tsFileBatch.deepCopyPipe2WeightMap(); final List events = tsFileBatch.deepCopyEvents(); - final AtomicInteger eventsReferenceCount = new AtomicInteger(sealedFiles.size()); + final AtomicInteger eventsReferenceCount = new AtomicInteger(dbTsFilePairs.size()); final AtomicBoolean eventsHadBeenAddedToRetryQueue = new AtomicBoolean(false); - for (final File sealedFile : sealedFiles) { + for (final Pair sealedFile : dbTsFilePairs) { transfer( new PipeTransferTsFileHandler( this, @@ -201,10 +201,10 @@ private void transferInBatchWithoutCheck( events, eventsReferenceCount, eventsHadBeenAddedToRetryQueue, - sealedFile, + sealedFile.right, null, false, - null)); + sealedFile.left)); } } else { LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java index 8370dd667ab3..cc03af180f9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java @@ -26,7 +26,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.connector.payload.evolvable.batch.PipeTabletEventPlainBatch; import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector; -import org.apache.iotdb.db.pipe.connector.util.LeaderCacheUtils; +import org.apache.iotdb.db.pipe.connector.util.cacher.LeaderCacheUtils; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java index 87f9b964d12c..783d5ff7aa5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferCompressedReq; import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.pipe.connector.client.IoTDBDataNodeAsyncClientManager; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceReq; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceWithModReq; @@ -238,7 +239,11 @@ protected boolean onCompleteInternal(final TPipeTransferResp response) { // Delete current file when using tsFile as batch if (events.stream().anyMatch(event -> !(event instanceof PipeTsFileInsertionEvent))) { - FileUtils.delete(currentFile); + RetryUtils.retryOnException( + () -> { + FileUtils.delete(currentFile); + return null; + }); } } catch (final IOException e) { LOGGER.warn( @@ -341,7 +346,11 @@ protected void onErrorInternal(final Exception exception) { // Delete current file when using tsFile as batch if (events.stream().anyMatch(event -> !(event instanceof PipeTsFileInsertionEvent))) { - FileUtils.delete(currentFile); + RetryUtils.retryOnException( + () -> { + FileUtils.delete(currentFile); + return null; + }); } } catch (final IOException e) { LOGGER.warn("Failed to close file reader or delete tsFile when failed to transfer file.", e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java index 14f2ff5fd070..99b2e71c731e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.pipe.connector.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferFilePieceReq; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.pipe.connector.payload.evolvable.batch.PipeTabletEventBatch; import org.apache.iotdb.db.pipe.connector.payload.evolvable.batch.PipeTabletEventPlainBatch; import org.apache.iotdb.db.pipe.connector.payload.evolvable.batch.PipeTabletEventTsFileBatch; @@ -35,7 +36,7 @@ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceReq; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFilePieceWithModReq; import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTsFileSealWithModReq; -import org.apache.iotdb.db.pipe.connector.util.LeaderCacheUtils; +import org.apache.iotdb.db.pipe.connector.util.cacher.LeaderCacheUtils; import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; @@ -306,18 +307,22 @@ private void doTransfer( private void doTransfer(final PipeTabletEventTsFileBatch batchToTransfer) throws IOException, WriteProcessException { - final List sealedFiles = batchToTransfer.sealTsFiles(); + final List> dbTsFilePairs = batchToTransfer.sealTsFiles(); final Map, Double> pipe2WeightMap = batchToTransfer.deepCopyPipe2WeightMap(); - for (final File tsFile : sealedFiles) { - doTransfer(pipe2WeightMap, tsFile, null, null); + for (final Pair dbTsFile : dbTsFilePairs) { + doTransfer(pipe2WeightMap, dbTsFile.right, null, dbTsFile.left); try { - FileUtils.delete(tsFile); + RetryUtils.retryOnException( + () -> { + FileUtils.delete(dbTsFile.right); + return null; + }); } catch (final NoSuchFileException e) { - LOGGER.info("The file {} is not found, may already be deleted.", tsFile); + LOGGER.info("The file {} is not found, may already be deleted.", dbTsFile); } catch (final Exception e) { LOGGER.warn( - "Failed to delete batch file {}, this file should be deleted manually later", tsFile); + "Failed to delete batch file {}, this file should be deleted manually later", dbTsFile); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModeTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModeTsFileBuilder.java new file mode 100644 index 000000000000..6ac5ac911bfa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModeTsFileBuilder.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.connector.util.builder; + +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.WriteUtils; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeTableModeTsFileBuilder extends PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTableModeTsFileBuilder.class); + + private final Map> dataBase2TabletList = new HashMap<>(); + + public PipeTableModeTsFileBuilder(AtomicLong currentBatchId, AtomicLong tsFileIdGenerator) { + super(currentBatchId, tsFileIdGenerator); + } + + @Override + public void bufferTableModelTablet(String dataBase, Tablet tablet) { + dataBase2TabletList.computeIfAbsent(dataBase, db -> new ArrayList<>()).add(tablet); + } + + @Override + public void bufferTreeModelTablet(Tablet tablet, Boolean isAligned) { + throw new UnsupportedOperationException( + "PipeTableModeTsFileBuilder does not support tree model tablet to build TSFile"); + } + + @Override + public List> convertTabletToTsFileWithDBInfo() throws IOException { + if (dataBase2TabletList.isEmpty()) { + return new ArrayList<>(0); + } + List> pairList = new ArrayList<>(); + for (Map.Entry> entry : dataBase2TabletList.entrySet()) { + final LinkedHashSet>>>> linkedHashSet = + new LinkedHashSet<>(); + pairList.addAll( + writeTableModelTabletsToTsFiles(entry.getValue(), entry.getKey(), linkedHashSet)); + } + return pairList; + } + + @Override + public boolean isEmpty() { + return dataBase2TabletList.isEmpty(); + } + + @Override + public synchronized void onSuccess() { + super.onSuccess(); + dataBase2TabletList.clear(); + } + + @Override + public synchronized void close() { + super.close(); + dataBase2TabletList.clear(); + } + + private >>> + List> writeTableModelTabletsToTsFiles( + final List tabletList, + final String dataBase, + LinkedHashSet> linkedHashSet) + throws IOException { + + final Map> tableName2Tablets = new HashMap<>(); + + // Sort the tablets by dataBaseName + for (final Tablet tablet : tabletList) { + tableName2Tablets + .computeIfAbsent(tablet.getTableName(), k -> new ArrayList<>()) + .add((T) new Pair<>(tablet, WriteUtils.splitTabletByDevice(tablet))); + } + + // Replace ArrayList with LinkedList to improve performance + final LinkedHashSet> table2Tablets = new LinkedHashSet<>(); + + // Sort the tablets by start time in first device + for (final List tablets : tableName2Tablets.values()) { + tablets.sort( + (o1, o2) -> { + final IDeviceID deviceID = o1.right.get(0).left; + final int result; + if ((result = deviceID.compareTo(o2.right.get(0).left)) == 0) { + return Long.compare(o1.left.timestamps[0], o2.left.timestamps[0]); + } + return result; + }); + } + + // Sort the tables by table name + tableName2Tablets.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.naturalOrder())) + .forEach(entry -> linkedHashSet.add(new LinkedList<>(entry.getValue()))); + + // Help GC + tableName2Tablets.clear(); + + final List> sealedFiles = new ArrayList<>(); + + // Try making the tsfile size as large as possible + while (!linkedHashSet.isEmpty()) { + if (Objects.isNull(fileWriter)) { + createFileWriter(); + } + + try { + tryBestToWriteTabletsIntoOneFile(linkedHashSet); + } catch (final Exception e) { + LOGGER.warn( + "Batch id = {}: Failed to write tablets into tsfile, because {}", + currentBatchId.get(), + e.getMessage(), + e); + + try { + fileWriter.close(); + } catch (final Exception closeException) { + LOGGER.warn( + "Batch id = {}: Failed to close the tsfile {} after failed to write tablets into, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + closeException.getMessage(), + closeException); + } finally { + // Add current writing file to the list and delete the file + sealedFiles.add(new Pair<>(dataBase, fileWriter.getIOWriter().getFile())); + } + + for (final Pair sealedFile : sealedFiles) { + final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + LOGGER.warn( + "Batch id = {}: {} delete the tsfile {} after failed to write tablets into {}. {}", + currentBatchId.get(), + deleteSuccess ? "Successfully" : "Failed to", + sealedFile.right.getPath(), + fileWriter.getIOWriter().getFile().getPath(), + deleteSuccess ? "" : "Maybe the tsfile needs to be deleted manually."); + } + sealedFiles.clear(); + + fileWriter = null; + + throw e; + } + + fileWriter.close(); + final File sealedFile = fileWriter.getIOWriter().getFile(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Batch id = {}: Seal tsfile {} successfully.", + currentBatchId.get(), + sealedFile.getPath()); + } + sealedFiles.add(new Pair<>(dataBase, sealedFile)); + fileWriter = null; + } + + return sealedFiles; + } + + private >>> + void tryBestToWriteTabletsIntoOneFile( + final LinkedHashSet> device2TabletsLinkedList) throws IOException { + final Iterator> iterator = device2TabletsLinkedList.iterator(); + + while (iterator.hasNext()) { + final LinkedList tablets = iterator.next(); + + final List tabletsToWrite = new ArrayList<>(); + final Map deviceLastTimestampMap = new HashMap<>(); + while (!tablets.isEmpty()) { + final T pair = tablets.peekFirst(); + if (timestampsAreNonOverlapping( + (Pair>>) pair, deviceLastTimestampMap)) { + tabletsToWrite.add(pair); + tablets.pollFirst(); + continue; + } + break; + } + + if (tablets.isEmpty()) { + iterator.remove(); + } + boolean schemaNotRegistered = true; + for (final Pair>> pair : tabletsToWrite) { + final Tablet tablet = pair.left; + if (schemaNotRegistered) { + fileWriter.registerTableSchema( + new TableSchema(tablet.getTableName(), tablet.getSchemas(), tablet.getColumnTypes())); + schemaNotRegistered = false; + } + try { + fileWriter.writeTable(tablet, pair.right); + } catch (WriteProcessException e) { + LOGGER.warn( + "Batch id = {}: Failed to build the table model TSFile. Please check whether the written Tablet has time overlap and whether the Table Schema is correct.", + currentBatchId.get(), + e); + throw new PipeException( + "The written Tablet time may overlap or the Schema may be incorrect"); + } + } + } + } + + /** + * A Map is used to record the maximum time each {@link IDeviceID} is written. {@link Pair} + * records the Index+1 of the maximum timestamp of IDevice in each {@link Tablet}. + * + * @return If false, the tablet overlaps with the previous tablet; if true, there is no time + * overlap. + */ + private >>> + boolean timestampsAreNonOverlapping( + final T tabletPair, final Map deviceLastTimestampMap) { + int currentTimestampIndex = 0; + for (Pair deviceTimestampIndexPair : tabletPair.right) { + final Long lastDeviceTimestamp = deviceLastTimestampMap.get(deviceTimestampIndexPair.left); + if (lastDeviceTimestamp != null + && lastDeviceTimestamp >= tabletPair.left.timestamps[currentTimestampIndex]) { + return false; + } + currentTimestampIndex = deviceTimestampIndexPair.right; + deviceLastTimestampMap.put( + deviceTimestampIndexPair.left, tabletPair.left.timestamps[currentTimestampIndex - 1]); + } + + return true; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilder.java new file mode 100644 index 000000000000..8d30dd0172f2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilder.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.connector.util.builder; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class PipeTreeModelTsFileBuilder extends PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTreeModelTsFileBuilder.class); + + private final List tabletList = new ArrayList<>(); + private final List isTabletAlignedList = new ArrayList<>(); + + public PipeTreeModelTsFileBuilder( + final AtomicLong currentBatchId, final AtomicLong tsFileIdGenerator) { + super(currentBatchId, tsFileIdGenerator); + } + + @Override + public void bufferTableModelTablet(final String dataBase, final Tablet tablet) { + throw new UnsupportedOperationException( + "PipeTreeModelTsFileBuilder does not support table model tablet to build TSFile"); + } + + @Override + public void bufferTreeModelTablet(final Tablet tablet, final Boolean isAligned) { + tabletList.add(tablet); + isTabletAlignedList.add(isAligned); + } + + @Override + public List> convertTabletToTsFileWithDBInfo() + throws IOException, WriteProcessException { + return writeTabletsToTsFiles(); + } + + @Override + public boolean isEmpty() { + return tabletList.isEmpty(); + } + + @Override + public void onSuccess() { + super.onSuccess(); + tabletList.clear(); + isTabletAlignedList.clear(); + } + + @Override + public synchronized void close() { + super.close(); + tabletList.clear(); + isTabletAlignedList.clear(); + } + + private List> writeTabletsToTsFiles() + throws IOException, WriteProcessException { + final Map> device2Tablets = new HashMap<>(); + final Map device2Aligned = new HashMap<>(); + + // Sort the tablets by device id + for (int i = 0, size = tabletList.size(); i < size; ++i) { + final Tablet tablet = tabletList.get(i); + final String deviceId = tablet.getDeviceId(); + device2Tablets.computeIfAbsent(deviceId, k -> new ArrayList<>()).add(tablet); + device2Aligned.put(deviceId, isTabletAlignedList.get(i)); + } + + // Sort the tablets by start time in each device + for (final List tablets : device2Tablets.values()) { + tablets.sort( + // Each tablet has at least one timestamp + Comparator.comparingLong(tablet -> tablet.timestamps[0])); + } + + // Sort the devices by device id + final List devices = new ArrayList<>(device2Tablets.keySet()); + devices.sort(Comparator.naturalOrder()); + + // Replace ArrayList with LinkedList to improve performance + final LinkedHashMap> device2TabletsLinkedList = + new LinkedHashMap<>(); + for (final String device : devices) { + device2TabletsLinkedList.put(device, new LinkedList<>(device2Tablets.get(device))); + } + + // Help GC + devices.clear(); + device2Tablets.clear(); + + // Write the tablets to the tsfile device by device, and the tablets + // in the same device are written in order of start time. Tablets in + // the same device should not be written if their time ranges overlap. + // If overlapped, we try to write the tablets whose device id is not + // the same as the previous one. For the tablets not written in the + // previous round, we write them in a new tsfile. + final List> sealedFiles = new ArrayList<>(); + + // Try making the tsfile size as large as possible + while (!device2TabletsLinkedList.isEmpty()) { + if (Objects.isNull(fileWriter)) { + createFileWriter(); + } + try { + tryBestToWriteTabletsIntoOneFile(device2TabletsLinkedList, device2Aligned); + } catch (final Exception e) { + LOGGER.warn( + "Batch id = {}: Failed to write tablets into tsfile, because {}", + currentBatchId.get(), + e.getMessage(), + e); + + try { + fileWriter.close(); + } catch (final Exception closeException) { + LOGGER.warn( + "Batch id = {}: Failed to close the tsfile {} after failed to write tablets into, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + closeException.getMessage(), + closeException); + } finally { + // Add current writing file to the list and delete the file + sealedFiles.add(new Pair<>(null, fileWriter.getIOWriter().getFile())); + } + + for (final Pair sealedFile : sealedFiles) { + final boolean deleteSuccess = FileUtils.deleteQuietly(sealedFile.right); + LOGGER.warn( + "Batch id = {}: {} delete the tsfile {} after failed to write tablets into {}. {}", + currentBatchId.get(), + deleteSuccess ? "Successfully" : "Failed to", + sealedFile.right.getPath(), + fileWriter.getIOWriter().getFile().getPath(), + deleteSuccess ? "" : "Maybe the tsfile needs to be deleted manually."); + } + sealedFiles.clear(); + + fileWriter = null; + + throw e; + } + + fileWriter.close(); + final File sealedFile = fileWriter.getIOWriter().getFile(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Batch id = {}: Seal tsfile {} successfully.", + currentBatchId.get(), + sealedFile.getPath()); + } + sealedFiles.add(new Pair<>(null, sealedFile)); + fileWriter = null; + } + + return sealedFiles; + } + + private void tryBestToWriteTabletsIntoOneFile( + final LinkedHashMap> device2TabletsLinkedList, + final Map device2Aligned) + throws IOException, WriteProcessException { + final Iterator>> iterator = + device2TabletsLinkedList.entrySet().iterator(); + + while (iterator.hasNext()) { + final Map.Entry> entry = iterator.next(); + final String deviceId = entry.getKey(); + final LinkedList tablets = entry.getValue(); + + final List tabletsToWrite = new ArrayList<>(); + + Tablet lastTablet = null; + while (!tablets.isEmpty()) { + final Tablet tablet = tablets.peekFirst(); + if (Objects.isNull(lastTablet) + // lastTablet.rowSize is not 0 + || lastTablet.timestamps[lastTablet.getRowSize() - 1] < tablet.timestamps[0]) { + tabletsToWrite.add(tablet); + lastTablet = tablet; + tablets.pollFirst(); + } else { + break; + } + } + + if (tablets.isEmpty()) { + iterator.remove(); + } + + final boolean isAligned = device2Aligned.get(deviceId); + if (isAligned) { + final Map> deviceId2MeasurementSchemas = new HashMap<>(); + tabletsToWrite.forEach( + tablet -> + deviceId2MeasurementSchemas.compute( + tablet.getDeviceId(), + (k, v) -> { + if (Objects.isNull(v)) { + return new ArrayList<>(tablet.getSchemas()); + } + v.addAll(tablet.getSchemas()); + return v; + })); + for (final Map.Entry> deviceIdWithMeasurementSchemas : + deviceId2MeasurementSchemas.entrySet()) { + fileWriter.registerAlignedTimeseries( + new Path(deviceIdWithMeasurementSchemas.getKey()), + deviceIdWithMeasurementSchemas.getValue()); + } + for (final Tablet tablet : tabletsToWrite) { + fileWriter.writeAligned(tablet); + } + } else { + for (final Tablet tablet : tabletsToWrite) { + for (final IMeasurementSchema schema : tablet.getSchemas()) { + try { + fileWriter.registerTimeseries( + IDeviceID.Factory.DEFAULT_FACTORY.create(tablet.getDeviceId()), schema); + } catch (final WriteProcessException ignore) { + // Do nothing if the timeSeries has been registered + } + } + + fileWriter.writeTree(tablet); + } + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java new file mode 100644 index 000000000000..d320bff0d173 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTsFileBuilder.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.connector.util.builder; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager; +import org.apache.iotdb.db.storageengine.rescon.disk.strategy.DirectoryStrategyType; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +public abstract class PipeTsFileBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTsFileBuilder.class); + + private static final AtomicReference FOLDER_MANAGER = new AtomicReference<>(); + protected final AtomicLong currentBatchId; + private final File batchFileBaseDir; + + private static final String TS_FILE_PREFIX = "tb"; // tb means tablet batch + private final AtomicLong tsFileIdGenerator; + + @SuppressWarnings("java:S3077") + protected volatile TsFileWriter fileWriter; + + public PipeTsFileBuilder(final AtomicLong currentBatchId, final AtomicLong tsFileIdGenerator) { + this.currentBatchId = currentBatchId; + this.tsFileIdGenerator = tsFileIdGenerator; + try { + this.batchFileBaseDir = getNextBaseDir(); + } catch (final Exception e) { + throw new PipeException( + String.format("Failed to create file dir for batch: %s", e.getMessage())); + } + } + + private File getNextBaseDir() throws DiskSpaceInsufficientException { + if (FOLDER_MANAGER.get() == null) { + synchronized (FOLDER_MANAGER) { + if (FOLDER_MANAGER.get() == null) { + FOLDER_MANAGER.set( + new FolderManager( + Arrays.stream(IoTDBDescriptor.getInstance().getConfig().getPipeReceiverFileDirs()) + .map(fileDir -> fileDir + File.separator + ".batch") + .collect(Collectors.toList()), + DirectoryStrategyType.SEQUENCE_STRATEGY)); + } + } + } + + final File baseDir = + new File(FOLDER_MANAGER.get().getNextFolder(), Long.toString(currentBatchId.get())); + if (baseDir.exists()) { + FileUtils.deleteQuietly(baseDir); + } + if (!baseDir.exists() && !baseDir.mkdirs()) { + LOGGER.warn( + "Batch id = {}: Failed to create batch file dir {}.", + currentBatchId.get(), + baseDir.getPath()); + throw new PipeException( + String.format( + "Failed to create batch file dir %s. (Batch id = %s)", + baseDir.getPath(), currentBatchId.get())); + } + LOGGER.info( + "Batch id = {}: Create batch dir successfully, batch file dir = {}.", + currentBatchId.get(), + baseDir.getPath()); + return baseDir; + } + + public abstract void bufferTableModelTablet(String dataBase, Tablet tablet); + + public abstract void bufferTreeModelTablet(Tablet tablet, Boolean isAligned); + + public abstract List> convertTabletToTsFileWithDBInfo() + throws IOException, WriteProcessException; + + public abstract boolean isEmpty(); + + public synchronized void onSuccess() { + fileWriter = null; + } + + public synchronized void close() { + if (Objects.nonNull(fileWriter)) { + try { + fileWriter.close(); + } catch (final Exception e) { + LOGGER.info( + "Batch id = {}: Failed to close the tsfile {} when trying to close batch, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + e.getMessage(), + e); + } + + try { + FileUtils.delete(fileWriter.getIOWriter().getFile()); + } catch (final Exception e) { + LOGGER.info( + "Batch id = {}: Failed to delete the tsfile {} when trying to close batch, because {}", + currentBatchId.get(), + fileWriter.getIOWriter().getFile().getPath(), + e.getMessage(), + e); + } + + fileWriter = null; + } + } + + protected void createFileWriter() throws IOException { + fileWriter = + new TsFileWriter( + new File( + batchFileBaseDir, + TS_FILE_PREFIX + + "_" + + IoTDBDescriptor.getInstance().getConfig().getDataNodeId() + + "_" + + currentBatchId.get() + + "_" + + tsFileIdGenerator.getAndIncrement() + + TsFileConstant.TSFILE_SUFFIX)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/LeaderCacheUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/cacher/LeaderCacheUtils.java similarity index 97% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/LeaderCacheUtils.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/cacher/LeaderCacheUtils.java index e31fec9b57b6..0151a83a9cff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/LeaderCacheUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/cacher/LeaderCacheUtils.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.pipe.connector.util; +package org.apache.iotdb.db.pipe.connector.util.cacher; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTableModelTabletEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTableModelTabletEventSorter.java new file mode 100644 index 000000000000..63ac5b847a1b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTableModelTabletEventSorter.java @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.connector.util.sorter; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PipeTableModelTabletEventSorter { + + private final Tablet tablet; + + private Integer[] index; + private boolean isUnSorted = false; + private boolean hasDuplicates = false; + private int deduplicatedSize; + private int initIndexSize; + + public PipeTableModelTabletEventSorter(final Tablet tablet) { + this.tablet = tablet; + deduplicatedSize = tablet == null ? 0 : tablet.getRowSize(); + } + + /** + * For the sorting and deduplication needs of the table model tablet, it is done according to the + * {@link IDeviceID}. For sorting, it is necessary to sort the {@link IDeviceID} first, and then + * sort by time. Deduplication is to remove the same timestamp in the same {@link IDeviceID}, and + * the same timestamp in different {@link IDeviceID} will not be processed. + */ + public void sortAndDeduplicateByDevIdTimestamp() { + if (tablet == null || tablet.getRowSize() < 1) { + return; + } + + HashMap>> deviceIDToIndexMap = new HashMap<>(); + final long[] timestamps = tablet.timestamps; + + IDeviceID lastDevice = tablet.getDeviceID(0); + long previousTimestamp = tablet.timestamps[0]; + int lasIndex = 0; + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final IDeviceID deviceID = tablet.getDeviceID(i); + final long currentTimestamp = timestamps[i]; + final int deviceComparison = deviceID.compareTo(lastDevice); + if (deviceComparison == 0) { + if (previousTimestamp == currentTimestamp) { + hasDuplicates = true; + continue; + } + if (previousTimestamp > currentTimestamp) { + isUnSorted = true; + } + previousTimestamp = currentTimestamp; + continue; + } + if (deviceComparison < 0) { + isUnSorted = true; + } + + List> list = + deviceIDToIndexMap.computeIfAbsent(lastDevice, k -> new ArrayList<>()); + + if (!list.isEmpty()) { + isUnSorted = true; + } + list.add(new Pair<>(lasIndex, i)); + lastDevice = deviceID; + lasIndex = i; + previousTimestamp = currentTimestamp; + } + + List> list = + deviceIDToIndexMap.computeIfAbsent(lastDevice, k -> new ArrayList<>()); + if (!list.isEmpty()) { + isUnSorted = true; + } + list.add(new Pair<>(lasIndex, tablet.getRowSize())); + + if (!isUnSorted && !hasDuplicates) { + return; + } + + initIndexSize = 0; + deduplicatedSize = 0; + index = new Integer[tablet.getRowSize()]; + deviceIDToIndexMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach( + entry -> { + final int start = initIndexSize; + int i = initIndexSize; + for (Pair pair : entry.getValue()) { + for (int j = pair.left; j < pair.right; j++) { + index[i++] = j; + } + } + if (isUnSorted) { + sortTimestamps(start, i); + deduplicateTimestamps(start, i); + initIndexSize = i; + return; + } + + if (hasDuplicates) { + deduplicateTimestamps(start, i); + } + initIndexSize = i; + }); + + sortAndDeduplicateValuesAndBitMaps(); + } + + private void sortAndDeduplicateValuesAndBitMaps() { + int columnIndex = 0; + tablet.timestamps = + (long[]) + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.timestamps, TSDataType.TIMESTAMP, index); + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.values[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.values[columnIndex], schema.getType(), index); + if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { + tablet.bitMaps[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.bitMaps[columnIndex], index); + } + columnIndex++; + } + } + + tablet.setRowSize(deduplicatedSize); + } + + private void sortTimestamps(final int startIndex, final int endIndex) { + Arrays.sort( + this.index, startIndex, endIndex, Comparator.comparingLong(i -> tablet.timestamps[i])); + } + + private void deduplicateTimestamps(final int startIndex, final int endIndex) { + long lastTime = tablet.timestamps[index[startIndex]]; + index[deduplicatedSize++] = index[startIndex]; + for (int i = startIndex + 1; i < endIndex; i++) { + if (lastTime != (lastTime = tablet.timestamps[index[i]])) { + index[deduplicatedSize++] = index[i]; + } + } + } + + /** Sort by time only, and remove only rows with the same DeviceID and time. */ + public void sortAndDeduplicateByTimestampIfNecessary() { + if (tablet == null || tablet.getRowSize() == 0) { + return; + } + + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final long currentTimestamp = tablet.timestamps[i]; + final long previousTimestamp = tablet.timestamps[i - 1]; + + if (currentTimestamp < previousTimestamp) { + isUnSorted = true; + break; + } + if (currentTimestamp == previousTimestamp) { + hasDuplicates = true; + } + } + + if (!isUnSorted && !hasDuplicates) { + return; + } + + index = new Integer[tablet.getRowSize()]; + for (int i = 0, size = tablet.getRowSize(); i < size; i++) { + index[i] = i; + } + + if (isUnSorted) { + sortTimestamps(); + + // Do deduplicate anyway. + // isDeduplicated may be false positive when isUnSorted is true. + deduplicateTimestamps(); + hasDuplicates = false; + } + + if (hasDuplicates) { + deduplicateTimestamps(); + } + + sortAndDeduplicateValuesAndBitMapsIgnoreTimestamp(); + } + + private void sortTimestamps() { + Arrays.sort(this.index, Comparator.comparingLong(i -> tablet.timestamps[i])); + Arrays.sort(tablet.timestamps, 0, tablet.getRowSize()); + } + + private void deduplicateTimestamps() { + deduplicatedSize = 1; + long lastTime = tablet.timestamps[0]; + IDeviceID deviceID = tablet.getDeviceID(index[0]); + final Set deviceIDSet = new HashSet<>(); + deviceIDSet.add(deviceID); + for (int i = 1, size = tablet.getRowSize(); i < size; i++) { + deviceID = tablet.getDeviceID(index[i]); + if ((lastTime == (lastTime = tablet.timestamps[i]))) { + if (!deviceIDSet.contains(deviceID)) { + tablet.timestamps[deduplicatedSize] = lastTime; + index[deduplicatedSize++] = index[i]; + deviceIDSet.add(deviceID); + } + } else { + tablet.timestamps[deduplicatedSize] = lastTime; + index[deduplicatedSize++] = index[i]; + deviceIDSet.clear(); + deviceIDSet.add(deviceID); + } + } + } + + private void sortAndDeduplicateValuesAndBitMapsIgnoreTimestamp() { + int columnIndex = 0; + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.values[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.values[columnIndex], schema.getType(), index); + if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { + tablet.bitMaps[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.bitMaps[columnIndex], index); + } + columnIndex++; + } + } + + tablet.setRowSize(deduplicatedSize); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java similarity index 56% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java index ee34099eb216..f3756b2a46c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/PipeTabletEventSorter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTabletEventSorter.java @@ -17,116 +17,19 @@ * under the License. */ -package org.apache.iotdb.db.pipe.connector.util; +package org.apache.iotdb.db.pipe.connector.util.sorter; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.write.UnSupportedDataTypeException; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; import java.time.LocalDate; -import java.util.Arrays; -import java.util.Comparator; public class PipeTabletEventSorter { - private final Tablet tablet; - - private boolean isSorted = true; - private boolean isDeduplicated = true; - - private Integer[] index; - private int deduplicatedSize; - - public PipeTabletEventSorter(final Tablet tablet) { - this.tablet = tablet; - deduplicatedSize = tablet == null ? 0 : tablet.getRowSize(); - } - - public void deduplicateAndSortTimestampsIfNecessary() { - if (tablet == null || tablet.getRowSize() == 0) { - return; - } - - for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { - final long currentTimestamp = tablet.timestamps[i]; - final long previousTimestamp = tablet.timestamps[i - 1]; - - if (currentTimestamp < previousTimestamp) { - isSorted = false; - } - if (currentTimestamp == previousTimestamp) { - isDeduplicated = false; - } - - if (!isSorted && !isDeduplicated) { - break; - } - } - - if (isSorted && isDeduplicated) { - return; - } - - index = new Integer[tablet.getRowSize()]; - for (int i = 0, size = tablet.getRowSize(); i < size; i++) { - index[i] = i; - } - - if (!isSorted) { - sortTimestamps(); - - // Do deduplicate anyway. - // isDeduplicated may be false positive when isSorted is false. - deduplicateTimestamps(); - isDeduplicated = true; - } - - if (!isDeduplicated) { - deduplicateTimestamps(); - } - - sortAndDeduplicateValuesAndBitMaps(); - } - - private void sortTimestamps() { - Arrays.sort(index, Comparator.comparingLong(i -> tablet.timestamps[i])); - Arrays.sort(tablet.timestamps, 0, tablet.getRowSize()); - } - - private void deduplicateTimestamps() { - deduplicatedSize = 1; - for (int i = 1, size = tablet.getRowSize(); i < size; i++) { - if (tablet.timestamps[i] != tablet.timestamps[i - 1]) { - index[deduplicatedSize] = index[i]; - tablet.timestamps[deduplicatedSize] = tablet.timestamps[i]; - - ++deduplicatedSize; - } - } - tablet.setRowSize(deduplicatedSize); - } - - private void sortAndDeduplicateValuesAndBitMaps() { - int columnIndex = 0; - for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { - final IMeasurementSchema schema = tablet.getSchemas().get(i); - if (schema != null) { - tablet.values[columnIndex] = - reorderValueList(deduplicatedSize, tablet.values[columnIndex], schema.getType(), index); - if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { - tablet.bitMaps[columnIndex] = - reorderBitMap(deduplicatedSize, tablet.bitMaps[columnIndex], index); - } - columnIndex++; - } - } - } - - private static Object reorderValueList( - int deduplicatedSize, + public static Object reorderValueList( + final int deduplicatedSize, final Object valueList, final TSDataType dataType, final Integer[] index) { @@ -189,8 +92,8 @@ private static Object reorderValueList( } } - private static BitMap reorderBitMap( - int deduplicatedSize, final BitMap bitMap, final Integer[] index) { + public static BitMap reorderBitMap( + final int deduplicatedSize, final BitMap bitMap, final Integer[] index) { final BitMap deduplicatedBitMap = new BitMap(bitMap.getSize()); for (int i = 0; i < deduplicatedSize; i++) { if (bitMap.isMarked(index[i])) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTreeModelTabletEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTreeModelTabletEventSorter.java new file mode 100644 index 000000000000..42f8c2dac52c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/sorter/PipeTreeModelTabletEventSorter.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.connector.util.sorter; + +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +import java.util.Arrays; +import java.util.Comparator; + +public class PipeTreeModelTabletEventSorter { + + private final Tablet tablet; + + private boolean isSorted = true; + private boolean isDeduplicated = true; + + private Integer[] index; + private int deduplicatedSize; + + public PipeTreeModelTabletEventSorter(final Tablet tablet) { + this.tablet = tablet; + deduplicatedSize = tablet == null ? 0 : tablet.getRowSize(); + } + + public void deduplicateAndSortTimestampsIfNecessary() { + if (tablet == null || tablet.getRowSize() == 0) { + return; + } + + for (int i = 1, size = tablet.getRowSize(); i < size; ++i) { + final long currentTimestamp = tablet.timestamps[i]; + final long previousTimestamp = tablet.timestamps[i - 1]; + + if (currentTimestamp < previousTimestamp) { + isSorted = false; + break; + } + if (currentTimestamp == previousTimestamp) { + isDeduplicated = false; + } + } + + if (isSorted && isDeduplicated) { + return; + } + + index = new Integer[tablet.getRowSize()]; + for (int i = 0, size = tablet.getRowSize(); i < size; i++) { + index[i] = i; + } + + if (!isSorted) { + sortTimestamps(); + + // Do deduplicate anyway. + // isDeduplicated may be false positive when isSorted is false. + deduplicateTimestamps(); + isDeduplicated = true; + } + + if (!isDeduplicated) { + deduplicateTimestamps(); + } + + sortAndDeduplicateValuesAndBitMaps(); + } + + private void sortTimestamps() { + Arrays.sort(index, Comparator.comparingLong(i -> tablet.timestamps[i])); + Arrays.sort(tablet.timestamps, 0, tablet.getRowSize()); + } + + private void deduplicateTimestamps() { + deduplicatedSize = 1; + for (int i = 1, size = tablet.getRowSize(); i < size; i++) { + if (tablet.timestamps[i] != tablet.timestamps[i - 1]) { + index[deduplicatedSize] = index[i]; + tablet.timestamps[deduplicatedSize] = tablet.timestamps[i]; + + ++deduplicatedSize; + } + } + tablet.setRowSize(deduplicatedSize); + } + + private void sortAndDeduplicateValuesAndBitMaps() { + int columnIndex = 0; + for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { + final IMeasurementSchema schema = tablet.getSchemas().get(i); + if (schema != null) { + tablet.values[columnIndex] = + PipeTabletEventSorter.reorderValueList( + deduplicatedSize, tablet.values[columnIndex], schema.getType(), index); + if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { + tablet.bitMaps[columnIndex] = + PipeTabletEventSorter.reorderBitMap( + deduplicatedSize, tablet.bitMaps[columnIndex], index); + } + columnIndex++; + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java index bf9a67061b43..0a033aff297b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java @@ -48,6 +48,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -134,7 +135,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { .getStringOrDefault( SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); - final boolean isTreeModelDataAllowedToBeCaptured = + final boolean isCaptureTree = validator .getParameters() .getBooleanOrDefault( @@ -142,7 +143,7 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), isTreeDialect); - final boolean isTableModelDataAllowedToBeCaptured = + final boolean isCaptureTable = validator .getParameters() .getBooleanOrDefault( @@ -150,6 +151,21 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), !isTreeDialect); + if (!isCaptureTree && !isCaptureTable) { + throw new PipeParameterNotValidException( + "capture.tree and capture.table can not both be specified as false"); + } + + final boolean isDoubleLiving = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + final boolean isTreeModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTree; + final boolean isTableModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTable; if (!isTreeModelDataAllowedToBeCaptured && validator .getParameters() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java index 6bf7b7b7f6e7..eb7491e460ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/pipeconsensus/PipeConsensusReceiver.java @@ -31,6 +31,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp; import org.apache.iotdb.commons.pipe.receiver.IoTDBReceiverAgent; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; import org.apache.iotdb.consensus.pipe.PipeConsensus; import org.apache.iotdb.consensus.pipe.PipeConsensusServerImpl; @@ -88,8 +89,6 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; -import static org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils.generateTsFileResource; - public class PipeConsensusReceiver { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusReceiver.class); private static final IoTDBConfig IOTDB_CONFIG = IoTDBDescriptor.getInstance().getConfig(); @@ -849,7 +848,7 @@ private void closeCurrentWritingFileWriter( private void deleteFile(File file) { if (file.exists()) { try { - FileUtils.delete(file); + RetryUtils.retryOnException(() -> FileUtils.delete(file)); LOGGER.info( "PipeConsensus-PipeName-{}: Original writing file {} was deleted.", consensusPipeName, @@ -943,7 +942,11 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, if (receiverFileDirWithIdSuffix.get() != null) { if (receiverFileDirWithIdSuffix.get().exists()) { try { - FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + return null; + }); LOGGER.info( "PipeConsensus-PipeName-{}: Original receiver file dir {} was deleted successfully.", consensusPipeName, @@ -1005,7 +1008,11 @@ private void initiateTsFileBufferFolder() throws DiskSpaceInsufficientException, } // Remove exists dir if (newReceiverDir.exists()) { - FileUtils.deleteDirectory(newReceiverDir); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(newReceiverDir); + return null; + }); LOGGER.info( "PipeConsensus-PipeName-{}: Origin receiver file dir {} was deleted.", consensusPipeName, @@ -1036,7 +1043,11 @@ public synchronized void handleExit() { if (receiverFileDirWithIdSuffix.get() != null) { if (receiverFileDirWithIdSuffix.get().exists()) { try { - FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + return null; + }); LOGGER.info( "PipeConsensus-PipeName-{}: Receiver exit: Original receiver file dir {} was deleted.", consensusPipeName, @@ -1172,7 +1183,11 @@ public void setFilePath(String receiverBasePath) throws IOException { File tsFileWriterDirectory = new File(this.localWritingDirPath); // Remove exists dir if (tsFileWriterDirectory.exists()) { - FileUtils.deleteDirectory(tsFileWriterDirectory); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(tsFileWriterDirectory); + return null; + }); LOGGER.info( "PipeConsensus-PipeName-{}: Origin receiver tsFileWriter-{} file dir {} was deleted.", consensusPipeName, @@ -1267,7 +1282,7 @@ public void closeSelf(ConsensusPipeName consensusPipeName) { // close file if (writingFile != null) { try { - FileUtils.delete(writingFile); + RetryUtils.retryOnException(() -> FileUtils.delete(writingFile)); LOGGER.info( "PipeConsensus-PipeName-{}: TsFileWriter exit: Writing file {} was deleted.", consensusPipeName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index a87e229a03a2..b6d5d525cf67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -35,6 +35,7 @@ import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.db.auth.AuthorityChecker; @@ -559,7 +560,11 @@ private TSStatus loadTsFileAsync(final String dataBaseName, final List a final File sourceFile = new File(absolutePath); if (!Objects.equals( loadActiveListeningPipeDir, sourceFile.getParentFile().getAbsolutePath())) { - FileUtils.moveFileWithMD5Check(sourceFile, new File(loadActiveListeningPipeDir)); + RetryUtils.retryOnException( + () -> { + FileUtils.moveFileWithMD5Check(sourceFile, new File(loadActiveListeningPipeDir)); + return null; + }); } } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java index f249a7734552..edf8b636ecf3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertRowStatement.java @@ -28,10 +28,14 @@ import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.ZoneId; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; public class PipeConvertedInsertRowStatement extends InsertRowStatement { @@ -43,7 +47,6 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme // Statement isDebug = insertRowStatement.isDebug(); // InsertBaseStatement - insertRowStatement.removeAllFailedMeasurementMarks(); devicePath = insertRowStatement.getDevicePath(); isAligned = insertRowStatement.isAligned(); measurementSchemas = insertRowStatement.getMeasurementSchemas(); @@ -59,6 +62,31 @@ public PipeConvertedInsertRowStatement(final InsertRowStatement insertRowStateme values = insertRowStatement.getValues(); isNeedInferType = insertRowStatement.isNeedInferType(); deviceID = insertRowStatement.getRawTableDeviceID(); + + // To ensure that the measurement remains unchanged during the WAL writing process, the array + // needs to be copied before the failed Measurement mark can be deleted. + final MeasurementSchema[] measurementSchemas = insertRowStatement.getMeasurementSchemas(); + if (measurementSchemas != null) { + this.measurementSchemas = Arrays.copyOf(measurementSchemas, measurementSchemas.length); + } + + final String[] measurements = insertRowStatement.getMeasurements(); + if (measurements != null) { + this.measurements = Arrays.copyOf(measurements, measurements.length); + } + + final TSDataType[] dataTypes = insertRowStatement.getDataTypes(); + if (dataTypes != null) { + this.dataTypes = Arrays.copyOf(dataTypes, dataTypes.length); + } + + final Map failedMeasurementIndex2Info = + insertRowStatement.getFailedMeasurementInfoMap(); + if (failedMeasurementIndex2Info != null) { + this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); + } + + removeAllFailedMeasurementMarks(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java index 606309366cc3..eee47a5b0b18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/statement/PipeConvertedInsertTabletStatement.java @@ -24,25 +24,27 @@ import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + public class PipeConvertedInsertTabletStatement extends InsertTabletStatement { private static final Logger LOGGER = LoggerFactory.getLogger(PipeConvertedInsertTabletStatement.class); - public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + public PipeConvertedInsertTabletStatement( + final InsertTabletStatement insertTabletStatement, boolean isCopyMeasurement) { super(); // Statement isDebug = insertTabletStatement.isDebug(); // InsertBaseStatement - insertTabletStatement.removeAllFailedMeasurementMarks(); devicePath = insertTabletStatement.getDevicePath(); isAligned = insertTabletStatement.isAligned(); - measurementSchemas = insertTabletStatement.getMeasurementSchemas(); - measurements = insertTabletStatement.getMeasurements(); - dataTypes = insertTabletStatement.getDataTypes(); columnCategories = insertTabletStatement.getColumnCategories(); idColumnIndices = insertTabletStatement.getIdColumnIndices(); attrColumnIndices = insertTabletStatement.getAttrColumnIndices(); @@ -55,6 +57,42 @@ public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabl deviceIDs = insertTabletStatement.getRawTableDeviceIDs(); singleDevice = insertTabletStatement.isSingleDevice(); rowCount = insertTabletStatement.getRowCount(); + + // To ensure that the measurement remains unchanged during the WAL writing process, the array + // needs to be copied before the failed Measurement mark can be deleted. + if (isCopyMeasurement) { + final MeasurementSchema[] measurementSchemas = insertTabletStatement.getMeasurementSchemas(); + if (measurementSchemas != null) { + this.measurementSchemas = Arrays.copyOf(measurementSchemas, measurementSchemas.length); + } + + final String[] measurements = insertTabletStatement.getMeasurements(); + if (measurements != null) { + this.measurements = Arrays.copyOf(measurements, measurements.length); + } + + final TSDataType[] dataTypes = insertTabletStatement.getDataTypes(); + if (dataTypes != null) { + this.dataTypes = Arrays.copyOf(dataTypes, dataTypes.length); + } + + final Map failedMeasurementIndex2Info = + insertTabletStatement.getFailedMeasurementInfoMap(); + if (failedMeasurementIndex2Info != null) { + this.failedMeasurementIndex2Info = new HashMap<>(failedMeasurementIndex2Info); + } + } else { + this.measurementSchemas = insertTabletStatement.getMeasurementSchemas(); + this.measurements = insertTabletStatement.getMeasurements(); + this.dataTypes = insertTabletStatement.getDataTypes(); + this.failedMeasurementIndex2Info = insertTabletStatement.getFailedMeasurementInfoMap(); + } + + removeAllFailedMeasurementMarks(); + } + + public PipeConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + this(insertTabletStatement, true); } @Override @@ -71,6 +109,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { return true; } + protected boolean originalCheckAndCastDataType(int columnIndex, TSDataType dataType) { + return super.checkAndCastDataType(columnIndex, dataType); + } + @TableModel @Override public boolean isForceTypeConversion() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 39cae6ab23d8..11858853a2da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; -import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReqV2; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.table.TsFileInsertionEventTableParser; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; @@ -137,10 +137,12 @@ public Optional visitLoadFile( final PipeConvertedInsertTabletStatement statement = new PipeConvertedInsertTabletStatement( - PipeTransferTabletRawReq.toTPipeTransferRawReq( + PipeTransferTabletRawReqV2.toTPipeTransferRawReq( rawTabletInsertionEvent.convertToTablet(), - rawTabletInsertionEvent.isAligned()) - .constructStatement()); + rawTabletInsertionEvent.isAligned(), + databaseName) + .constructStatement(), + false); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java index eef40c714dc8..ed47e519b00e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java @@ -106,7 +106,8 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { new PipeConvertedInsertTabletStatement( PipeTransferTabletRawReq.toTPipeTransferRawReq( tabletWithIsAligned.getLeft(), tabletWithIsAligned.getRight()) - .constructStatement()); + .constructStatement(), + false); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 8854c0df3c5b..f50fbfd63bdb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -91,6 +91,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp; import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TDescTableResp; import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq; @@ -155,12 +156,15 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTTLResp; +import org.apache.iotdb.confignode.rpc.thrift.TShowTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TShowTableResp; import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1052,12 +1056,24 @@ public TSStatus startPipe(String pipeName) throws TException { () -> client.startPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus startPipeExtended(TStartPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.startPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus stopPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( () -> client.stopPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.stopPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus dropPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( @@ -1288,6 +1304,12 @@ public TShowTableResp showTables(final String database, final boolean isDetails) () -> client.showTables(database, isDetails), resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TShowTable4InformationSchemaResp showTables4InformationSchema() throws TException { + return executeRemoteCallWithRetry( + () -> client.showTables4InformationSchema(), resp -> !updateConfigNodeLeader(resp.status)); + } + @Override public TDescTableResp describeTable( final String database, final String tableName, final boolean isDetails) throws TException { @@ -1296,6 +1318,12 @@ public TDescTableResp describeTable( resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TDescTable4InformationSchemaResp descTables4InformationSchema() throws TException { + return executeRemoteCallWithRetry( + () -> client.descTables4InformationSchema(), resp -> !updateConfigNodeLeader(resp.status)); + } + @Override public TFetchTableResp fetchTables(final Map> fetchTableMap) throws TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java index e9f6fba26f68..bbc9b2c155ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/EnforceSingleRowOperator.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.process; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -59,7 +60,7 @@ public TsBlock next() throws Exception { return tsBlock; } if (tsBlock.getPositionCount() > 1 || finished) { - throw new IllegalStateException(MULTIPLE_ROWS_ERROR_MESSAGE); + throw new SemanticException(MULTIPLE_ROWS_ERROR_MESSAGE); } finished = true; return tsBlock; @@ -83,7 +84,7 @@ public boolean isFinished() throws Exception { if (childFinished && !finished) { // finished == false means the child has no result returned up to now, but we need at least // one result. - throw new IllegalStateException(NO_RESULT_ERROR_MESSAGE); + throw new SemanticException(NO_RESULT_ERROR_MESSAGE); } // Even if finished == true, we can not return true here, we need to call child.next() to check // if child has more data. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java index a64ff35257d0..33620690fed6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/JoinKeyComparatorFactory.java @@ -36,7 +36,7 @@ public static List getComparators( return comparators; } - private static JoinKeyComparator getComparator(Type type, boolean isAscending) { + public static JoinKeyComparator getComparator(Type type, boolean isAscending) { switch (type.getTypeEnum()) { case INT32: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java index dfa0cf4baf1f..fd18ff5ed83d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceQuerySource.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; @@ -189,8 +190,7 @@ public static List getDevicePatternList( final String tableName, final List> idDeterminedPredicateList) { if (Objects.isNull(DataNodeTableCache.getInstance().getTable(database, tableName))) { - throw new SchemaExecutionException( - String.format("Table '%s.%s' does not exist.", database, tableName)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } return DeviceFilterUtil.convertToDevicePattern( database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index 41abb73f54e4..ccb75ebb22f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -19,76 +19,364 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.schema.table.InformationSchema; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; +import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; +import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.Pair; +import java.security.AccessControlException; +import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE; +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; +import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; +import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; public class InformationSchemaContentSupplierFactory { private InformationSchemaContentSupplierFactory() {} public static Iterator getSupplier( - final String tableName, final List dataTypes) { - if (tableName.equals(InformationSchema.QUERIES)) { - return new Iterator() { - private final TsBlockBuilder resultBuilder = new TsBlockBuilder(dataTypes); - private final ColumnBuilder[] columnBuilders = resultBuilder.getValueColumnBuilders(); + final String tableName, final List dataTypes, final String userName) { + switch (tableName) { + case InformationSchema.QUERIES: + return new QueriesSupplier(dataTypes); + case InformationSchema.DATABASES: + return new DatabaseSupplier(dataTypes, userName); + case InformationSchema.TABLES: + return new TableSupplier(dataTypes, userName); + case InformationSchema.COLUMNS: + return new ColumnSupplier(dataTypes, userName); + default: + throw new UnsupportedOperationException("Unknown table: " + tableName); + } + } + + private static class QueriesSupplier extends TsBlockSupplier { + private final long currTime = System.currentTimeMillis(); + // We initialize it later for the convenience of data preparation + protected int totalSize; + protected int nextConsumedIndex; + private final List queryExecutions = + Coordinator.getInstance().getAllQueryExecutions(); + + private QueriesSupplier(final List dataTypes) { + super(dataTypes); + this.totalSize = queryExecutions.size(); + } + + @Override + protected void constructLine() { + IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); + + if (queryExecution.getSQLDialect().equals(IClientSession.SqlDialect.TABLE)) { + String[] splits = queryExecution.getQueryId().split("_"); + int dataNodeId = Integer.parseInt(splits[splits.length - 1]); + + columnBuilders[0].writeBinary(BytesUtils.valueOf(queryExecution.getQueryId())); + columnBuilders[1].writeLong(queryExecution.getStartExecutionTime()); + columnBuilders[2].writeInt(dataNodeId); + columnBuilders[3].writeFloat( + (float) (currTime - queryExecution.getStartExecutionTime()) / 1000); + columnBuilders[4].writeBinary( + BytesUtils.valueOf(queryExecution.getExecuteSQL().orElse("UNKNOWN"))); + resultBuilder.declarePosition(); + } + nextConsumedIndex++; + } + + @Override + public boolean hasNext() { + return nextConsumedIndex < totalSize; + } + } + + private static class DatabaseSupplier extends TsBlockSupplier { + private Iterator> iterator; + private TDatabaseInfo currentDatabase; + private final String userName; + private boolean hasShownInformationSchema; + + private DatabaseSupplier(final List dataTypes, final String userName) { + super(dataTypes); + this.userName = userName; + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final TShowDatabaseResp resp = + client.showDatabase( + new TGetDatabaseReq(Arrays.asList(ALL_RESULT_NODES), ALL_MATCH_SCOPE.serialize()) + .setIsTableModel(true)); + iterator = resp.getDatabaseInfoMap().entrySet().iterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + if (!hasShownInformationSchema) { + InformationSchemaUtils.buildDatabaseTsBlock(s -> true, resultBuilder, true, false); + hasShownInformationSchema = true; + return; + } + columnBuilders[0].writeBinary( + new Binary(currentDatabase.getName(), TSFileConfig.STRING_CHARSET)); + + if (Long.MAX_VALUE == currentDatabase.getTTL()) { + columnBuilders[1].writeBinary( + new Binary(IoTDBConstant.TTL_INFINITE, TSFileConfig.STRING_CHARSET)); + } else { + columnBuilders[1].writeBinary( + new Binary(String.valueOf(currentDatabase.getTTL()), TSFileConfig.STRING_CHARSET)); + } + columnBuilders[2].writeInt(currentDatabase.getSchemaReplicationFactor()); + columnBuilders[3].writeInt(currentDatabase.getDataReplicationFactor()); + columnBuilders[4].writeLong(currentDatabase.getTimePartitionInterval()); + columnBuilders[5].writeInt(currentDatabase.getSchemaRegionNum()); + columnBuilders[6].writeInt(currentDatabase.getDataRegionNum()); + resultBuilder.declarePosition(); + currentDatabase = null; + } + + @Override + public boolean hasNext() { + if (!hasShownInformationSchema) { + if (!canShowDB(userName, InformationSchema.INFORMATION_DATABASE)) { + hasShownInformationSchema = true; + } else { + return true; + } + } + while (iterator.hasNext()) { + final Map.Entry result = iterator.next(); + if (!canShowDB(userName, result.getKey())) { + continue; + } + currentDatabase = result.getValue(); + break; + } + return Objects.nonNull(currentDatabase); + } + } - private final List queryExecutions = - Coordinator.getInstance().getAllQueryExecutions(); + private static class TableSupplier extends TsBlockSupplier { + private Iterator>> dbIterator; + private Iterator tableInfoIterator = null; + private String dbName; + private final String userName; - private final long currTime = System.currentTimeMillis(); + private TableSupplier(final List dataTypes, final String userName) { + super(dataTypes); + this.userName = userName; + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final Map> databaseTableInfoMap = + client.showTables4InformationSchema().getDatabaseTableInfoMap(); + databaseTableInfoMap.put( + InformationSchema.INFORMATION_DATABASE, + InformationSchema.getSchemaTables().values().stream() + .map( + table -> { + final TTableInfo info = + new TTableInfo( + table.getTableName(), + table.getPropValue(TTL_PROPERTY).orElse(TTL_INFINITE)); + info.setState(TableNodeStatus.USING.ordinal()); + return info; + }) + .collect(Collectors.toList())); + dbIterator = databaseTableInfoMap.entrySet().iterator(); + } catch (final Exception e) { + lastException = e; + } + } - private final int totalSize = queryExecutions.size(); - private int nextConsumedIndex; + @Override + protected void constructLine() { + final TTableInfo info = tableInfoIterator.next(); + columnBuilders[0].writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary(new Binary(info.getTableName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary(new Binary(info.getTTL(), TSFileConfig.STRING_CHARSET)); + columnBuilders[3].writeBinary( + new Binary( + TableNodeStatus.values()[info.getState()].toString(), TSFileConfig.STRING_CHARSET)); + resultBuilder.declarePosition(); + } - @Override - public boolean hasNext() { - return nextConsumedIndex < totalSize; + @Override + public boolean hasNext() { + // Get next table info iterator + while (Objects.isNull(tableInfoIterator) || !tableInfoIterator.hasNext()) { + if (!dbIterator.hasNext()) { + return false; } + final Map.Entry> entry = dbIterator.next(); + dbName = entry.getKey(); + if (!canShowDB(userName, dbName)) { + continue; + } + tableInfoIterator = entry.getValue().iterator(); + } + return true; + } + } - @Override - public TsBlock next() { - while (nextConsumedIndex < totalSize && !resultBuilder.isFull()) { - - IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); - - if (queryExecution.getSQLDialect().equals(IClientSession.SqlDialect.TABLE)) { - String[] splits = queryExecution.getQueryId().split("_"); - int dataNodeId = Integer.parseInt(splits[splits.length - 1]); - - columnBuilders[0].writeBinary(BytesUtils.valueOf(queryExecution.getQueryId())); - columnBuilders[1].writeLong(queryExecution.getStartExecutionTime()); - columnBuilders[2].writeInt(dataNodeId); - columnBuilders[3].writeFloat( - (float) (currTime - queryExecution.getStartExecutionTime()) / 1000); - columnBuilders[4].writeBinary( - BytesUtils.valueOf(queryExecution.getExecuteSQL().orElse("UNKNOWN"))); - resultBuilder.declarePosition(); - } - nextConsumedIndex++; + private static class ColumnSupplier extends TsBlockSupplier { + private Iterator>>>> dbIterator; + private Iterator>>> tableInfoIterator; + private Iterator columnSchemaIterator; + private String dbName; + private String tableName; + private Set preDeletedColumns; + private final String userName; + + private ColumnSupplier(final List dataTypes, final String userName) { + super(dataTypes); + this.userName = userName; + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final TDescTable4InformationSchemaResp resp = client.descTables4InformationSchema(); + final Map>>> resultMap = + resp.getTableColumnInfoMap().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + tableEntry -> + new Pair<>( + TsTableInternalRPCUtil.deserializeSingleTsTable( + tableEntry.getValue().getTableInfo()), + tableEntry.getValue().getPreDeletedColumns()))))); + resultMap.put( + InformationSchema.INFORMATION_DATABASE, + InformationSchema.getSchemaTables().values().stream() + .collect( + Collectors.toMap( + TsTable::getTableName, + table -> new Pair<>(table, Collections.emptySet())))); + dbIterator = resultMap.entrySet().iterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + final TsTableColumnSchema schema = columnSchemaIterator.next(); + columnBuilders[0].writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary(new Binary(tableName, TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary( + new Binary(schema.getColumnName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[3].writeBinary( + new Binary(schema.getDataType().name(), TSFileConfig.STRING_CHARSET)); + columnBuilders[4].writeBinary( + new Binary(schema.getColumnCategory().name(), TSFileConfig.STRING_CHARSET)); + columnBuilders[5].writeBinary( + new Binary( + preDeletedColumns.contains(schema.getColumnName()) ? "PRE_DELETE" : "USING", + TSFileConfig.STRING_CHARSET)); + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + while (Objects.isNull(columnSchemaIterator) || !columnSchemaIterator.hasNext()) { + while (Objects.isNull(tableInfoIterator) || !tableInfoIterator.hasNext()) { + if (!dbIterator.hasNext()) { + return false; + } + final Map.Entry>>> entry = + dbIterator.next(); + dbName = entry.getKey(); + if (!canShowDB(userName, dbName)) { + continue; } - TsBlock result = - resultBuilder.build( - new RunLengthEncodedColumn( - TableScanOperator.TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount())); - resultBuilder.reset(); - return result; + tableInfoIterator = entry.getValue().entrySet().iterator(); } - }; - } else { - throw new UnsupportedOperationException("Unknown table: " + tableName); + final Map.Entry>> tableEntry = tableInfoIterator.next(); + tableName = tableEntry.getKey(); + preDeletedColumns = tableEntry.getValue().getRight(); + columnSchemaIterator = tableEntry.getValue().getLeft().getColumnList().iterator(); + } + return true; } } + + private static boolean canShowDB(final String userName, final String dbName) { + try { + Coordinator.getInstance().getAccessControl().checkCanShowOrUseDatabase(userName, dbName); + } catch (final AccessControlException e) { + return false; + } + return true; + } + + private abstract static class TsBlockSupplier implements Iterator { + + protected final TsBlockBuilder resultBuilder; + protected final ColumnBuilder[] columnBuilders; + protected Exception lastException; + + private TsBlockSupplier(final List dataTypes) { + this.resultBuilder = new TsBlockBuilder(dataTypes); + this.columnBuilders = resultBuilder.getValueColumnBuilders(); + } + + @Override + public TsBlock next() { + if (Objects.nonNull(lastException)) { + throw new NoSuchElementException(lastException.getMessage()); + } + if (!hasNext()) { + throw new NoSuchElementException(); + } + while (hasNext() && !resultBuilder.isFull()) { + constructLine(); + } + final TsBlock result = + resultBuilder.build( + new RunLengthEncodedColumn( + TableScanOperator.TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount())); + resultBuilder.reset(); + return result; + } + + protected abstract void constructLine(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java new file mode 100644 index 000000000000..b21ed578965f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/MergeSortSemiJoinOperator.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparator; + +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Collections; +import java.util.List; + +public class MergeSortSemiJoinOperator extends AbstractMergeSortJoinOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(MergeSortSemiJoinOperator.class); + + private final int outputColumnNum; + + private boolean rightHasNullValue = false; + + public MergeSortSemiJoinOperator( + OperatorContext operatorContext, + Operator leftChild, + int leftJoinKeyPosition, + int[] leftOutputSymbolIdx, + Operator rightChild, + int rightJoinKeyPosition, + JoinKeyComparator joinKeyComparator, + List dataTypes) { + super( + operatorContext, + leftChild, + new int[] {leftJoinKeyPosition}, + leftOutputSymbolIdx, + rightChild, + new int[] {rightJoinKeyPosition}, + null, + Collections.singletonList(joinKeyComparator), + dataTypes); + outputColumnNum = dataTypes.size(); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + + return !leftFinished; + } + + @Override + protected boolean prepareInput() throws Exception { + gotCandidateBlocks(); + if (rightFinished) { + return leftBlockNotEmpty(); + } + return leftBlockNotEmpty() && rightBlockNotEmpty() && gotNextRightBlock(); + } + + @Override + protected boolean processFinished() { + if (rightFinished) { + appendAllLeftBlock(); + return true; + } + + // skip all NULL values in right, because NULL value will not match the left value + while (currentRightHasNullValue()) { + rightHasNullValue = true; + if (rightFinishedWithIncIndex()) { + return true; + } + } + // all the join keys in rightTsBlock are less than leftTsBlock, just skip right + if (allRightLessThanLeft()) { + resetRightBlockList(); + return true; + } + + // all the join Keys in leftTsBlock are less than rightTsBlock, just append the left value + if (allLeftLessThanRight()) { + appendAllLeftBlock(); + resetLeftBlock(); + return true; + } + + // continue right < left, until right >= left + while (lessThan( + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex, + leftBlock, + leftJoinKeyPositions, + leftIndex)) { + if (rightFinishedWithIncIndex()) { + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // if current left is null, append null to result + while (currentLeftHasNullValue()) { + appendNullValueToResult(); + if (leftFinishedWithIncIndex()) { + return true; + } + } + + // continue left < right, until left >= right + while (lessThan( + leftBlock, + leftJoinKeyPositions, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex)) { + appendWhenNotMatch(); + leftIndex++; + if (leftIndex >= leftBlock.getPositionCount()) { + resetLeftBlock(); + return true; + } + } + if (currentRoundNeedStop()) { + return true; + } + + // has right value equals to current left, append to join result, inc leftIndex + return hasMatchedRightValueToProbeLeft() && leftFinishedWithIncIndex(); + } + + @Override + protected boolean hasMatchedRightValueToProbeLeft() { + boolean matches = + equalsTo( + leftBlock, + leftJoinKeyPositions, + leftIndex, + rightBlockList.get(rightBlockListIdx), + rightJoinKeyPositions, + rightIndex); + if (matches) { + appendValueToResult(true); + } else { + appendWhenNotMatch(); + } + + return matches; + } + + private void appendWhenNotMatch() { + // current left won't match any right, append left with false SemiJoin result + if (!rightHasNullValue) { + appendValueToResult(false); + } else { + // if right has null value, append null to result. This behaves like MySQL and Trino. + appendNullValueToResult(); + } + } + + private void appendValueToResult(boolean matches) { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.writeBoolean(matches); + resultBuilder.declarePosition(); + } + + private void appendNullValueToResult() { + appendLeftBlockData(leftOutputSymbolIdx, resultBuilder, leftBlock, leftIndex); + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(outputColumnNum - 1); + columnBuilder.appendNull(); + resultBuilder.declarePosition(); + } + + private void appendAllLeftBlock() { + if (rightHasNullValue) { + while (leftBlockNotEmpty()) { + appendNullValueToResult(); + leftIndex++; + } + } else { + while (leftBlockNotEmpty()) { + appendValueToResult(false); + leftIndex++; + } + } + } + + @Override + protected void recordsWhenDataMatches() { + // do nothing + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(leftChild) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(rightChild) + + RamUsageEstimator.sizeOf(leftOutputSymbolIdx) + + RamUsageEstimator.sizeOf(rightOutputSymbolIdx) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 9151ca14cba4..1b1225234421 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -527,6 +527,10 @@ public static Coordinator getInstance() { return INSTANCE; } + public AccessControl getAccessControl() { + return accessControl; + } + public void recordExecutionTime(long queryId, long executionTime) { IQueryExecution queryExecution = getQueryExecution(queryId); if (queryExecution != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index e97ecd32d40e..156ceb5a73b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -2983,6 +2983,12 @@ public Analysis visitInsertRowsOfOneDevice( @Override public Analysis visitPipeEnrichedStatement( PipeEnrichedStatement pipeEnrichedStatement, MPPQueryContext context) { + // The LoadTsFileStatement is a special case, it needs isGeneratedByPipe information + // in the analyzer to execute the tsfile-tablet conversion in some cases. + if (pipeEnrichedStatement.getInnerStatement() instanceof LoadTsFileStatement) { + ((LoadTsFileStatement) pipeEnrichedStatement.getInnerStatement()).markIsGeneratedByPipe(); + } + Analysis analysis = pipeEnrichedStatement.getInnerStatement().accept(this, context); analysis.setDatabaseName(context.getDatabaseName().orElse(null)); @@ -3020,12 +3026,16 @@ private LoadTsFileAnalyzer getAnalyzer( LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { if (Objects.equals(loadTsFileStatement.getModel(), LoadTsFileConfigurator.MODEL_TREE_VALUE)) { // Load to tree-model - return new LoadTsFileToTreeModelAnalyzer(loadTsFileStatement, context); + return new LoadTsFileToTreeModelAnalyzer( + loadTsFileStatement, loadTsFileStatement.isGeneratedByPipe(), context); } else { // Load to table-model if (Objects.nonNull(loadTsFileStatement.getDatabase())) { return new LoadTsFileToTableModelAnalyzer( - loadTsFileStatement, LocalExecutionPlanner.getInstance().metadata, context); + loadTsFileStatement, + loadTsFileStatement.isGeneratedByPipe(), + LocalExecutionPlanner.getInstance().metadata, + context); } else { throw new SemanticException( "Database name must be specified when loading data into the table model."); @@ -3511,6 +3521,18 @@ public Analysis visitDeleteData( logicalViewSchema = (LogicalViewSchema) measurementSchema; if (logicalViewSchema.isWritable()) { sourcePathOfAliasSeries = logicalViewSchema.getSourcePathIfWritable(); + // if the source path can be matched by any of the deletion pattern, do not add it + boolean pathMatched = false; + for (MeasurementPath deletionPattern : deleteDataStatement.getPathList()) { + if (deletionPattern.matchFullPath(sourcePathOfAliasSeries)) { + pathMatched = true; + break; + } + } + if (pathMatched) { + continue; + } + deletePatternSet.add(new MeasurementPath(sourcePathOfAliasSeries.getNodes())); deduplicatedDeviceIDs.add(sourcePathOfAliasSeries.getIDeviceID()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 2679b6c196af..83d685c007cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -22,8 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadReadOnlyException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -63,6 +63,8 @@ public abstract class LoadTsFileAnalyzer implements AutoCloseable { private final boolean isTableModelStatement; + private final boolean isGeneratedByPipe; + protected final List tsFiles; protected final String statementString; protected final boolean isVerifySchema; @@ -82,7 +84,8 @@ public abstract class LoadTsFileAnalyzer implements AutoCloseable { final IPartitionFetcher partitionFetcher = ClusterPartitionFetcher.getInstance(); final ISchemaFetcher schemaFetcher = ClusterSchemaFetcher.getInstance(); - LoadTsFileAnalyzer(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { + LoadTsFileAnalyzer( + LoadTsFileStatement loadTsFileStatement, boolean isGeneratedByPipe, MPPQueryContext context) { this.loadTsFileTreeStatement = loadTsFileStatement; this.tsFiles = loadTsFileStatement.getTsFiles(); this.statementString = loadTsFileStatement.toString(); @@ -95,10 +98,12 @@ public abstract class LoadTsFileAnalyzer implements AutoCloseable { this.loadTsFileTableStatement = null; this.isTableModelStatement = false; + this.isGeneratedByPipe = isGeneratedByPipe; this.context = context; } - LoadTsFileAnalyzer(LoadTsFile loadTsFileTableStatement, MPPQueryContext context) { + LoadTsFileAnalyzer( + LoadTsFile loadTsFileTableStatement, boolean isGeneratedByPipe, MPPQueryContext context) { this.loadTsFileTableStatement = loadTsFileTableStatement; this.tsFiles = loadTsFileTableStatement.getTsFiles(); this.statementString = loadTsFileTableStatement.toString(); @@ -111,6 +116,7 @@ public abstract class LoadTsFileAnalyzer implements AutoCloseable { this.loadTsFileTreeStatement = null; this.isTableModelStatement = true; + this.isGeneratedByPipe = isGeneratedByPipe; this.context = context; } @@ -143,8 +149,8 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { } catch (AuthException e) { setFailAnalysisForAuthException(analysis, e); return false; - } catch (VerifyMetadataTypeMismatchException e) { - executeDataTypeConversionOnTypeMismatch(analysis, e); + } catch (LoadAnalyzeException e) { + executeTabletConversion(analysis, e); // just return false to STOP the analysis process, // the real result on the conversion will be set in the analysis. return false; @@ -170,15 +176,14 @@ protected boolean doAnalyzeFileByFile(IAnalysis analysis) { } protected abstract void analyzeSingleTsFile(final File tsFile) - throws IOException, AuthException, VerifyMetadataException; + throws IOException, AuthException, LoadAnalyzeException; - protected void executeDataTypeConversionOnTypeMismatch( - final IAnalysis analysis, final VerifyMetadataTypeMismatchException e) { + protected void executeTabletConversion(final IAnalysis analysis, final LoadAnalyzeException e) { final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter = - new LoadTsFileDataTypeConverter(); + new LoadTsFileDataTypeConverter(isGeneratedByPipe); final TSStatus status = - isConvertOnTypeMismatch + (!(e instanceof LoadAnalyzeTypeMismatchException) || isConvertOnTypeMismatch) ? (isTableModelStatement ? loadTsFileDataTypeConverter .convertForTableModel(loadTsFileTableStatement) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index a3edfabadb8a..776bc63d823c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -23,8 +23,8 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -254,11 +254,11 @@ private static Object[] truncateNullSuffixesOfDeviceIdSegments(Object[] segments } public void createTable(TableSchema fileSchema, MPPQueryContext context, Metadata metadata) - throws VerifyMetadataException { + throws LoadAnalyzeException { final TableSchema realSchema = metadata.validateTableHeaderSchema(database, fileSchema, context, true, true).orElse(null); if (Objects.isNull(realSchema)) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Failed to validate schema for table {%s, %s}", fileSchema.getTableName(), fileSchema)); @@ -267,7 +267,7 @@ public void createTable(TableSchema fileSchema, MPPQueryContext context, Metadat } private void verifyTableDataTypeAndGenerateIdColumnMapper( - TableSchema fileSchema, TableSchema realSchema) throws VerifyMetadataException { + TableSchema fileSchema, TableSchema realSchema) throws LoadAnalyzeException { final int realIdColumnCount = realSchema.getIdColumns().size(); final Map idColumnMapping = tableIdColumnMapper @@ -282,7 +282,7 @@ private void verifyTableDataTypeAndGenerateIdColumnMapper( if (realIndex != -1) { idColumnMapping.put(idColumnIndex++, realIndex); } else { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Id column %s in TsFile is not found in IoTDB table %s", fileColumn.getName(), realSchema.getTableName())); @@ -291,7 +291,7 @@ private void verifyTableDataTypeAndGenerateIdColumnMapper( final ColumnSchema realColumn = realSchema.getColumn(fileColumn.getName(), fileColumn.getColumnCategory()); if (!fileColumn.getType().equals(realColumn.getType())) { - throw new VerifyMetadataTypeMismatchException( + throw new LoadAnalyzeTypeMismatchException( String.format( "Data type mismatch for column %s in table %s, type in TsFile: %s, type in IoTDB: %s", realColumn.getName(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java index 5262a02af952..fd75d2b06232 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.analyze.load; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; -import org.apache.iotdb.db.exception.VerifyMetadataException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; import org.apache.iotdb.db.exception.load.LoadEmptyFileException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -70,15 +70,21 @@ public class LoadTsFileToTableModelAnalyzer extends LoadTsFileAnalyzer { private final LoadTsFileTableSchemaCache schemaCache; public LoadTsFileToTableModelAnalyzer( - LoadTsFileStatement loadTsFileStatement, Metadata metadata, MPPQueryContext context) { - super(loadTsFileStatement, context); + LoadTsFileStatement loadTsFileStatement, + boolean isGeneratedByPipe, + Metadata metadata, + MPPQueryContext context) { + super(loadTsFileStatement, isGeneratedByPipe, context); this.metadata = metadata; this.schemaCache = new LoadTsFileTableSchemaCache(metadata, context); } public LoadTsFileToTableModelAnalyzer( - LoadTsFile loadTsFileTableStatement, Metadata metadata, MPPQueryContext context) { - super(loadTsFileTableStatement, context); + LoadTsFile loadTsFileTableStatement, + boolean isGeneratedByPipe, + Metadata metadata, + MPPQueryContext context) { + super(loadTsFileTableStatement, isGeneratedByPipe, context); this.metadata = metadata; this.schemaCache = new LoadTsFileTableSchemaCache(metadata, context); } @@ -92,7 +98,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { try { autoCreateDatabaseIfAbsent(database); - } catch (VerifyMetadataException e) { + } catch (LoadAnalyzeException e) { LOGGER.warn("Auto create database failed: {}", database, e); analysis.setFinishQueryAfterAnalyze(true); analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage())); @@ -112,8 +118,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } @Override - protected void analyzeSingleTsFile(final File tsFile) - throws IOException, VerifyMetadataException { + protected void analyzeSingleTsFile(final File tsFile) throws IOException, LoadAnalyzeException { try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { // can be reused when constructing tsfile resource final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator = @@ -183,7 +188,7 @@ protected void analyzeSingleTsFile(final File tsFile) } } - private void autoCreateDatabaseIfAbsent(final String database) throws VerifyMetadataException { + private void autoCreateDatabaseIfAbsent(final String database) throws LoadAnalyzeException { validateDatabaseName(database); if (DataNodeTableCache.getInstance().isDatabaseExist(database)) { return; @@ -196,13 +201,13 @@ private void autoCreateDatabaseIfAbsent(final String database) throws VerifyMeta task.execute(ClusterConfigTaskExecutor.getInstance()); final ConfigTaskResult result = future.get(); if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Auto create database failed: %s, status code: %s", database, result.getStatusCode())); } } catch (final ExecutionException | InterruptedException e) { - throw new VerifyMetadataException("Auto create database failed because: " + e.getMessage()); + throw new LoadAnalyzeException("Auto create database failed because: " + e.getMessage()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java index a34483af9d83..84d684733dcc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java @@ -21,7 +21,8 @@ import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadEmptyFileException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -58,14 +59,14 @@ public class LoadTsFileToTreeModelAnalyzer extends LoadTsFileAnalyzer { private final TreeSchemaAutoCreatorAndVerifier schemaAutoCreatorAndVerifier; public LoadTsFileToTreeModelAnalyzer( - LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) { - super(loadTsFileStatement, context); + LoadTsFileStatement loadTsFileStatement, boolean isGeneratedByPipe, MPPQueryContext context) { + super(loadTsFileStatement, isGeneratedByPipe, context); this.schemaAutoCreatorAndVerifier = new TreeSchemaAutoCreatorAndVerifier(this); } public LoadTsFileToTreeModelAnalyzer( - LoadTsFile loadTsFileTableStatement, MPPQueryContext context) { - super(loadTsFileTableStatement, context); + LoadTsFile loadTsFileTableStatement, boolean isGeneratedByPipe, MPPQueryContext context) { + super(loadTsFileTableStatement, isGeneratedByPipe, context); this.schemaAutoCreatorAndVerifier = new TreeSchemaAutoCreatorAndVerifier(this); } @@ -86,8 +87,8 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { } catch (AuthException e) { setFailAnalysisForAuthException(analysis, e); return analysis; - } catch (VerifyMetadataTypeMismatchException e) { - executeDataTypeConversionOnTypeMismatch(analysis, e); + } catch (LoadAnalyzeException e) { + executeTabletConversion(analysis, e); return analysis; } catch (Exception e) { final String exceptionMessage = @@ -110,7 +111,7 @@ public IAnalysis analyzeFileByFile(IAnalysis analysis) { @Override protected void analyzeSingleTsFile(final File tsFile) - throws IOException, AuthException, VerifyMetadataTypeMismatchException { + throws IOException, AuthException, LoadAnalyzeTypeMismatchException { try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { // can be reused when constructing tsfile resource final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java index ac389b2a6dbf..3348a7f83f35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java @@ -34,8 +34,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.exception.VerifyMetadataException; -import org.apache.iotdb.db.exception.VerifyMetadataTypeMismatchException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeException; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -105,7 +105,7 @@ public void setCurrentModificationsAndTimeIndex( public void autoCreateAndVerify( TsFileSequenceReader reader, Map> device2TimeseriesMetadataList) - throws IOException, AuthException, VerifyMetadataTypeMismatchException { + throws IOException, AuthException, LoadAnalyzeTypeMismatchException { for (final Map.Entry> entry : device2TimeseriesMetadataList.entrySet()) { final IDeviceID device = entry.getKey(); @@ -206,14 +206,14 @@ public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticExcept schemaCache.clearDeviceIsAlignedCacheIfNecessary(); } - public void flush() throws AuthException, VerifyMetadataTypeMismatchException { + public void flush() throws AuthException, LoadAnalyzeTypeMismatchException { doAutoCreateAndVerify(); schemaCache.clearTimeSeries(); } private void doAutoCreateAndVerify() - throws SemanticException, AuthException, VerifyMetadataTypeMismatchException { + throws SemanticException, AuthException, LoadAnalyzeTypeMismatchException { if (schemaCache.getDevice2TimeSeries().isEmpty()) { return; } @@ -236,7 +236,7 @@ private void doAutoCreateAndVerify() } } catch (AuthException e) { throw e; - } catch (VerifyMetadataTypeMismatchException e) { + } catch (LoadAnalyzeTypeMismatchException e) { if (loadTsFileAnalyzer.isConvertOnTypeMismatch()) { // throw exception to convert data type in the upper layer (LoadTsFileAnalyzer) throw e; @@ -256,7 +256,7 @@ private void handleException(Exception e, String statementString) throws Semanti statementString, e.getMessage())); } - private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataException { + private void makeSureNoDuplicatedMeasurementsInDevices() throws LoadAnalyzeException { for (final Map.Entry> entry : schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); @@ -264,7 +264,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataEx for (final MeasurementSchema timeseriesSchema : entry.getValue()) { final String measurement = timeseriesSchema.getMeasurementName(); if (measurement2Schema.containsKey(measurement)) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format("Duplicated measurements %s in device %s.", measurement, device)); } measurement2Schema.put(measurement, timeseriesSchema); @@ -273,7 +273,7 @@ private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataEx } private void autoCreateDatabase() - throws VerifyMetadataException, LoadFileException, IllegalPathException, AuthException { + throws LoadAnalyzeException, LoadFileException, IllegalPathException, AuthException { final int databasePrefixNodesLength = loadTsFileAnalyzer.getDatabaseLevel() + 1; final Set databasesNeededToBeSet = new HashSet<>(); @@ -282,7 +282,7 @@ private void autoCreateDatabase() final String[] devicePrefixNodes = devicePath.getNodes(); if (devicePrefixNodes.length < databasePrefixNodesLength) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Database level %d is longer than device %s.", databasePrefixNodesLength, device)); } @@ -411,7 +411,7 @@ private ISchemaTree autoCreateSchema() throws IllegalPathException { } private void verifySchema(ISchemaTree schemaTree) - throws VerifyMetadataException, IllegalPathException { + throws LoadAnalyzeException, IllegalPathException { for (final Map.Entry> entry : schemaCache.getDevice2TimeSeries().entrySet()) { final IDeviceID device = entry.getKey(); @@ -424,7 +424,7 @@ private void verifySchema(ISchemaTree schemaTree) .collect(Collectors.toList())); if (iotdbDeviceSchemaInfo == null) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Device %s does not exist in IoTDB and can not be created. " + "Please check weather auto-create-schema is enabled.", @@ -435,7 +435,7 @@ private void verifySchema(ISchemaTree schemaTree) final boolean isAlignedInTsFile = schemaCache.getDeviceIsAligned(device); final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned(); if (isAlignedInTsFile != isAlignedInIoTDB) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Device %s in TsFile is %s, but in IoTDB is %s.", device, @@ -450,7 +450,7 @@ private void verifySchema(ISchemaTree schemaTree) final IMeasurementSchema tsFileSchema = tsfileTimeseriesSchemas.get(i); final IMeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i); if (iotdbSchema == null) { - throw new VerifyMetadataException( + throw new LoadAnalyzeException( String.format( "Measurement %s does not exist in IoTDB and can not be created. " + "Please check weather auto-create-schema is enabled.", @@ -459,7 +459,7 @@ private void verifySchema(ISchemaTree schemaTree) // check datatype if (!tsFileSchema.getType().equals(iotdbSchema.getType())) { - throw new VerifyMetadataTypeMismatchException( + throw new LoadAnalyzeTypeMismatchException( String.format( "Measurement %s%s%s datatype not match, TsFile: %s, IoTDB: %s", device, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 4a14c71a3d37..3c41ea2ac2ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -133,6 +133,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -2044,6 +2046,7 @@ public SettableFuture alterPipe(final AlterPipeStatement alter req.setExtractorAttributes(alterPipeStatement.getExtractorAttributes()); req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllExtractorAttributes()); req.setIfExistsCondition(alterPipeStatement.hasIfExistsCondition()); + req.setIsTableModel(alterPipeStatement.isTableModel()); final TSStatus tsStatus = configNodeClient.alterPipe(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); @@ -2073,7 +2076,11 @@ public SettableFuture startPipe(final StartPipeStatement start try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.startPipe(startPipeStatement.getPipeName()); + final TSStatus tsStatus = + configNodeClient.startPipeExtended( + new TStartPipeReq() + .setPipeName(startPipeStatement.getPipeName()) + .setIsTableModel(startPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2106,7 +2113,8 @@ public SettableFuture dropPipe(final DropPipeStatement dropPip configNodeClient.dropPipeExtended( new TDropPipeReq() .setPipeName(dropPipeStatement.getPipeName()) - .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition())); + .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition()) + .setIsTableModel(dropPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2135,7 +2143,12 @@ public SettableFuture stopPipe(final StopPipeStatement stopPip try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.stopPipe(stopPipeStatement.getPipeName()); + + final TSStatus tsStatus = + configNodeClient.stopPipeExtended( + new TStopPipeReq() + .setPipeName(stopPipeStatement.getPipeName()) + .setIsTableModel(stopPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2159,6 +2172,7 @@ public SettableFuture showPipes(final ShowPipesStatement showP if (showPipesStatement.getWhereClause()) { tShowPipeReq.setWhereClause(true); } + tShowPipeReq.setIsTableModel(showPipesStatement.isTableModel()); final List tShowPipeInfoList = configNodeClient.showPipe(tShowPipeReq).getPipeInfoList(); ShowPipeTask.buildTSBlock(tShowPipeInfoList, future); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index 364f15d1f06f..2f5141ff071d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -84,7 +84,7 @@ private static void buildTSBlockForNonDetails( .collect(Collectors.toList()); final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); - InformationSchemaUtils.buildDatabaseTsBlock(canSeenDB, builder, false); + InformationSchemaUtils.buildDatabaseTsBlock(canSeenDB, builder, false, true); for (final Map.Entry entry : storageGroupInfoMap.entrySet()) { final String dbName = entry.getKey(); if (Boolean.FALSE.equals(canSeenDB.test(dbName))) { @@ -124,7 +124,7 @@ private static void buildTSBlockForDetails( .collect(Collectors.toList()); final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); - InformationSchemaUtils.buildDatabaseTsBlock(canSeenDB, builder, true); + InformationSchemaUtils.buildDatabaseTsBlock(canSeenDB, builder, true, true); for (final Map.Entry entry : storageGroupInfoMap.entrySet()) { final String dbName = entry.getKey(); if (!canSeenDB.test(dbName)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index f6d0ebab2cbf..159c6941628e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -37,13 +37,13 @@ public class AlterPipeTask implements IConfigTask { private final AlterPipeStatement alterPipeStatement; - public AlterPipeTask(AlterPipeStatement alterPipeStatement) { + public AlterPipeTask(final AlterPipeStatement alterPipeStatement) { // support now() function applyNowFunctionToExtractorAttributes(alterPipeStatement.getExtractorAttributes()); this.alterPipeStatement = alterPipeStatement; } - public AlterPipeTask(AlterPipe node) { + public AlterPipeTask(final AlterPipe node) { alterPipeStatement = new AlterPipeStatement(StatementType.ALTER_PIPE); alterPipeStatement.setPipeName(node.getPipeName()); alterPipeStatement.setIfExists(node.hasIfExistsCondition()); @@ -57,10 +57,12 @@ public AlterPipeTask(AlterPipe node) { alterPipeStatement.setReplaceAllExtractorAttributes(node.isReplaceAllExtractorAttributes()); alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); + + alterPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.alterPipe(alterPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java index a28ca91f3c65..a10c6042241c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java @@ -32,18 +32,19 @@ public class DropPipeTask implements IConfigTask { private final DropPipeStatement dropPipeStatement; - public DropPipeTask(DropPipeStatement dropPipeStatement) { + public DropPipeTask(final DropPipeStatement dropPipeStatement) { this.dropPipeStatement = dropPipeStatement; } - public DropPipeTask(DropPipe node) { + public DropPipeTask(final DropPipe node) { dropPipeStatement = new DropPipeStatement(StatementType.DROP_PIPE); dropPipeStatement.setPipeName(node.getPipeName()); dropPipeStatement.setIfExists(node.hasIfExistsCondition()); + dropPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.dropPipe(dropPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index 6481d0713ac2..c43a9953b584 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -52,10 +52,11 @@ public ShowPipeTask(final ShowPipesStatement showPipesStatement) { this.showPipesStatement = showPipesStatement; } - public ShowPipeTask(ShowPipes node) { + public ShowPipeTask(final ShowPipes node) { showPipesStatement = new ShowPipesStatement(); showPipesStatement.setPipeName(node.getPipeName()); showPipesStatement.setWhereClause(node.hasWhereClause()); + showPipesStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java index 39ff162a43c7..c118f0a3a58a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java @@ -32,17 +32,18 @@ public class StartPipeTask implements IConfigTask { private final StartPipeStatement startPipeStatement; - public StartPipeTask(StartPipeStatement startPipeStatement) { + public StartPipeTask(final StartPipeStatement startPipeStatement) { this.startPipeStatement = startPipeStatement; } - public StartPipeTask(StartPipe node) { + public StartPipeTask(final StartPipe node) { startPipeStatement = new StartPipeStatement(StatementType.START_PIPE); startPipeStatement.setPipeName(node.getPipeName()); + startPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.startPipe(startPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java index 04806bef42f6..91764c43263d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java @@ -32,17 +32,18 @@ public class StopPipeTask implements IConfigTask { private final StopPipeStatement stopPipeStatement; - public StopPipeTask(StopPipeStatement stopPipeStatement) { + public StopPipeTask(final StopPipeStatement stopPipeStatement) { this.stopPipeStatement = stopPipeStatement; } - public StopPipeTask(StopPipe node) { + public StopPipeTask(final StopPipe node) { stopPipeStatement = new StopPipeStatement(StatementType.STOP_PIPE); stopPipeStatement.setPipeName(node.getPipeName()); + stopPipeStatement.setTableModel(true); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.stopPipe(stopPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 9b45b9b03d15..e21aaa3096b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.cache.CacheClearOptions; @@ -60,6 +61,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.InferenceWindow; import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.TailInferenceWindow; import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType; import org.apache.iotdb.db.queryengine.plan.expression.binary.AdditionExpression; @@ -1912,7 +1914,12 @@ private ResultSetFormat parseAlignBy(IoTDBSqlParser.AlignByClauseContext ctx) { @Override public Statement visitInsertStatement(IoTDBSqlParser.InsertStatementContext ctx) { InsertStatement insertStatement = new InsertStatement(); - insertStatement.setDevice(parsePrefixPath(ctx.prefixPath())); + try { + insertStatement.setDevice( + DataNodeDevicePathCache.getInstance().getPartialPath(ctx.prefixPath().getText())); + } catch (IllegalPathException e) { + throw new SemanticException(e); + } int timeIndex = parseInsertColumnSpec(ctx.insertColumnsSpec(), insertStatement); parseInsertValuesSpec(ctx.insertValuesSpec(), insertStatement, timeIndex); insertStatement.setAligned(ctx.ALIGNED() != null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index d1cf042ef52e..df229a1598c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITableTimeRangeIterator; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.TableDateBinTimeRangeIterator; @@ -83,6 +84,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.relational.LastQueryAggTableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortFullOuterJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortInnerJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.MergeSortSemiJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeAlignedDeviceViewAggregationScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TreeAlignedDeviceViewScanOperator; @@ -143,6 +145,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; @@ -214,6 +217,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD; @@ -628,7 +632,14 @@ public Operator visitInformationSchemaTableScan( return new InformationSchemaTableScanOperator( operatorContext, node.getPlanNodeId(), - getSupplier(node.getQualifiedObjectName().getObjectName(), dataTypes)); + getSupplier( + node.getQualifiedObjectName().getObjectName(), + dataTypes, + context + .getDriverContext() + .getFragmentInstanceContext() + .getSessionInfo() + .getUserName())); } @Override @@ -1412,10 +1423,9 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { Type leftJoinKeyType = context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getLeft()); - checkArgument( - leftJoinKeyType - == context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getRight()), - "Join key type mismatch."); + checkIfJoinKeyTypeMatches( + leftJoinKeyType, + context.getTypeProvider().getTableModelType(node.getCriteria().get(i).getRight())); joinKeyTypes.add(leftJoinKeyType); } @@ -1491,6 +1501,68 @@ private BiFunction buildUpdateLastRowFunction(Type join } } + @Override + public Operator visitSemiJoin(SemiJoinNode node, LocalExecutionPlanContext context) { + List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); + + Operator leftChild = node.getLeftChild().accept(this, context); + Operator rightChild = node.getRightChild().accept(this, context); + + ImmutableMap sourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getSource().getOutputSymbols()); + List sourceOutputSymbols = node.getSource().getOutputSymbols(); + int[] sourceOutputSymbolIdx = new int[node.getSource().getOutputSymbols().size()]; + for (int i = 0; i < sourceOutputSymbolIdx.length; i++) { + Integer index = sourceColumnNamesMap.get(sourceOutputSymbols.get(i)); + checkNotNull(index, "Source of SemiJoinNode doesn't contain sourceOutputSymbol."); + sourceOutputSymbolIdx[i] = index; + } + + ImmutableMap filteringSourceColumnNamesMap = + makeLayoutFromOutputSymbols(node.getRightChild().getOutputSymbols()); + + Integer sourceJoinKeyPosition = sourceColumnNamesMap.get(node.getSourceJoinSymbol()); + checkNotNull(sourceJoinKeyPosition, "Source of SemiJoinNode doesn't contain sourceJoinSymbol."); + + Integer filteringSourceJoinKeyPosition = + filteringSourceColumnNamesMap.get(node.getFilteringSourceJoinSymbol()); + checkNotNull( + filteringSourceJoinKeyPosition, + "FilteringSource of SemiJoinNode doesn't contain filteringSourceJoinSymbol."); + + Type sourceJoinKeyType = + context.getTypeProvider().getTableModelType(node.getSourceJoinSymbol()); + checkIfJoinKeyTypeMatches( + sourceJoinKeyType, + context.getTypeProvider().getTableModelType(node.getFilteringSourceJoinSymbol())); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + MergeSortSemiJoinOperator.class.getSimpleName()); + return new MergeSortSemiJoinOperator( + operatorContext, + leftChild, + sourceJoinKeyPosition, + sourceOutputSymbolIdx, + rightChild, + filteringSourceJoinKeyPosition, + JoinKeyComparatorFactory.getComparator(sourceJoinKeyType, true), + dataTypes); + } + + private void checkIfJoinKeyTypeMatches(Type leftJoinKeyType, Type rightJoinKeyType) { + if (leftJoinKeyType != rightJoinKeyType) { + throw new SemanticException( + "Join key type mismatch. Left join key type: " + + leftJoinKeyType + + ", right join key type: " + + rightJoinKeyType); + } + } + @Override public Operator visitEnforceSingleRow( EnforceSingleRowNode node, LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 1811d7f46ae8..442d59f8be33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -72,6 +72,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -935,6 +936,17 @@ public List visitJoin( return render(node, boxValue, context); } + @Override + public List visitSemiJoin(SemiJoinNode node, GraphContext context) { + List boxValue = new ArrayList<>(); + boxValue.add(String.format("SemiJoin-%s", node.getPlanNodeId().getId())); + boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); + boxValue.add(String.format("SourceJoinSymbol: %s", node.getSourceJoinSymbol())); + boxValue.add( + String.format("FilteringSourceJoinSymbol: %s", node.getFilteringSourceJoinSymbol())); + return render(node, boxValue, context); + } + private String printRegion(TRegionReplicaSet regionReplicaSet) { return String.format( "Partition: %s", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 9a8e5ed31e64..3e86d189e1ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -122,6 +122,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -288,6 +289,7 @@ public enum PlanNodeType { AGGREGATION_TREE_DEVICE_VIEW_SCAN_NODE((short) 1022), TREE_ALIGNED_DEVICE_VIEW_SCAN_NODE((short) 1023), TREE_NONALIGNED_DEVICE_VIEW_SCAN_NODE((short) 1024), + TABLE_SEMI_JOIN_NODE((short) 1025), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), @@ -653,6 +655,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TreeAlignedDeviceViewScanNode.deserialize(buffer); case 1024: return TreeNonAlignedDeviceViewScanNode.deserialize(buffer); + case 1025: + return SemiJoinNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 348104f95f97..d04c57e77fbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -127,6 +127,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; @@ -754,6 +755,10 @@ public R visitJoin( return visitTwoChildProcess(node, context); } + public R visitSemiJoin(SemiJoinNode node, C context) { + return visitTwoChildProcess(node, context); + } + public R visitGroupReference(GroupReference node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 404c88b2e1e8..1798a0ed0b7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; @@ -106,6 +107,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterators.getOnlyElement; import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; @@ -164,6 +166,9 @@ public class ExpressionAnalyzer { private final Map, LabelPrefixedReference> labelDereferences = new LinkedHashMap<>(); + private static final String SUBQUERY_COLUMN_NUM_CHECK = + "Subquery must return only one column for now. Row Type is not supported for now."; + private ExpressionAnalyzer( Metadata metadata, MPPQueryContext context, @@ -415,8 +420,7 @@ protected Type visitDereferenceExpression( return handleResolvedField(node, resolvedField.get(), context); } if (!scope.isColumnReference(qualifiedName)) { - throw new SemanticException( - String.format("Column '%s' cannot be resolved", qualifiedName)); + TableMetadataImpl.throwColumnNotExistsException(qualifiedName); } } @@ -446,7 +450,7 @@ protected Type visitDereferenceExpression( } if (rowFieldType == null) { - throw new SemanticException(String.format("Column '%s' cannot be resolved", qualifiedName)); + TableMetadataImpl.throwColumnNotExistsException(qualifiedName); } return setExpressionType(node, rowFieldType); @@ -928,6 +932,10 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { @Override protected Type visitInPredicate(InPredicate node, StackableAstVisitorContext context) { Expression value = node.getValue(); + // Attention: remove this check after supporting RowType + if (value instanceof Row) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } Expression valueList = node.getValueList(); // When an IN-predicate containing a subquery: `x IN (SELECT ...)` is planned, both `value` @@ -996,21 +1004,15 @@ private Type analyzePredicateWithSubquery( Type declaredValueType, SubqueryExpression subquery, StackableAstVisitorContext context) { + // For now, we only support one column in subqueries, we have checked this before. Type valueRowType = declaredValueType; - if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { + /*if (!(declaredValueType instanceof RowType) && !(declaredValueType instanceof UnknownType)) { valueRowType = RowType.anonymous(ImmutableList.of(declaredValueType)); - } + }*/ Type subqueryType = analyzeSubquery(subquery, context); setExpressionType(subquery, subqueryType); - if (subqueryType.equals(valueRowType)) { - throw new SemanticException( - String.format( - "Value expression and result of subquery must be of the same type: %s vs %s", - valueRowType, subqueryType)); - } - Optional valueCoercion = Optional.empty(); // if (!valueRowType.equals(commonType.get())) { // valueCoercion = commonType; @@ -1047,8 +1049,17 @@ private Type analyzeSubquery( } } + List fieldList = fields.build(); + + // Attention: remove this check after supporting RowType + if (fieldList.size() != 1 || fieldList.get(0).getType() instanceof RowType) { + throw new SemanticException(SUBQUERY_COLUMN_NUM_CHECK); + } + sourceFields.addAll(queryScope.getRelationType().getVisibleFields()); - return RowType.from(fields.build()); + // return RowType.from(fields.build()); + // For now, we only support one column in subqueries, we have checked this before. + return getOnlyElement(fields.build().stream().iterator()).getType(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java index dfcc21e4e9c4..0e7588a59f49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Scope.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; +import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.collect.ImmutableMap; @@ -228,7 +230,11 @@ public boolean isLocalScope(Scope other) { public ResolvedField resolveField(Expression expression, QualifiedName name) { return tryResolveField(expression, name) .orElseThrow( - () -> new SemanticException(String.format("Column '%s' cannot be resolved", name))); + () -> + new SemanticException( + new IoTDBException( + String.format("Column '%s' cannot be resolved", name), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()))); } public Optional tryResolveField(Expression expression) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 2f11adf246ed..93ec5bef3a50 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.planner.PlannerContext; import org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware; @@ -462,9 +463,7 @@ protected Scope visitDeleteDevice(final DeleteDevice node, final Optional final TsTable table = DataNodeTableCache.getInstance().getTable(node.getDatabase(), node.getTableName()); if (Objects.isNull(table)) { - throw new SemanticException( - String.format( - "Table '%s.%s' does not exist.", node.getDatabase(), node.getTableName())); + TableMetadataImpl.throwTableNotExistsException(node.getDatabase(), node.getTableName()); } node.parseModEntries(table); analyzeTraverseDevice(node, context, node.getWhere().isPresent()); @@ -548,6 +547,12 @@ protected Scope visitDelete(Delete node, Optional scope) { @Override protected Scope visitPipeEnriched(PipeEnriched node, Optional scope) { + // The LoadTsFile statement is a special case, it needs isGeneratedByPipe information + // in the analyzer to execute the tsfile-tablet conversion in some cases. + if (node.getInnerStatement() instanceof LoadTsFile) { + ((LoadTsFile) node.getInnerStatement()).markIsGeneratedByPipe(); + } + Scope ret = node.getInnerStatement().accept(this, scope); createAndAssignScope(node, scope); analysis.setScope(node, ret); @@ -587,10 +592,12 @@ private LoadTsFileAnalyzer getAnalyzer(LoadTsFile loadTsFile) { } loadTsFile.setDatabase(queryContext.getDatabaseName().get()); } - return new LoadTsFileToTableModelAnalyzer(loadTsFile, metadata, queryContext); + return new LoadTsFileToTableModelAnalyzer( + loadTsFile, loadTsFile.isGeneratedByPipe(), metadata, queryContext); } else { // Load to tree-model - return new LoadTsFileToTreeModelAnalyzer(loadTsFile, queryContext); + return new LoadTsFileToTreeModelAnalyzer( + loadTsFile, loadTsFile.isGeneratedByPipe(), queryContext); } } @@ -1757,7 +1764,8 @@ protected Scope visitTable(Table table, Optional scope) { Optional tableSchema = metadata.getTableSchema(sessionContext, name); // This can only be a table if (!tableSchema.isPresent()) { - throw new SemanticException(String.format("Table '%s' does not exist", name)); + TableMetadataImpl.throwTableNotExistsException( + name.getDatabaseName(), name.getObjectName()); } analysis.addEmptyColumnReferencesForTable(accessControl, sessionContext.getIdentity(), name); @@ -2953,8 +2961,7 @@ private TranslationMap analyzeTraverseDevice( } if (!metadata.tableExists(new QualifiedObjectName(database, tableName))) { - throw new SemanticException( - String.format("Table '%s.%s' does not exist.", database, tableName)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } node.setColumnHeaderList(); @@ -2964,7 +2971,7 @@ private TranslationMap analyzeTraverseDevice( final Optional tableSchema = metadata.getTableSchema(sessionContext, name); // This can only be a table if (!tableSchema.isPresent()) { - throw new SemanticException(String.format("Table '%s' does not exist", name)); + TableMetadataImpl.throwTableNotExistsException(database, tableName); } final TableSchema originalSchema = tableSchema.get(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 23c74b525e5a..1234c1cf1458 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -112,7 +113,8 @@ Optional validateTableHeaderSchema( final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn); + final boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException; /** * This method is used for table device validation and should be invoked after column validation. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 7bee39bdfb1d..fb437d3d5966 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; @@ -27,6 +29,7 @@ import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction; import org.apache.iotdb.commons.udf.utils.TableUDFUtils; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -49,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.utils.constant.SqlConstant; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; import org.apache.iotdb.udf.api.customizer.analysis.ScalarFunctionAnalysis; import org.apache.iotdb.udf.api.customizer.parameter.FunctionArguments; @@ -734,7 +738,8 @@ public Optional validateTableHeaderSchema( TableSchema tableSchema, MPPQueryContext context, boolean allowCreateTable, - boolean isStrictIdColumn) { + boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException { return TableHeaderSchemaValidator.getInstance() .validateTableHeaderSchema( database, tableSchema, context, allowCreateTable, isStrictIdColumn); @@ -898,4 +903,15 @@ public static boolean isTwoTypeCalculable(List argumentTypes) { } return isArithmeticType(left) && isArithmeticType(right); } + + public static void throwTableNotExistsException(final String database, final String tableName) { + throw new SemanticException(new TableNotExistsException(database, tableName)); + } + + public static void throwColumnNotExistsException(final Object columnName) { + throw new SemanticException( + new IoTDBException( + String.format("Column '%s' cannot be resolved.", columnName), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode())); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 294a4d78a7b9..b26563c521be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -27,7 +27,6 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; @@ -36,6 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.schema.ConvertSchemaPredicateToFilterVisitor; import org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractTraverseDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -172,7 +172,7 @@ public List fetchDeviceSchemaForDataQuery( final ShowDevice statement = new ShowDevice(database, table); final TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); if (tableInstance == null) { - throw new SemanticException(String.format("Table '%s.%s' does not exist", database, table)); + TableMetadataImpl.throwTableNotExistsException(database, table); } if (parseFilter4TraverseDevice( @@ -261,7 +261,8 @@ public boolean parseFilter4TraverseDevice( check, attributeColumns, fetchPaths, - isDirectDeviceQuery)) { + isDirectDeviceQuery, + queryContext)) { idSingleMatchPredicateNotInCache.add(index); } } @@ -312,8 +313,9 @@ private boolean tryGetDeviceInCache( final Predicate check, final List attributeColumns, final List fetchPaths, - final boolean isDirectDeviceQuery) { - String[] idValues = new String[tableInstance.getIdNums()]; + final boolean isDirectDeviceQuery, + final MPPQueryContext queryContext) { + final String[] idValues = new String[tableInstance.getIdNums()]; for (final List schemaFilters : idFilters.values()) { final IdFilter idFilter = (IdFilter) schemaFilters.get(0); final SchemaFilter childFilter = idFilter.getChild(); @@ -347,6 +349,8 @@ private boolean tryGetDeviceInCache( // because now we do not support combining memory source and other sources if (isDirectDeviceQuery) { fetchPaths.add(deviceID); + } else { + queryContext.reserveMemoryForFrontEnd(deviceEntry.ramBytesUsed()); } } return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 05cb52e84f61..41932c8944d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.ColumnCreationFailException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -37,6 +38,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -86,7 +88,8 @@ public Optional validateTableHeaderSchema( final TableSchema tableSchema, final MPPQueryContext context, final boolean allowCreateTable, - final boolean isStrictIdColumn) { + final boolean isStrictIdColumn) + throws LoadAnalyzeTableColumnDisorderException { InformationSchemaUtils.checkDBNameInWrite(database); // The schema cache R/W and fetch operation must be locked together thus the cache clean @@ -121,7 +124,7 @@ public Optional validateTableHeaderSchema( "auto create table succeed, but cannot get table schema in current node's DataNodeTableCache, may be caused by concurrently auto creating table"); } } else { - throw new SemanticException("Table " + tableSchema.getTableName() + " does not exist"); + TableMetadataImpl.throwTableNotExistsException(database, tableSchema.getTableName()); } } else { // If table with this name already exists and isStrictIdColumn is true, make sure the existing @@ -136,7 +139,7 @@ public Optional validateTableHeaderSchema( final String idName = realIdColumns.get(indexReal).getColumnName(); final int indexIncoming = tableSchema.getIndexAmongIdColumns(idName); if (indexIncoming != indexReal) { - throw new SemanticException( + throw new LoadAnalyzeTableColumnDisorderException( String.format( "Can not create table because incoming table has no less id columns than existing table, " + "and the existing id columns are not the prefix of the incoming id columns. " @@ -151,7 +154,7 @@ public Optional validateTableHeaderSchema( final String idName = incomingIdColumns.get(indexIncoming).getName(); final int indexReal = table.getIdColumnOrdinal(idName); if (indexReal != indexIncoming) { - throw new SemanticException( + throw new LoadAnalyzeTableColumnDisorderException( String.format( "Can not create table because existing table has more id columns than incoming table, " + "and the incoming id columns are not the prefix of the existing id columns. " diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java index cdf01015f40e..1a19f40132f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrTypeAnalyzer.java @@ -54,6 +54,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -63,6 +64,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.DateType; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; import org.apache.tsfile.read.common.type.Type; @@ -75,6 +77,7 @@ import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; @@ -416,6 +419,14 @@ protected Type visitInPredicate(InPredicate node, Context context) { return setExpressionType(node, BOOLEAN); } + @Override + protected Type visitRow(Row node, Context context) { + List types = + node.getItems().stream().map(child -> process(child, context)).collect(toImmutableList()); + + return setExpressionType(node, RowType.anonymous(types)); + } + @Override protected Type visitLikePredicate(LikePredicate node, Context context) { process(node.getValue(), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 33c2207fc43a..747288a48c87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -256,7 +256,7 @@ public RelationPlan plan(QuerySpecification node) { } List orderBy = analysis.getOrderByExpressions(node); - if (!orderBy.isEmpty()) { + if (!orderBy.isEmpty() || node.getSelect().isDistinct()) { builder = builder.appendProjections( Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java index 184fc0739436..ebc32b6a247e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryPlanner.java @@ -45,7 +45,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import com.google.common.collect.ImmutableList; @@ -72,7 +71,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression.Quantifier.ALL; -import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; class SubqueryPlanner { @@ -117,32 +115,39 @@ public PlanBuilder handleSubqueries( public PlanBuilder handleSubqueries( PlanBuilder builder, Expression expression, Analysis.SubqueryAnalysis subqueries) { - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getInPredicatesSubqueries()))) { - builder = planInPredicate(builder, cluster, subqueries); - } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getSubqueries()))) { - builder = planScalarSubquery(builder, cluster); + List inPredicates = subqueries.getInPredicatesSubqueries(); + if (!inPredicates.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, inPredicates))) { + builder = planInPredicate(builder, cluster, subqueries); + } } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries(builder, expression, subqueries.getExistsSubqueries()))) { - builder = planExists(builder, cluster); + + List scalarSubqueries = subqueries.getSubqueries(); + if (!scalarSubqueries.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, scalarSubqueries))) { + builder = planScalarSubquery(builder, cluster); + } } - for (Cluster cluster : - cluster( - builder.getScope(), - selectSubqueries( - builder, expression, subqueries.getQuantifiedComparisonSubqueries()))) { - builder = planQuantifiedComparison(builder, cluster, subqueries); + + List existsPredicates = subqueries.getExistsSubqueries(); + if (!existsPredicates.isEmpty()) { + for (Cluster cluster : + cluster(builder.getScope(), selectSubqueries(builder, expression, existsPredicates))) { + builder = planExists(builder, cluster); + } } + List quantifiedComparisons = + subqueries.getQuantifiedComparisonSubqueries(); + if (!quantifiedComparisons.isEmpty()) { + for (Cluster cluster : + cluster( + builder.getScope(), selectSubqueries(builder, expression, quantifiedComparisons))) { + builder = planQuantifiedComparison(builder, cluster, subqueries); + } + } return builder; } @@ -259,6 +264,11 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldMappings = relationPlan.getFieldMappings(); Symbol column; + // Attention: remove this check after supporting RowType + checkArgument( + descriptor.getVisibleFieldCount() <= 1, + "For now, only single column subqueries are supported"); + /* if (descriptor.getVisibleFieldCount() > 1) { column = symbolAllocator.newSymbol("row", type); @@ -274,8 +284,9 @@ private PlanBuilder planScalarSubquery(PlanBuilder subPlan, Cluster fieldsList = fields.build(); + // Attention: remove this check after supporting RowType + checkArgument(fieldsList.size() == 1, "For now, only single column subqueries are supported."); + /*subqueryPlan = + subqueryPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + relationPlan.getRoot(), + Assignments.of(column, new Cast(new Row(fields.build()), toSqlType(type)))));*/ + subqueryPlan = subqueryPlan.withNewRoot( new ProjectNode( idAllocator.genPlanNodeId(), relationPlan.getRoot(), - Assignments.of( - column, - new Cast( - new Row(fields.build()), - new GenericDataType( - new Identifier(type.toString()), ImmutableList.of()))))); + Assignments.of(column, fieldsList.get(0)))); return coerceIfNecessary(subqueryPlan, column, subquery, coercion); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 53d6516e2e3f..d1fcd56799fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -57,6 +57,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; @@ -464,6 +465,20 @@ public List visitJoin(JoinNode node, PlanContext context) { } @Override + public List visitSemiJoin(SemiJoinNode node, PlanContext context) { + List leftChildrenNodes = node.getLeftChild().accept(this, context); + List rightChildrenNodes = node.getRightChild().accept(this, context); + checkArgument( + leftChildrenNodes.size() == 1, + "The size of left children node of SemiJoinNode should be 1"); + checkArgument( + rightChildrenNodes.size() == 1, + "The size of right children node of SemiJoinNode should be 1"); + node.setLeftChild(leftChildrenNodes.get(0)); + node.setRightChild(rightChildrenNodes.get(0)); + return Collections.singletonList(node); + } + public List visitDeviceTableScan( final DeviceTableScanNode node, final PlanContext context) { final Map tableScanNodeMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java new file mode 100644 index 000000000000..11dcaefba17a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyColumns.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.Sets.intersection; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts the outputs of ApplyNode's input and subquery based on which ApplyNode's + * output symbols are referenced. + * + *

A symbol from input source can be pruned, when - it is not a referenced output symbol - it is + * not a correlation symbol - it is not referenced in subqueryAssignments + * + *

A symbol from subquery source can be pruned, when it is not referenced in subqueryAssignments. + * + *

A subquery assignment can be removed, when its key is not a referenced output symbol. + * + *

Note: this rule does not remove any symbols from the correlation list. This is responsibility + * of PruneApplyCorrelation rule. + * + *

Transforms: + * + * 

+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *              r2 -> b in s2,
+ *          - Input (a, b, corr)
+ *          - Subquery (s1, s2)
+ *
+ * + * Into: + * + * 
+ * - Project (i1, r1)
+ *      - Apply
+ *          correlation: [corr]
+ *          assignments:
+ *              r1 -> a in s1,
+ *          - Project (a, corr)
+ *              - Input (a, b, corr)
+ *          - Project (s1)
+ *              - Subquery (s1, s2)
+ *
+ */ +public class PruneApplyColumns extends ProjectOffPushDownRule { + public PruneApplyColumns() { + super(applyNode()); + } + + @Override + protected Optional pushDownProjectOff( + Context context, ApplyNode applyNode, Set referencedOutputs) { + // remove unused apply node + if (intersection(applyNode.getSubqueryAssignments().keySet(), referencedOutputs).isEmpty()) { + return Optional.of(applyNode.getInput()); + } + + // extract referenced assignments + ImmutableSet.Builder requiredAssignmentsSymbols = ImmutableSet.builder(); + ImmutableMap.Builder newSubqueryAssignments = + ImmutableMap.builder(); + for (Map.Entry entry : + applyNode.getSubqueryAssignments().entrySet()) { + if (referencedOutputs.contains(entry.getKey())) { + requiredAssignmentsSymbols.addAll(entry.getValue().inputs()); + newSubqueryAssignments.put(entry); + } + } + + // prune subquery symbols + Optional newSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), requiredAssignmentsSymbols.build()); + + // prune input symbols + Set requiredInputSymbols = + ImmutableSet.builder() + .addAll(referencedOutputs) + .addAll(applyNode.getCorrelation()) + .addAll(requiredAssignmentsSymbols.build()) + .build(); + + Optional newInput = + restrictOutputs(context.getIdAllocator(), applyNode.getInput(), requiredInputSymbols); + + boolean pruned = + newSubquery.isPresent() + || newInput.isPresent() + || newSubqueryAssignments.buildOrThrow().size() + < applyNode.getSubqueryAssignments().size(); + + if (pruned) { + return Optional.of( + new ApplyNode( + applyNode.getPlanNodeId(), + newInput.orElse(applyNode.getInput()), + newSubquery.orElse(applyNode.getSubquery()), + newSubqueryAssignments.buildOrThrow(), + applyNode.getCorrelation(), + applyNode.getOriginSubquery())); + } + + return Optional.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java new file mode 100644 index 000000000000..942c5ba6e639 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplyCorrelation.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule updates ApplyNode's correlation list. A symbol can be removed from the correlation list + * if it is not referenced by the subquery node. Note: This rule does not restrict ApplyNode's + * children outputs. It requires additional information about context (symbols required by the outer + * plan) and is done in PruneApplyColumns rule. + */ +public class PruneApplyCorrelation implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subquerySymbols = extractUnique(applyNode.getSubquery(), context.getLookup()); + List newCorrelation = + applyNode.getCorrelation().stream() + .filter(subquerySymbols::contains) + .collect(toImmutableList()); + + if (newCorrelation.size() < applyNode.getCorrelation().size()) { + return Result.ofPlanNode( + new ApplyNode( + applyNode.getPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + applyNode.getSubqueryAssignments(), + newCorrelation, + applyNode.getOriginSubquery())); + } + + return Result.empty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java new file mode 100644 index 000000000000..746866102d2c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneApplySourceColumns.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +/** + * This rule restricts outputs of ApplyNode's subquery to include only the symbols needed for + * subqueryAssignments. Symbols from the subquery are not produced at ApplyNode's output. They are + * only used for the assignments. Transforms: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Into: + * + * 
+ * - Apply
+ *      correlation: [corr_symbol]
+ *      assignments:
+ *          result_1 -> a in subquery_symbol_1,
+ *          result_2 -> b > ALL subquery_symbol_2
+ *    - Input (a, b, corr_symbol)
+ *    - Project
+ *          subquery_symbol_1 -> subquery_symbol_1
+ *          subquery_symbol_2 -> subquery_symbol_2
+ *        - Subquery (subquery_symbol_1, subquery_symbol_2, subquery_symbol_3)
+ *
+ * + * Note: ApplyNode's input symbols are produced on ApplyNode's output. They cannot be pruned without + * outer context. + */ +public class PruneApplySourceColumns implements Rule { + private static final Pattern PATTERN = applyNode(); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + Set subqueryAssignmentsSymbols = + applyNode.getSubqueryAssignments().values().stream() + .flatMap(expression -> expression.inputs().stream()) + .collect(toImmutableSet()); + + Optional prunedSubquery = + restrictOutputs( + context.getIdAllocator(), applyNode.getSubquery(), subqueryAssignmentsSymbols); + return prunedSubquery + .map( + subquery -> applyNode.replaceChildren(ImmutableList.of(applyNode.getInput(), subquery))) + .map(Result::ofPlanNode) + .orElse(Result.empty()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java new file mode 100644 index 000000000000..fcecc2c3b821 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarApplyNodes.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; + +public class RemoveUnreferencedScalarApplyNodes implements Rule { + private static final Pattern PATTERN = + applyNode().matching(applyNode -> applyNode.getSubqueryAssignments().isEmpty()); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + return Result.ofPlanNode(applyNode.getInput()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java new file mode 100644 index 000000000000..badc7dff3c49 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/RemoveUnreferencedScalarSubqueries.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CorrelatedJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.RIGHT; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.CorrelatedJoin.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.correlatedJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isAtLeastScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.isScalar; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class RemoveUnreferencedScalarSubqueries implements Rule { + private static final Pattern PATTERN = + correlatedJoin().with(filter().equalTo(TRUE_LITERAL)); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context) { + PlanNode input = correlatedJoinNode.getInput(); + PlanNode subquery = correlatedJoinNode.getSubquery(); + + if (isUnreferencedScalar(input, context.getLookup()) + && correlatedJoinNode.getCorrelation().isEmpty()) { + if (correlatedJoinNode.getJoinType() == INNER + || correlatedJoinNode.getJoinType() == RIGHT + || isAtLeastScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(subquery); + } + } + + if (isUnreferencedScalar(subquery, context.getLookup())) { + return Result.ofPlanNode(input); + } + + return Result.empty(); + } + + private boolean isUnreferencedScalar(PlanNode planNode, Lookup lookup) { + return planNode.getOutputSymbols().isEmpty() && isScalar(planNode, lookup); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java new file mode 100644 index 000000000000..67f814af6616 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformFilteringSemiJoinToInnerJoin.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Assignments; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ExpressionSymbolInliner.inlineSymbols; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.and; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.extractConjuncts; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleAggregation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode.JoinType.INNER; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.filter; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.semiJoin; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.source; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Capture.newCapture; + +/** + * Rewrite filtering semi-join to inner join. + * + *

Transforms: + * + * 

+ * - Filter (semiJoinSymbol AND predicate)
+ *    - SemiJoin (semiJoinSymbol <- (a IN b))
+ *        source: plan A producing symbol a
+ *        filtering source: plan B producing symbol b
+ *
+ * + *

Into: + * + * 

+ * - Project (semiJoinSymbol <- TRUE)
+ *    - Join INNER on (a = b), joinFilter (predicate with semiJoinSymbol replaced with TRUE)
+ *       - source
+ *       - Aggregation distinct(b)
+ *          - filtering source
+ *
+ */ +public class TransformFilteringSemiJoinToInnerJoin implements Rule { + private static final Capture SEMI_JOIN = newCapture(); + + private static final Pattern PATTERN = + filter().with(source().matching(semiJoin().capturedAs(SEMI_JOIN))); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(FilterNode filterNode, Captures captures, Context context) { + SemiJoinNode semiJoin = captures.get(SEMI_JOIN); + + Symbol semiJoinSymbol = semiJoin.getSemiJoinOutput(); + Predicate isSemiJoinSymbol = + expression -> expression.equals(semiJoinSymbol.toSymbolReference()); + + List conjuncts = extractConjuncts(filterNode.getPredicate()); + if (conjuncts.stream().noneMatch(isSemiJoinSymbol)) { + return Result.empty(); + } + Expression filteredPredicate = + and(conjuncts.stream().filter(isSemiJoinSymbol.negate()).collect(toImmutableList())); + + Expression simplifiedPredicate = + inlineSymbols( + symbol -> { + if (symbol.equals(semiJoinSymbol)) { + return TRUE_LITERAL; + } + return symbol.toSymbolReference(); + }, + filteredPredicate); + + Optional joinFilter = + simplifiedPredicate.equals(TRUE_LITERAL) + ? Optional.empty() + : Optional.of(simplifiedPredicate); + + PlanNode filteringSourceDistinct = + singleAggregation( + context.getIdAllocator().genPlanNodeId(), + semiJoin.getFilteringSource(), + ImmutableMap.of(), + singleGroupingSet(ImmutableList.of(semiJoin.getFilteringSourceJoinSymbol()))); + + JoinNode innerJoin = + new JoinNode( + semiJoin.getPlanNodeId(), + INNER, + semiJoin.getSource(), + filteringSourceDistinct, + ImmutableList.of( + new JoinNode.EquiJoinClause( + semiJoin.getSourceJoinSymbol(), semiJoin.getFilteringSourceJoinSymbol())), + semiJoin.getSource().getOutputSymbols(), + ImmutableList.of(), + joinFilter, + Optional.empty()); + + ProjectNode project = + new ProjectNode( + context.getIdAllocator().genPlanNodeId(), + innerJoin, + Assignments.builder() + .putIdentities(innerJoin.getOutputSymbols()) + .put(semiJoinSymbol, TRUE_LITERAL) + .build()); + + return Result.ofPlanNode(project); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java new file mode 100644 index 000000000000..80fe79c7e5f4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/TransformUncorrelatedInPredicateSubqueryToSemiJoin.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Rule; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.Apply.correlation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.applyNode; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Pattern.empty; + +/** + * This optimizers looks for InPredicate expressions in ApplyNodes and replaces the nodes with + * SemiJoin nodes. + * + *

Plan before optimizer: + * + * 

+ * Filter(a IN b):
+ *   Apply
+ *     - correlation: []  // empty
+ *     - input: some plan A producing symbol a
+ *     - subquery: some plan B producing symbol b
+ *
+ * + *

Plan after optimizer: + * + * 

+ * Filter(semijoinresult):
+ *   SemiJoin
+ *     - source: plan A
+ *     - filteringSource: B
+ *     - sourceJoinSymbol: symbol a
+ *     - filteringSourceJoinSymbol: symbol b
+ *     - semiJoinOutput: semijoinresult
+ *
+ */ +public class TransformUncorrelatedInPredicateSubqueryToSemiJoin implements Rule { + private static final Pattern PATTERN = applyNode().with(empty(correlation())); + + @Override + public Pattern getPattern() { + return PATTERN; + } + + @Override + public Result apply(ApplyNode applyNode, Captures captures, Context context) { + if (applyNode.getSubqueryAssignments().size() != 1) { + return Result.empty(); + } + + ApplyNode.SetExpression expression = + getOnlyElement(applyNode.getSubqueryAssignments().values()); + if (!(expression instanceof ApplyNode.In)) { + return Result.empty(); + } + + ApplyNode.In inPredicate = (ApplyNode.In) expression; + + Symbol semiJoinSymbol = getOnlyElement(applyNode.getSubqueryAssignments().keySet()); + + SemiJoinNode replacement = + new SemiJoinNode( + context.getIdAllocator().genPlanNodeId(), + applyNode.getInput(), + applyNode.getSubquery(), + inPredicate.getValue(), + inPredicate.getReference(), + semiJoinSymbol); + + return Result.ofPlanNode(replacement); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index f0056e024971..604e2b06a3a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -143,10 +143,9 @@ public static Pattern project() { return typeOf(SampleNode.class); }*/ - // public static Pattern semiJoin() - // { - // return typeOf(SemiJoinNode.class); - // } + public static Pattern semiJoin() { + return typeOf(SemiJoinNode.class); + } public static Pattern gapFill() { return typeOf(GapFillNode.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java new file mode 100644 index 000000000000..bb8b85752d59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SemiJoinNode.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TwoChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.ImmutableList; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class SemiJoinNode extends TwoChildProcessNode { + private final Symbol sourceJoinSymbol; + private final Symbol filteringSourceJoinSymbol; + private final Symbol semiJoinOutput; + + public SemiJoinNode( + PlanNodeId id, + PlanNode source, + PlanNode filteringSource, + Symbol sourceJoinSymbol, + Symbol filteringSourceJoinSymbol, + Symbol semiJoinOutput) { + super(id, source, filteringSource); + this.sourceJoinSymbol = requireNonNull(sourceJoinSymbol, "sourceJoinSymbol is null"); + this.filteringSourceJoinSymbol = + requireNonNull(filteringSourceJoinSymbol, "filteringSourceJoinSymbol is null"); + this.semiJoinOutput = requireNonNull(semiJoinOutput, "semiJoinOutput is null"); + + if (source != null) { + checkArgument( + source.getOutputSymbols().contains(sourceJoinSymbol), + "Source does not contain join symbol"); + } + + if (filteringSource != null) { + checkArgument( + filteringSource.getOutputSymbols().contains(filteringSourceJoinSymbol), + "Filtering source does not contain filtering join symbol"); + } + } + + public PlanNode getSource() { + return leftChild; + } + + public PlanNode getFilteringSource() { + return rightChild; + } + + public Symbol getSourceJoinSymbol() { + return sourceJoinSymbol; + } + + public Symbol getFilteringSourceJoinSymbol() { + return filteringSourceJoinSymbol; + } + + public Symbol getSemiJoinOutput() { + return semiJoinOutput; + } + + @Override + public List getOutputSymbols() { + return ImmutableList.builder() + .addAll(leftChild.getOutputSymbols()) + .add(semiJoinOutput) + .build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitSemiJoin(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 2, "expected newChildren to contain 2 nodes"); + return new SemiJoinNode( + getPlanNodeId(), + newChildren.get(0), + newChildren.get(1), + sourceJoinSymbol, + filteringSourceJoinSymbol, + semiJoinOutput); + } + + @Override + public PlanNode clone() { + // clone without children + return new SemiJoinNode( + getPlanNodeId(), null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } + + @Override + public List getOutputColumnNames() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || !this.getClass().equals(obj.getClass())) { + return false; + } + + if (!super.equals(obj)) { + return false; + } + + SemiJoinNode other = (SemiJoinNode) obj; + + return Objects.equals(this.sourceJoinSymbol, other.sourceJoinSymbol) + && Objects.equals(this.filteringSourceJoinSymbol, other.filteringSourceJoinSymbol) + && Objects.equals(this.semiJoinOutput, other.semiJoinOutput); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(byteBuffer); + + Symbol.serialize(sourceJoinSymbol, byteBuffer); + Symbol.serialize(filteringSourceJoinSymbol, byteBuffer); + Symbol.serialize(semiJoinOutput, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_SEMI_JOIN_NODE.serialize(stream); + + Symbol.serialize(sourceJoinSymbol, stream); + Symbol.serialize(filteringSourceJoinSymbol, stream); + Symbol.serialize(semiJoinOutput, stream); + } + + public static SemiJoinNode deserialize(ByteBuffer byteBuffer) { + Symbol sourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol filteringSourceJoinSymbol = Symbol.deserialize(byteBuffer); + Symbol semiJoinOutput = Symbol.deserialize(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new SemiJoinNode( + planNodeId, null, null, sourceJoinSymbol, filteringSourceJoinSymbol, semiJoinOutput); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index 1d21d91e4292..b91c9eecaf42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@ -24,18 +24,25 @@ import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.schema.table.InformationSchema; import org.apache.iotdb.confignode.rpc.thrift.TGetDataNodeLocationsResp; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; +import org.apache.iotdb.db.queryengine.common.DataNodeEndPoints; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; +import java.util.Collections; import java.util.List; import static org.apache.iotdb.rpc.TSStatusCode.QUERY_PROCESS_ERROR; public class DataNodeLocationSupplierFactory { + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private DataNodeLocationSupplierFactory() {} public static DataNodeLocationSupplier getSupplier() { @@ -82,6 +89,10 @@ private static InformationSchemaTableDataNodeLocationSupplier getInstance() { public List getDataNodeLocations(final String tableName) { if (tableName.equals(InformationSchema.QUERIES)) { return getReadableDataNodeLocations(); + } else if (tableName.equals(InformationSchema.DATABASES) + || tableName.equals(InformationSchema.TABLES) + || tableName.equals(InformationSchema.COLUMNS)) { + return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); } else { throw new UnsupportedOperationException("Unknown table: " + tableName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index d52c23cad31c..62bdd1bf1b9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -32,6 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.MergeLimits; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneAggregationSourceColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplyCorrelation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneApplySourceColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneCorrelatedJoinCorrelation; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneDistinctAggregation; @@ -54,7 +57,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantEnforceSingleRowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveRedundantIdentityProjections; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveTrivialFilters; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveUnreferencedScalarSubqueries; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.SimplifyExpressions; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedInPredicateSubqueryToSemiJoin; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.TransformUncorrelatedSubqueryToJoin; import com.google.common.collect.ImmutableList; @@ -78,6 +83,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { // TODO After ValuesNode introduced // new RemoveEmptyGlobalAggregation(), new PruneAggregationSourceColumns(), + new PruneApplyColumns(), + new PruneApplyCorrelation(), + new PruneApplySourceColumns(), new PruneCorrelatedJoinColumns(), new PruneCorrelatedJoinCorrelation(), new PruneEnforceSingleRowColumns(), @@ -204,13 +212,19 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { plannerContext, ruleStats, ImmutableSet.of( - new RemoveRedundantEnforceSingleRowNode(), - new TransformUncorrelatedSubqueryToJoin())), + new RemoveRedundantEnforceSingleRowNode(), new RemoveUnreferencedScalarSubqueries(), + new TransformUncorrelatedSubqueryToJoin(), + new TransformUncorrelatedInPredicateSubqueryToSemiJoin())), new CheckSubqueryNodesAreRewritten(), new IterativeOptimizer( plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())), simplifyOptimizer, new PushPredicateIntoTableScan(), + // Currently, Distinct is not supported, so we cant use this rule for now. + // new IterativeOptimizer( + // plannerContext, + // ruleStats, + // ImmutableSet.of(new TransformFilteringSemiJoinToInnerJoin())), // redo columnPrune and inlineProjections after pushPredicateIntoTableScan columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 79da48a8cce6..7cf45ff3cbd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -51,6 +51,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -61,6 +62,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; @@ -73,6 +75,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -87,6 +90,7 @@ import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.TABLE_TYPE; import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_LAST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor.extractUnique; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.DeterminismEvaluator.isDeterministic; @@ -102,6 +106,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.processInnerJoin; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.QueryCardinalityUtil.extractCardinality; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; /** * Optimization phase: Logical plan planning. @@ -751,6 +756,205 @@ private void appendSortNodeForMergeSortJoin(JoinNode joinNode) { joinNode.setRightChild(rightSortNode); } + @Override + public PlanNode visitSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + if (!extractConjuncts(inheritedPredicate) + .contains(node.getSemiJoinOutput().toSymbolReference())) { + return visitNonFilteringSemiJoin(node, context); + } + return visitFilteringSemiJoin(node, context); + } + + private PlanNode visitNonFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + + List sourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // TODO: see if there are predicates that can be inferred from the semi join output + PlanNode rewrittenFilteringSource = + node.getFilteringSource().accept(this, new RewriteContext()); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + ImmutableSet sourceScope = ImmutableSet.copyOf(node.getSource().getOutputSymbols()); + EqualityInference inheritedInference = new EqualityInference(metadata, inheritedPredicate); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = inheritedInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Add the inherited equality predicates back in + EqualityInference.EqualityPartition equalityPartition = + inheritedInference.generateEqualitiesPartitionedBy(sourceScope); + sourceConjuncts.addAll(equalityPartition.getScopeEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeComplementEqualities()); + postJoinConjuncts.addAll(equalityPartition.getScopeStraddlingEqualities()); + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + + PlanNode output = appendSortNodeForSemiJoin(node, rewrittenSource, rewrittenFilteringSource); + + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + + private SemiJoinNode appendSortNodeForSemiJoin( + SemiJoinNode node, PlanNode rewrittenSource, PlanNode rewrittenFilteringSource) { + OrderingScheme sourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getSourceJoinSymbol()), + ImmutableMap.of(node.getSourceJoinSymbol(), ASC_NULLS_LAST)); + OrderingScheme filteringSourceOrderingScheme = + new OrderingScheme( + ImmutableList.of(node.getFilteringSourceJoinSymbol()), + // NULL first is used to make sure that we can know if there's null value in the + // result set of right table. + // For x in (subquery), if subquery returns null and some value a,b, and x is not + // in(a,b), the result of SemiJoinOutput should be NULL. + ImmutableMap.of(node.getFilteringSourceJoinSymbol(), ASC_NULLS_FIRST)); + SortNode sourceSortNode = + new SortNode( + queryId.genPlanNodeId(), rewrittenSource, sourceOrderingScheme, false, false); + SortNode filteringSourceSortNode = + new SortNode( + queryId.genPlanNodeId(), + rewrittenFilteringSource, + filteringSourceOrderingScheme, + false, + false); + return new SemiJoinNode( + node.getPlanNodeId(), + sourceSortNode, + filteringSourceSortNode, + node.getSourceJoinSymbol(), + node.getFilteringSourceJoinSymbol(), + node.getSemiJoinOutput()); + } + + private PlanNode visitFilteringSemiJoin(SemiJoinNode node, RewriteContext context) { + Expression inheritedPredicate = + context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; + Expression deterministicInheritedPredicate = filterDeterministicConjuncts(inheritedPredicate); + Expression sourceEffectivePredicate = TRUE_LITERAL; + Expression filteringSourceEffectivePredicate = TRUE_LITERAL; + // Expression sourceEffectivePredicate = + // filterDeterministicConjuncts(effectivePredicateExtractor.extract(session, node.getSource(), + // types, typeAnalyzer)); + // Expression filteringSourceEffectivePredicate = filterDeterministicConjuncts(metadata, + // effectivePredicateExtractor.extract(session, node.getFilteringSource(), types, + // typeAnalyzer)); + Expression joinExpression = + new ComparisonExpression( + EQUAL, + node.getSourceJoinSymbol().toSymbolReference(), + node.getFilteringSourceJoinSymbol().toSymbolReference()); + + List sourceSymbols = node.getSource().getOutputSymbols(); + List filteringSourceSymbols = node.getFilteringSource().getOutputSymbols(); + + List sourceConjuncts = new ArrayList<>(); + List filteringSourceConjuncts = new ArrayList<>(); + List postJoinConjuncts = new ArrayList<>(); + + // Generate equality inferences + EqualityInference allInference = + new EqualityInference( + metadata, + deterministicInheritedPredicate, + sourceEffectivePredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutSourceInferred = + new EqualityInference( + metadata, + deterministicInheritedPredicate, + filteringSourceEffectivePredicate, + joinExpression); + EqualityInference allInferenceWithoutFilteringSourceInferred = + new EqualityInference( + metadata, deterministicInheritedPredicate, sourceEffectivePredicate, joinExpression); + + // Push inheritedPredicates down to the source if they don't involve the semi join output + Set sourceScope = ImmutableSet.copyOf(sourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, inheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, sourceScope); + // Since each source row is reflected exactly once in the output, ok to push + // non-deterministic predicates down + if (rewrittenConjunct != null) { + sourceConjuncts.add(rewrittenConjunct); + } else { + postJoinConjuncts.add(conjunct); + } + }); + + // Push inheritedPredicates down to the filtering source if possible + Set filterScope = ImmutableSet.copyOf(filteringSourceSymbols); + EqualityInference.nonInferrableConjuncts(metadata, deterministicInheritedPredicate) + .forEach( + conjunct -> { + Expression rewrittenConjunct = allInference.rewrite(conjunct, filterScope); + // We cannot push non-deterministic predicates to filtering side. Each filtering + // side row have to be + // logically reevaluated for each source row. + if (rewrittenConjunct != null) { + filteringSourceConjuncts.add(rewrittenConjunct); + } + }); + + // move effective predicate conjuncts source <-> filter + // See if we can push the filtering source effective predicate to the source side + EqualityInference.nonInferrableConjuncts(metadata, filteringSourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, sourceScope)) + .filter(Objects::nonNull) + .forEach(sourceConjuncts::add); + + // See if we can push the source effective predicate to the filtering source side + EqualityInference.nonInferrableConjuncts(metadata, sourceEffectivePredicate) + .map(conjunct -> allInference.rewrite(conjunct, filterScope)) + .filter(Objects::nonNull) + .forEach(filteringSourceConjuncts::add); + + // Add equalities from the inference back in + sourceConjuncts.addAll( + allInferenceWithoutSourceInferred + .generateEqualitiesPartitionedBy(sourceScope) + .getScopeEqualities()); + filteringSourceConjuncts.addAll( + allInferenceWithoutFilteringSourceInferred + .generateEqualitiesPartitionedBy(filterScope) + .getScopeEqualities()); + + PlanNode rewrittenSource = + node.getSource().accept(this, new RewriteContext(combineConjuncts(sourceConjuncts))); + PlanNode rewrittenFilteringSource = + node.getFilteringSource() + .accept(this, new RewriteContext(combineConjuncts(filteringSourceConjuncts))); + + PlanNode output = appendSortNodeForSemiJoin(node, rewrittenSource, rewrittenFilteringSource); + if (!postJoinConjuncts.isEmpty()) { + output = + new FilterNode(queryId.genPlanNodeId(), output, combineConjuncts(postJoinConjuncts)); + } + return output; + } + @Override public PlanNode visitInsertTablet(InsertTabletNode node, RewriteContext context) { return node; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index dda33ea1c2f6..2b4144389837 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; @@ -731,6 +732,34 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { node.isSpillable()), outputMapping); } + + @Override + public PlanAndMappings visitSemiJoin(SemiJoinNode node, UnaliasContext context) { + // it is assumed that symbols are distinct between SemiJoin source and filtering source. Only + // symbols from outer correlation might be the exception + PlanAndMappings rewrittenSource = node.getSource().accept(this, context); + PlanAndMappings rewrittenFilteringSource = node.getFilteringSource().accept(this, context); + + Map outputMapping = new HashMap<>(); + outputMapping.putAll(rewrittenSource.getMappings()); + outputMapping.putAll(rewrittenFilteringSource.getMappings()); + + SymbolMapper mapper = symbolMapper(outputMapping); + + Symbol newSourceJoinSymbol = mapper.map(node.getSourceJoinSymbol()); + Symbol newFilteringSourceJoinSymbol = mapper.map(node.getFilteringSourceJoinSymbol()); + Symbol newSemiJoinOutput = mapper.map(node.getSemiJoinOutput()); + + return new PlanAndMappings( + new SemiJoinNode( + node.getPlanNodeId(), + rewrittenSource.getRoot(), + rewrittenFilteringSource.getRoot(), + newSourceJoinSymbol, + newFilteringSourceJoinSymbol, + newSemiJoinOutput), + outputMapping); + } } private static class UnaliasContext { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AbstractQueryDeviceWithCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AbstractQueryDeviceWithCache.java index 6617999b4b51..7f43347a0a11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AbstractQueryDeviceWithCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AbstractQueryDeviceWithCache.java @@ -61,6 +61,8 @@ public boolean parseRawExpression( final boolean needFetch = super.parseRawExpression(entries, tableInstance, attributeColumns, context); if (!needFetch) { + context.reserveMemoryForFrontEnd( + entries.stream().map(DeviceEntry::ramBytesUsed).reduce(0L, Long::sum)); results = entries.stream() .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java index 4d398b55fcc2..a40fab917a51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java @@ -88,7 +88,7 @@ public boolean isReplaceAllConnectorAttributes() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } @@ -106,25 +106,25 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - AlterPipe alterPipe = (AlterPipe) obj; - return Objects.equals(pipeName, alterPipe.pipeName) - && Objects.equals(ifExistsCondition, alterPipe.ifExistsCondition) - && Objects.equals(extractorAttributes, alterPipe.extractorAttributes) - && Objects.equals(processorAttributes, alterPipe.processorAttributes) - && Objects.equals(connectorAttributes, alterPipe.connectorAttributes) + final AlterPipe that = (AlterPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition) + && Objects.equals(this.extractorAttributes, that.extractorAttributes) + && Objects.equals(this.processorAttributes, that.processorAttributes) + && Objects.equals(this.connectorAttributes, that.connectorAttributes) && Objects.equals( - isReplaceAllExtractorAttributes, alterPipe.isReplaceAllExtractorAttributes) + this.isReplaceAllExtractorAttributes, that.isReplaceAllExtractorAttributes) && Objects.equals( - isReplaceAllProcessorAttributes, alterPipe.isReplaceAllProcessorAttributes) + this.isReplaceAllProcessorAttributes, that.isReplaceAllProcessorAttributes) && Objects.equals( - isReplaceAllConnectorAttributes, alterPipe.isReplaceAllConnectorAttributes); + this.isReplaceAllConnectorAttributes, that.isReplaceAllConnectorAttributes); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 43bc9223b114..43cc625c855f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -613,6 +613,10 @@ protected R visitShowStatement(ShowStatement node, C context) { return visitStatement(node, context); } + protected R visitCountStatement(CountStatement node, C context) { + return visitStatement(node, context); + } + protected R visitKillQuery(KillQuery node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CountStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CountStatement.java new file mode 100644 index 000000000000..146b2467aa47 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CountStatement.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CountStatement extends Statement { + private final String tableName; + + private final Optional where; + + public CountStatement( + final NodeLocation location, final String tableName, final Optional where) { + super(requireNonNull(location, "location is null")); + this.tableName = tableName; + this.where = where; + } + + public String getTableName() { + return tableName; + } + + public Optional getWhere() { + return where; + } + + @Override + public R accept(final AstVisitor visitor, final C context) { + return visitor.visitCountStatement(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CountStatement that = (CountStatement) o; + return Objects.equals(tableName, that.tableName) && Objects.equals(where, that.where); + } + + @Override + public int hashCode() { + return Objects.hash(tableName, where); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("tableName", tableName) + .add("where", where.orElse(null)) + .omitNullValues() + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java index 6952c06d13d6..1b3837ba2147 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java @@ -43,7 +43,7 @@ public boolean hasIfExistsCondition() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } @@ -53,16 +53,16 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - DropPipe other = (DropPipe) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(ifExistsCondition, other.ifExistsCondition); + final DropPipe that = (DropPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 2025df1a973d..669d8da6645c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -44,6 +44,7 @@ public class LoadTsFile extends Statement { private boolean deleteAfterLoad = false; private boolean convertOnTypeMismatch = true; private boolean autoCreateDatabase = true; + private boolean isGeneratedByPipe = false; private String model = LoadTsFileConfigurator.MODEL_TABLE_VALUE; private final Map loadAttributes; @@ -112,6 +113,14 @@ public LoadTsFile setDatabase(String database) { return this; } + public void markIsGeneratedByPipe() { + isGeneratedByPipe = true; + } + + public boolean isGeneratedByPipe() { + return isGeneratedByPipe; + } + public String getModel() { return model; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java index e65079610931..c0dfe9379de1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java @@ -44,7 +44,7 @@ public boolean hasWhereClause() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } @@ -54,16 +54,16 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - ShowPipes other = (ShowPipes) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(hasWhereClause, other.hasWhereClause); + final ShowPipes that = (ShowPipes) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.hasWhereClause, that.hasWhereClause); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java index 540d6fef9ec8..15eb5b2413dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java @@ -37,7 +37,7 @@ public String getPipeName() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } @@ -47,15 +47,15 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StartPipe other = (StartPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StartPipe that = (StartPipe) obj; + return Objects.equals(this.pipeName, that.pipeName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java index daa9ab32e02b..6d1a629de882 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java @@ -37,7 +37,7 @@ public String getPipeName() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } @@ -47,15 +47,15 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StopPipe other = (StopPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StopPipe that = (StopPipe) obj; + return Objects.equals(this.pipeName, that.pipeName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index d3b38af0f581..361956dd49b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; @@ -1022,21 +1023,36 @@ public Node visitShowSubscriptionsStatement( @Override public Node visitShowDevicesStatement(final RelationalSqlParser.ShowDevicesStatementContext ctx) { - return new ShowDevice( - getLocation(ctx), - new Table(getLocation(ctx), getQualifiedName(ctx.qualifiedName())), - visitIfPresent(ctx.where, Expression.class).orElse(null), - visitIfPresent(ctx.limitOffsetClause().offset, Offset.class).orElse(null), - visitIfPresent(ctx.limitOffsetClause().limit, Node.class).orElse(null)); + final QualifiedName name = getQualifiedName(ctx.tableName); + return InformationSchema.INFORMATION_DATABASE.equals( + name.getPrefix().map(QualifiedName::toString).orElse(clientSession.getDatabaseName())) + ? new ShowStatement( + getLocation(ctx), + name.getSuffix(), + visitIfPresent(ctx.where, Expression.class), + Optional.empty(), + visitIfPresent(ctx.limitOffsetClause().offset, Offset.class), + visitIfPresent(ctx.limitOffsetClause().limit, Node.class)) + : new ShowDevice( + getLocation(ctx), + new Table(getLocation(ctx), name), + visitIfPresent(ctx.where, Expression.class).orElse(null), + visitIfPresent(ctx.limitOffsetClause().offset, Offset.class).orElse(null), + visitIfPresent(ctx.limitOffsetClause().limit, Node.class).orElse(null)); } @Override public Node visitCountDevicesStatement( final RelationalSqlParser.CountDevicesStatementContext ctx) { - return new CountDevice( - getLocation(ctx), - new Table(getLocation(ctx), getQualifiedName(ctx.qualifiedName())), - visitIfPresent(ctx.where, Expression.class).orElse(null)); + final QualifiedName name = getQualifiedName(ctx.tableName); + return InformationSchema.INFORMATION_DATABASE.equals( + name.getPrefix().map(QualifiedName::toString).orElse(clientSession.getDatabaseName())) + ? new CountStatement( + getLocation(ctx), name.getSuffix(), visitIfPresent(ctx.where, Expression.class)) + : new CountDevice( + getLocation(ctx), + new Table(getLocation(ctx), name), + visitIfPresent(ctx.where, Expression.class).orElse(null)); } @Override @@ -1855,8 +1871,7 @@ public Node visitInList(RelationalSqlParser.InListContext ctx) { @Override public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { - throw new SemanticException("Only TableSubquery is supported now"); - /*Expression result = + Expression result = new InPredicate( getLocation(ctx), (Expression) visit(ctx.value), @@ -1866,7 +1881,7 @@ public Node visitInSubquery(RelationalSqlParser.InSubqueryContext ctx) { result = new NotExpression(getLocation(ctx), result); } - return result;*/ + return result; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java index ff0a426657aa..d852319169d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java @@ -26,14 +26,20 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowStatement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -50,7 +56,7 @@ public final class ShowRewrite implements StatementRewrite.Rewrite { // private final SqlParser parser; // private final AccessControl accessControl; - public ShowRewrite(Metadata metadata) { + public ShowRewrite(final Metadata metadata) { this.metadata = requireNonNull(metadata, "metadata is null"); // this.parser = requireNonNull(parser, "parser is null"); // this.accessControl = requireNonNull(accessControl, "accessControl is null"); @@ -58,13 +64,13 @@ public ShowRewrite(Metadata metadata) { @Override public Statement rewrite( - StatementAnalyzerFactory analyzerFactory, - SessionInfo session, - Statement node, - List parameters, - Map, Expression> parameterLookup, - WarningCollector warningCollector) { - Visitor visitor = new Visitor(metadata, session); + final StatementAnalyzerFactory analyzerFactory, + final SessionInfo session, + final Statement node, + final List parameters, + final Map, Expression> parameterLookup, + final WarningCollector warningCollector) { + final Visitor visitor = new Visitor(metadata, session); return (Statement) visitor.process(node, null); } @@ -72,13 +78,13 @@ private static class Visitor extends AstVisitor { private final Metadata metadata; private final SessionInfo session; - public Visitor(Metadata metadata, SessionInfo session) { + public Visitor(final Metadata metadata, final SessionInfo session) { this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); } @Override - protected Node visitShowStatement(ShowStatement showStatement, Void context) { + protected Node visitShowStatement(final ShowStatement showStatement, final Void context) { // CatalogSchemaName schema = createCatalogSchemaName(session, showQueries, // showQueries.getSchema()); @@ -96,12 +102,33 @@ protected Node visitShowStatement(ShowStatement showStatement, Void context) { showStatement.getLimit()); } - private static Relation from(String db, String table) { + @Override + protected Node visitCountStatement(final CountStatement countStatement, final Void context) { + return simpleQuery( + new Select( + false, + Collections.singletonList( + new SingleColumn( + new FunctionCall( + QualifiedName.of(Collections.singletonList(new Identifier("count"))), + Collections.emptyList()), + new Identifier("count(devices)")))), + from(INFORMATION_DATABASE, countStatement.getTableName()), + countStatement.getWhere(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + + private static Relation from(final String db, final String table) { return table(QualifiedName.of(db, table)); } @Override - protected Node visitNode(Node node, Void context) { + protected Node visitNode(final Node node, final Void context) { return node; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index aab470880cfb..1a6b6b86d573 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -554,7 +554,7 @@ private boolean loadLocally(LoadSingleTsFileNode node) throws IoTDBException { private void convertFailedTsFilesToTabletsAndRetry() { final LoadTsFileDataTypeConverter loadTsFileDataTypeConverter = - new LoadTsFileDataTypeConverter(); + new LoadTsFileDataTypeConverter(isGeneratedByPipe); for (final int failedLoadTsFileIndex : failedTsFileNodeIndexes) { final LoadSingleTsFileNode failedNode = tsFileNodeList.get(failedLoadTsFileIndex); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java index eb6da0bc2ffa..7ae7b0232005 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java @@ -341,6 +341,10 @@ public List getFailedMeasurements() { .collect(Collectors.toList()); } + public Map getFailedMeasurementInfoMap() { + return failedMeasurementIndex2Info; + } + public List getFailedExceptions() { return failedMeasurementIndex2Info == null ? Collections.emptyList() @@ -364,7 +368,7 @@ public List getFailedMessages() { .collect(Collectors.toList()); } - protected static class FailedMeasurementInfo { + public static class FailedMeasurementInfo { protected String measurement; protected TSDataType dataType; protected Object value; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java index 85f82ff447db..d33a83b0c326 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java @@ -276,13 +276,25 @@ public void removeAllFailedMeasurementMarks() { } failedMeasurementIndex2Info.forEach( (index, info) -> { - measurements[index] = info.getMeasurement(); - dataTypes[index] = info.getDataType(); - values[index] = info.getValue(); + if (measurements != null) { + measurements[index] = info.getMeasurement(); + } + + if (dataTypes != null) { + dataTypes[index] = info.getDataType(); + } + + if (values != null) { + values[index] = info.getValue(); + } }); failedMeasurementIndex2Info.clear(); } + public Map getFailedMeasurementInfoMap() { + return failedMeasurementIndex2Info; + } + @Override public void semanticCheck() { super.semanticCheck(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 520700d38483..a28e7d037790 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -60,6 +60,7 @@ public class LoadTsFileStatement extends Statement { private boolean deleteAfterLoad = false; private boolean convertOnTypeMismatch = true; private boolean autoCreateDatabase = true; + private boolean isGeneratedByPipe = false; private String model = LoadTsFileConfigurator.MODEL_TREE_VALUE; private Map loadAttributes; @@ -198,6 +199,14 @@ public String getModel() { return model; } + public void markIsGeneratedByPipe() { + isGeneratedByPipe = true; + } + + public boolean isGeneratedByPipe() { + return isGeneratedByPipe; + } + public List getTsFiles() { return tsFiles; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java index de5dc3c1d594..44065cf31850 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java @@ -44,8 +44,9 @@ public class AlterPipeStatement extends Statement implements IConfigStatement { private boolean isReplaceAllExtractorAttributes; private boolean isReplaceAllProcessorAttributes; private boolean isReplaceAllConnectorAttributes; + private boolean isTableModel; - public AlterPipeStatement(StatementType alterPipeStatement) { + public AlterPipeStatement(final StatementType alterPipeStatement) { this.statementType = alterPipeStatement; } @@ -81,38 +82,46 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } - public void setExtractorAttributes(Map extractorAttributes) { + public void setExtractorAttributes(final Map extractorAttributes) { this.extractorAttributes = extractorAttributes; } - public void setProcessorAttributes(Map processorAttributes) { + public void setProcessorAttributes(final Map processorAttributes) { this.processorAttributes = processorAttributes; } - public void setConnectorAttributes(Map connectorAttributes) { + public void setConnectorAttributes(final Map connectorAttributes) { this.connectorAttributes = connectorAttributes; } - public void setReplaceAllExtractorAttributes(boolean replaceAllExtractorAttributes) { + public void setReplaceAllExtractorAttributes(final boolean replaceAllExtractorAttributes) { isReplaceAllExtractorAttributes = replaceAllExtractorAttributes; } - public void setReplaceAllProcessorAttributes(boolean replaceAllProcessorAttributes) { + public void setReplaceAllProcessorAttributes(final boolean replaceAllProcessorAttributes) { isReplaceAllProcessorAttributes = replaceAllProcessorAttributes; } - public void setReplaceAllConnectorAttributes(boolean replaceAllConnectorAttributes) { + public void setReplaceAllConnectorAttributes(final boolean replaceAllConnectorAttributes) { isReplaceAllConnectorAttributes = replaceAllConnectorAttributes; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -124,7 +133,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -134,7 +143,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java index a3403e00e689..c61a15d777d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java @@ -37,27 +37,36 @@ public class DropPipeStatement extends Statement implements IConfigStatement { private String pipeName; private boolean ifExistsCondition; + private boolean isTableModel; - public DropPipeStatement(StatementType dropPipeStatement) { + public DropPipeStatement(final StatementType dropPipeStatement) { this.statementType = dropPipeStatement; } + public String getPipeName() { + return pipeName; + } + public boolean hasIfExistsCondition() { return ifExistsCondition; } - public String getPipeName() { - return pipeName; + public boolean isTableModel() { + return isTableModel; } - public void setPipeName(String pipeName) { + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -69,7 +78,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -79,7 +88,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java index 54fe1099f39f..658f36a0635f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java @@ -39,6 +39,8 @@ public ShowPipesStatement() { private boolean whereClause; + private boolean isTableModel; + public String getPipeName() { return pipeName; } @@ -47,21 +49,29 @@ public boolean getWhereClause() { return whereClause; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setWhereClause(boolean whereClause) { + public void setWhereClause(final boolean whereClause) { this.whereClause = whereClause; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.READ; } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -71,7 +81,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java index 43b61a1fb445..9023d77c06ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java @@ -36,8 +36,9 @@ public class StartPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private boolean isTableModel; - public StartPipeStatement(StatementType startPipeStatement) { + public StartPipeStatement(final StatementType startPipeStatement) { this.statementType = startPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java index d463fda199d3..ea14a3dbd1b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java @@ -36,8 +36,9 @@ public class StopPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private boolean isTableModel; - public StopPipeStatement(StatementType stopPipeStatement) { + public StopPipeStatement(final StatementType stopPipeStatement) { this.statementType = stopPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public boolean isTableModel() { + return isTableModel; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java index 2e0d8b7612d6..54f5769aead0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/InformationSchemaUtils.java @@ -21,11 +21,11 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.table.TableNotExistsException; import org.apache.iotdb.commons.schema.column.ColumnHeader; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; @@ -57,11 +57,16 @@ public static void checkDBNameInWrite(final String dbName) { } public static void buildDatabaseTsBlock( - final Predicate canSeenDB, final TsBlockBuilder builder, final boolean details) { + final Predicate canSeenDB, + final TsBlockBuilder builder, + final boolean details, + final boolean withTime) { if (!canSeenDB.test(INFORMATION_DATABASE)) { return; } - builder.getTimeColumnBuilder().writeLong(0L); + if (withTime) { + builder.getTimeColumnBuilder().writeLong(0L); + } builder .getColumnBuilder(0) .writeBinary(new Binary(INFORMATION_DATABASE, TSFileConfig.STRING_CHARSET)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index fdb4e771148e..84ff12e3dc7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -34,6 +34,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.consensus.ConsensusFactory; @@ -175,7 +176,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -2718,31 +2718,32 @@ public int executeCompaction() throws InterruptedException { if (!isCompactionSelecting.compareAndSet(false, true)) { return 0; } - int trySubmitCount = 0; + CompactionScheduleContext context = new CompactionScheduleContext(); try { List timePartitions = new ArrayList<>(tsFileManager.getTimePartitions()); // Sort the time partition from largest to smallest timePartitions.sort(Comparator.reverseOrder()); - CompactionScheduleContext context = new CompactionScheduleContext(); - // schedule insert compaction - trySubmitCount += executeInsertionCompaction(timePartitions, context); - context.incrementSubmitTaskNum(CompactionTaskType.INSERTION, trySubmitCount); + int[] submitCountOfTimePartitions = executeInsertionCompaction(timePartitions, context); // schedule the other compactions - if (trySubmitCount == 0) { - // the name of this variable is trySubmitCount, because the task submitted to the queue - // could be evicted due to the low priority of the task - for (long timePartition : timePartitions) { - CompactionScheduler.sharedLockCompactionSelection(); - try { - trySubmitCount += - CompactionScheduler.scheduleCompaction(tsFileManager, timePartition, context); - } finally { - context.clearTimePartitionDeviceInfoCache(); - CompactionScheduler.sharedUnlockCompactionSelection(); - } + for (int i = 0; i < timePartitions.size(); i++) { + boolean skipOtherCompactionSchedule = + submitCountOfTimePartitions[i] > 0 + && !config + .getDataRegionConsensusProtocolClass() + .equals(ConsensusFactory.IOT_CONSENSUS_V2); + if (skipOtherCompactionSchedule) { + continue; + } + long timePartition = timePartitions.get(i); + CompactionScheduler.sharedLockCompactionSelection(); + try { + CompactionScheduler.scheduleCompaction(tsFileManager, timePartition, context); + } finally { + context.clearTimePartitionDeviceInfoCache(); + CompactionScheduler.sharedUnlockCompactionSelection(); } } if (context.hasSubmitTask()) { @@ -2755,7 +2756,7 @@ public int executeCompaction() throws InterruptedException { } finally { isCompactionSelecting.set(false); } - return trySubmitCount; + return context.getSubmitCompactionTaskNum(); } /** Schedule settle compaction for ttl check. */ @@ -2802,40 +2803,36 @@ public int executeTTLCheck() throws InterruptedException { return trySubmitCount; } - protected int executeInsertionCompaction( + protected int[] executeInsertionCompaction( List timePartitions, CompactionScheduleContext context) throws InterruptedException { - int trySubmitCount = 0; + int[] trySubmitCountOfTimePartitions = new int[timePartitions.size()]; CompactionScheduler.sharedLockCompactionSelection(); try { while (true) { int currentSubmitCount = 0; - for (long timePartition : timePartitions) { - while (true) { - Phaser insertionTaskPhaser = new Phaser(1); - int selectedTaskNum = - CompactionScheduler.scheduleInsertionCompaction( - tsFileManager, timePartition, insertionTaskPhaser, context); - insertionTaskPhaser.awaitAdvanceInterruptibly(insertionTaskPhaser.arrive()); - currentSubmitCount += selectedTaskNum; - if (selectedTaskNum <= 0) { - break; - } - } + for (int i = 0; i < timePartitions.size(); i++) { + long timePartition = timePartitions.get(i); + int selectedTaskNum = + CompactionScheduler.scheduleInsertionCompaction( + tsFileManager, timePartition, context); + currentSubmitCount += selectedTaskNum; + trySubmitCountOfTimePartitions[i] += selectedTaskNum; context.clearTimePartitionDeviceInfoCache(); } if (currentSubmitCount <= 0) { break; } - trySubmitCount += currentSubmitCount; + context.incrementSubmitTaskNum(CompactionTaskType.INSERTION, currentSubmitCount); } } catch (InterruptedException e) { throw e; } catch (Throwable e) { logger.error("Meet error in insertion compaction schedule.", e); } finally { + context.clearTimePartitionDeviceInfoCache(); CompactionScheduler.sharedUnlockCompactionSelection(); } - return trySubmitCount; + return trySubmitCountOfTimePartitions; } /** @@ -3025,8 +3022,7 @@ private boolean loadTsFileToUnSequence( final boolean deleteOriginFile, boolean isGeneratedByPipe) throws LoadFileException, DiskSpaceInsufficientException { - final File targetFile; - targetFile = + final File targetFile = fsFactory.getFile( TierManager.getInstance().getNextFolderForTsFile(0, false), databaseName @@ -3038,7 +3034,7 @@ private boolean loadTsFileToUnSequence( + tsFileResource.getTsFile().getName()); tsFileResource.setFile(targetFile); if (tsFileManager.contains(tsFileResource, false)) { - logger.error("The file {} has already been loaded in unsequence list", tsFileResource); + logger.warn("The file {} has already been loaded in unsequence list", tsFileResource); return false; } @@ -3055,12 +3051,20 @@ private boolean loadTsFileToUnSequence( } try { if (deleteOriginFile) { - FileUtils.moveFile(tsFileToLoad, targetFile); + RetryUtils.retryOnException( + () -> { + FileUtils.moveFile(tsFileToLoad, targetFile); + return null; + }); } else { - Files.copy(tsFileToLoad.toPath(), targetFile.toPath()); + RetryUtils.retryOnException( + () -> { + Files.copy(tsFileToLoad.toPath(), targetFile.toPath()); + return null; + }); } } catch (final IOException e) { - logger.error( + logger.warn( "File renaming failed when loading tsfile. Origin: {}, Target: {}", tsFileToLoad.getAbsolutePath(), targetFile.getAbsolutePath(), @@ -3077,13 +3081,20 @@ private boolean loadTsFileToUnSequence( fsFactory.getFile(targetFile.getAbsolutePath() + RESOURCE_SUFFIX); try { if (deleteOriginFile) { - FileUtils.moveFile(resourceFileToLoad, targetResourceFile); + RetryUtils.retryOnException( + () -> { + FileUtils.moveFile(resourceFileToLoad, targetResourceFile); + return null; + }); } else { - Files.copy(resourceFileToLoad.toPath(), targetResourceFile.toPath()); + RetryUtils.retryOnException( + () -> { + Files.copy(resourceFileToLoad.toPath(), targetResourceFile.toPath()); + return null; + }); } - } catch (final IOException e) { - logger.error( + logger.warn( "File renaming failed when loading .resource file. Origin: {}, Target: {}", resourceFileToLoad.getAbsolutePath(), targetResourceFile.getAbsolutePath(), @@ -3133,18 +3144,30 @@ private void moveModFile(File modFileToLoad, File targetModFile, boolean deleteO // when successfully loaded, the filepath of the resource will be changed to the IoTDB data // dir, so we can add a suffix to find the old modification file. try { - Files.deleteIfExists(targetModFile.toPath()); + RetryUtils.retryOnException( + () -> { + Files.deleteIfExists(targetModFile.toPath()); + return null; + }); } catch (final IOException e) { logger.warn("Cannot delete localModFile {}", targetModFile, e); } try { if (deleteOriginFile) { - FileUtils.moveFile(modFileToLoad, targetModFile); + RetryUtils.retryOnException( + () -> { + FileUtils.moveFile(modFileToLoad, targetModFile); + return null; + }); } else { - Files.copy(modFileToLoad.toPath(), targetModFile.toPath()); + RetryUtils.retryOnException( + () -> { + Files.copy(modFileToLoad.toPath(), targetModFile.toPath()); + return null; + }); } } catch (final IOException e) { - logger.error( + logger.warn( "File renaming failed when loading .mod file. Origin: {}, Target: {}", modFileToLoad.getAbsolutePath(), targetModFile.getAbsolutePath(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java index 4d51ba5010f0..1f01cac0b255 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java @@ -31,7 +31,9 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class CompactionScheduleContext { private int submitSeqInnerSpaceCompactionTaskNum = 0; @@ -49,8 +51,19 @@ public class CompactionScheduleContext { private final Map partitionFileDeviceInfoCache; private long cachedDeviceInfoSize = 0; + private final Set timePartitionsDelayInsertionSelection; + public CompactionScheduleContext() { this.partitionFileDeviceInfoCache = new HashMap<>(); + this.timePartitionsDelayInsertionSelection = new HashSet<>(); + } + + public void delayInsertionSelection(long timePartitionId) { + timePartitionsDelayInsertionSelection.add(timePartitionId); + } + + public boolean isInsertionSelectionDelayed(long timePartitionId) { + return timePartitionsDelayInsertionSelection.remove(timePartitionId); } public void addResourceDeviceTimeIndex( @@ -135,6 +148,14 @@ public int getSubmitSettleCompactionTaskNum() { return submitSettleCompactionTaskNum; } + public int getSubmitCompactionTaskNum() { + return submitSeqInnerSpaceCompactionTaskNum + + submitUnseqInnerSpaceCompactionTaskNum + + submitCrossSpaceCompactionTaskNum + + submitInsertionCrossSpaceCompactionTaskNum + + submitSettleCompactionTaskNum; + } + public boolean hasSubmitTask() { return submitCrossSpaceCompactionTaskNum + submitInsertionCrossSpaceCompactionTaskNum diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java index c465e6fa8631..a8877f9ae8f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java @@ -93,29 +93,32 @@ public static void exclusiveUnlockCompactionSelection() { * @param context the context of compaction schedule * @return the count of submitted task */ - public static int scheduleCompaction( + public static void scheduleCompaction( TsFileManager tsFileManager, long timePartition, CompactionScheduleContext context) throws InterruptedException { if (!tsFileManager.isAllowCompaction()) { - return 0; + return; } // the name of this variable is trySubmitCount, because the task submitted to the queue could be // evicted due to the low priority of the task - int trySubmitCount = 0; try { - trySubmitCount += + int submitInnerTaskNum = 0; + submitInnerTaskNum += tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, true, context); - trySubmitCount += + submitInnerTaskNum += tryToSubmitInnerSpaceCompactionTask(tsFileManager, timePartition, false, context); - trySubmitCount += tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, context); - trySubmitCount += - tryToSubmitSettleCompactionTask(tsFileManager, timePartition, context, false); + boolean executeDelayedInsertionSelection = + submitInnerTaskNum == 0 && context.isInsertionSelectionDelayed(timePartition); + if (executeDelayedInsertionSelection) { + scheduleInsertionCompaction(tsFileManager, timePartition, context); + } + tryToSubmitCrossSpaceCompactionTask(tsFileManager, timePartition, context); + tryToSubmitSettleCompactionTask(tsFileManager, timePartition, context, false); } catch (InterruptedException e) { throw e; } catch (Throwable e) { LOGGER.error("Meet error in compaction schedule.", e); } - return trySubmitCount; } @TestOnly @@ -124,22 +127,6 @@ public static void scheduleCompaction(TsFileManager tsFileManager, long timePart scheduleCompaction(tsFileManager, timePartition, new CompactionScheduleContext()); } - public static int scheduleInsertionCompaction( - TsFileManager tsFileManager, - long timePartition, - Phaser insertionTaskPhaser, - CompactionScheduleContext context) - throws InterruptedException { - if (!tsFileManager.isAllowCompaction()) { - return 0; - } - int trySubmitCount = 0; - trySubmitCount += - tryToSubmitInsertionCompactionTask( - tsFileManager, timePartition, insertionTaskPhaser, context); - return trySubmitCount; - } - public static int tryToSubmitInnerSpaceCompactionTask( TsFileManager tsFileManager, long timePartition, @@ -223,13 +210,31 @@ private static boolean canAddTaskToWaitingQueue(AbstractCompactionTask task) return true; } - private static int tryToSubmitInsertionCompactionTask( + public static int scheduleInsertionCompaction( + TsFileManager tsFileManager, long timePartition, CompactionScheduleContext context) + throws InterruptedException { + int count = 0; + while (true) { + Phaser insertionTaskPhaser = new Phaser(1); + int selectedTaskNum = + tryToSubmitInsertionCompactionTask( + tsFileManager, timePartition, insertionTaskPhaser, context); + insertionTaskPhaser.awaitAdvanceInterruptibly(insertionTaskPhaser.arrive()); + if (selectedTaskNum <= 0) { + break; + } + count += selectedTaskNum; + } + return count; + } + + public static int tryToSubmitInsertionCompactionTask( TsFileManager tsFileManager, long timePartition, Phaser insertionTaskPhaser, CompactionScheduleContext context) throws InterruptedException { - if (!config.isEnableCrossSpaceCompaction()) { + if (!tsFileManager.isAllowCompaction() || !config.isEnableCrossSpaceCompaction()) { return 0; } String logicalStorageGroupName = tsFileManager.getStorageGroupName(); @@ -351,6 +356,7 @@ public static int tryToSubmitSettleCompactionTask( trySubmitCount++; } } + context.incrementSubmitTaskNum(CompactionTaskType.SETTLE, trySubmitCount); return trySubmitCount; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java index ef1fe4e02083..804d441d2f25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java @@ -69,7 +69,8 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector protected TsFileManager tsFileManager; private static boolean hasPrintedLog = false; - private static int maxDeserializedFileNumToCheckInsertionCandidateValid = 500; + private static int maxDeserializedFileNumToCheckInsertionCandidateValid = 100; + private static int maxFileNumToSelectInsertionTaskInOnePartition = 200; private final long memoryBudget; private final int maxCrossCompactionFileNum; @@ -166,6 +167,14 @@ public InsertionCrossCompactionTaskResource selectOneInsertionTask( "Selecting insertion cross compaction task resources from {} seqFile, {} unseqFiles", candidate.getSeqFiles().size(), candidate.getUnseqFiles().size()); + boolean delaySelection = + candidate.getSeqFiles().size() + candidate.getUnseqFiles().size() + > maxFileNumToSelectInsertionTaskInOnePartition; + if (delaySelection) { + context.delayInsertionSelection(timePartition); + return new InsertionCrossCompactionTaskResource(); + } + InsertionCrossCompactionTaskResource result = insertionCrossSpaceCompactionSelector.executeInsertionCrossSpaceCompactionTaskSelection(); if (result.isValid()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index c4dc75f591b0..b88ab2c65200 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; @@ -386,7 +387,7 @@ private TsFileWriterManager(File taskDir) { private void clearDir(File dir) { if (dir.exists()) { - FileUtils.deleteFileOrDirectory(dir); + FileUtils.deleteFileOrDirectoryWithRetry(dir); } if (dir.mkdirs()) { LOGGER.info("Load TsFile dir {} is created.", dir.getPath()); @@ -563,7 +564,11 @@ private void close() { } final Path writerPath = writer.getFile().toPath(); if (Files.exists(writerPath)) { - Files.delete(writerPath); + RetryUtils.retryOnException( + () -> { + Files.delete(writerPath); + return null; + }); } } catch (IOException e) { LOGGER.warn("Close TsFileIOWriter {} error.", entry.getValue().getFile().getPath(), e); @@ -578,7 +583,11 @@ private void close() { modificationFile.close(); final Path modificationFilePath = modificationFile.getFile().toPath(); if (Files.exists(modificationFilePath)) { - Files.delete(modificationFilePath); + RetryUtils.retryOnException( + () -> { + Files.delete(modificationFilePath); + return null; + }); } } catch (IOException e) { LOGGER.warn("Close ModificationFile {} error.", entry.getValue().getFile(), e); @@ -586,7 +595,11 @@ private void close() { } } try { - Files.delete(taskDir.toPath()); + RetryUtils.retryOnException( + () -> { + Files.delete(taskDir.toPath()); + return null; + }); } catch (DirectoryNotEmptyException e) { LOGGER.info("Task dir {} is not empty, skip deleting.", taskDir.getPath()); } catch (IOException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadDirScanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadDirScanner.java index c5fc2b59b96f..34bb392dc284 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadDirScanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadDirScanner.java @@ -102,17 +102,22 @@ private void scan() throws IOException { listeningDir.equals(IOTDB_CONFIG.getLoadActiveListeningPipeDir()); try (final Stream fileStream = FileUtils.streamFiles(new File(listeningDir), true, (String[]) null)) { - fileStream - .filter(file -> !activeLoadTsFileLoader.isFilePendingOrLoading(file)) - .filter(File::exists) - .map( - file -> - (file.getName().endsWith(RESOURCE) || file.getName().endsWith(MODS)) - ? getTsFilePath(file.getAbsolutePath()) - : file.getAbsolutePath()) - .filter(this::isTsFileCompleted) - .limit(currentAllowedPendingSize) - .forEach(file -> activeLoadTsFileLoader.tryTriggerTsFileLoad(file, isGeneratedByPipe)); + try { + fileStream + .filter(file -> !activeLoadTsFileLoader.isFilePendingOrLoading(file)) + .filter(File::exists) + .map( + file -> + (file.getName().endsWith(RESOURCE) || file.getName().endsWith(MODS)) + ? getTsFilePath(file.getAbsolutePath()) + : file.getAbsolutePath()) + .filter(this::isTsFileCompleted) + .limit(currentAllowedPendingSize) + .forEach( + file -> activeLoadTsFileLoader.tryTriggerTsFileLoad(file, isGeneratedByPipe)); + } catch (final Exception e) { + LOGGER.warn("Exception occurred during scanning dir: {}", listeningDir, e); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java index 188ae30436d8..8cb79658b668 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.WrappedThreadPoolExecutor; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -94,7 +95,11 @@ private void initFailDirIfNecessary() { if (!Objects.equals(failDir.get(), IOTDB_CONFIG.getLoadActiveListeningFailDir())) { final File failDirFile = new File(IOTDB_CONFIG.getLoadActiveListeningFailDir()); try { - FileUtils.forceMkdir(failDirFile); + RetryUtils.retryOnException( + () -> { + FileUtils.forceMkdir(failDirFile); + return null; + }); } catch (final IOException e) { LOGGER.warn( "Error occurred during creating fail directory {} for active load.", @@ -258,7 +263,11 @@ private void removeToFailDir(final String filePath) { final File targetDir = new File(failDir.get()); try { - org.apache.iotdb.commons.utils.FileUtils.moveFileWithMD5Check(sourceFile, targetDir); + RetryUtils.retryOnException( + () -> { + org.apache.iotdb.commons.utils.FileUtils.moveFileWithMD5Check(sourceFile, targetDir); + return null; + }); } catch (final IOException e) { LOGGER.warn("Error occurred during moving file {} to fail directory.", filePath, e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java index 1a8144424028..1366b0720c55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadConvertedInsertTabletStatement.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.pipe.receiver.transform.statement.PipeConvertedInsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +33,21 @@ public class LoadConvertedInsertTabletStatement extends PipeConvertedInsertTable private static final Logger LOGGER = LoggerFactory.getLogger(LoadConvertedInsertTabletStatement.class); - public LoadConvertedInsertTabletStatement(final InsertTabletStatement insertTabletStatement) { + private final boolean shouldConvertOnTypeMismatch; + + public LoadConvertedInsertTabletStatement( + final InsertTabletStatement insertTabletStatement, + final boolean shouldConvertOnTypeMismatch) { super(insertTabletStatement); + this.shouldConvertOnTypeMismatch = shouldConvertOnTypeMismatch; } @Override protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { + if (!shouldConvertOnTypeMismatch) { + return originalCheckAndCastDataType(columnIndex, dataType); + } + LOGGER.info( "Load: Inserting tablet to {}.{}. Casting type from {} to {}.", devicePath, @@ -49,4 +59,10 @@ protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) { dataTypes[columnIndex] = dataType; return true; } + + @TableModel + @Override + public boolean isForceTypeConversion() { + return shouldConvertOnTypeMismatch; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 233f60124ac2..de9dd0908879 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; -import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq; +import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReqV2; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.table.TsFileInsertionEventTableParser; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; @@ -86,10 +86,12 @@ public Optional visitLoadTsFile( final LoadConvertedInsertTabletStatement statement = new LoadConvertedInsertTabletStatement( - PipeTransferTabletRawReq.toTPipeTransferRawReq( + PipeTransferTabletRawReqV2.toTPipeTransferRawReq( rawTabletInsertionEvent.convertToTablet(), - rawTabletInsertionEvent.isAligned()) - .constructStatement()); + rawTabletInsertionEvent.isAligned(), + databaseName) + .constructStatement(), + loadTsFileStatement.isConvertOnTypeMismatch()); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index 5793b96502a1..6f7d732e6fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -76,7 +76,8 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { new LoadConvertedInsertTabletStatement( PipeTransferTabletRawReq.toTPipeTransferRawReq( tabletWithIsAligned.getLeft(), tabletWithIsAligned.getRight()) - .constructStatement()); + .constructStatement(), + loadTsFileStatement.isConvertOnTypeMismatch()); TSStatus result; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java index e1ad5c19cd11..57e3df75ac03 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTsFileDataTypeConverter.java @@ -28,7 +28,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -42,45 +44,26 @@ public class LoadTsFileDataTypeConverter { private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); + public static final LoadConvertedInsertTabletStatementTSStatusVisitor STATEMENT_STATUS_VISITOR = + new LoadConvertedInsertTabletStatementTSStatusVisitor(); + public static final LoadConvertedInsertTabletStatementExceptionVisitor + STATEMENT_EXCEPTION_VISITOR = new LoadConvertedInsertTabletStatementExceptionVisitor(); + + private final boolean isGeneratedByPipe; + private final SqlParser relationalSqlParser = new SqlParser(); private final LoadTableStatementDataTypeConvertExecutionVisitor tableStatementDataTypeConvertExecutionVisitor = - new LoadTableStatementDataTypeConvertExecutionVisitor( - ((statement, databaseName) -> - Coordinator.getInstance() - .executeForTableModel( - statement, - relationalSqlParser, - SESSION_MANAGER.getCurrSession(), - SESSION_MANAGER.requestQueryId(), - SESSION_MANAGER.getSessionInfoOfPipeReceiver( - SESSION_MANAGER.getCurrSession(), databaseName), - "", - LocalExecutionPlanner.getInstance().metadata, - IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold()) - .status)); + new LoadTableStatementDataTypeConvertExecutionVisitor(this::executeForTableModel); private final LoadTreeStatementDataTypeConvertExecutionVisitor treeStatementDataTypeConvertExecutionVisitor = - new LoadTreeStatementDataTypeConvertExecutionVisitor( - statement -> - Coordinator.getInstance() - .executeForTreeModel( - statement, - SESSION_MANAGER.requestQueryId(), - SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()), - "", - ClusterPartitionFetcher.getInstance(), - ClusterSchemaFetcher.getInstance(), - IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(), - false) - .status); + new LoadTreeStatementDataTypeConvertExecutionVisitor(this::executeForTreeModel); - public static final LoadConvertedInsertTabletStatementTSStatusVisitor STATEMENT_STATUS_VISITOR = - new LoadConvertedInsertTabletStatementTSStatusVisitor(); - public static final LoadConvertedInsertTabletStatementExceptionVisitor - STATEMENT_EXCEPTION_VISITOR = new LoadConvertedInsertTabletStatementExceptionVisitor(); + public LoadTsFileDataTypeConverter(final boolean isGeneratedByPipe) { + this.isGeneratedByPipe = isGeneratedByPipe; + } - public Optional convertForTableModel(LoadTsFile loadTsFileTableStatement) { + public Optional convertForTableModel(final LoadTsFile loadTsFileTableStatement) { try { return loadTsFileTableStatement.accept( tableStatementDataTypeConvertExecutionVisitor, loadTsFileTableStatement.getDatabase()); @@ -94,7 +77,22 @@ public Optional convertForTableModel(LoadTsFile loadTsFileTableStateme } } - public Optional convertForTreeModel(LoadTsFileStatement loadTsFileTreeStatement) { + private TSStatus executeForTableModel(final Statement statement, final String databaseName) { + return Coordinator.getInstance() + .executeForTableModel( + isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement, + relationalSqlParser, + SESSION_MANAGER.getCurrSession(), + SESSION_MANAGER.requestQueryId(), + SESSION_MANAGER.getSessionInfoOfPipeReceiver( + SESSION_MANAGER.getCurrSession(), databaseName), + "", + LocalExecutionPlanner.getInstance().metadata, + IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold()) + .status; + } + + public Optional convertForTreeModel(final LoadTsFileStatement loadTsFileTreeStatement) { try { return loadTsFileTreeStatement.accept(treeStatementDataTypeConvertExecutionVisitor, null); } catch (Exception e) { @@ -105,6 +103,20 @@ public Optional convertForTreeModel(LoadTsFileStatement loadTsFileTree } } + private TSStatus executeForTreeModel(final Statement statement) { + return Coordinator.getInstance() + .executeForTreeModel( + isGeneratedByPipe ? new PipeEnrichedStatement(statement) : statement, + SESSION_MANAGER.requestQueryId(), + SESSION_MANAGER.getSessionInfo(SESSION_MANAGER.getCurrSession()), + "", + ClusterPartitionFetcher.getInstance(), + ClusterSchemaFetcher.getInstance(), + IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold(), + false) + .status; + } + public boolean isSuccessful(final TSStatus status) { return status != null && (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java index 514795db23a3..2781350db5d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java @@ -28,6 +28,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,14 +96,16 @@ protected List generateSubscriptionEvents() throws Exception } final List events = new ArrayList<>(); - final List tsFiles = batch.sealTsFiles(); - final AtomicInteger referenceCount = new AtomicInteger(tsFiles.size()); - for (final File tsFile : tsFiles) { + final List> dbTsFilePairs = batch.sealTsFiles(); + final AtomicInteger referenceCount = new AtomicInteger(dbTsFilePairs.size()); + for (final Pair tsFile : dbTsFilePairs) { final SubscriptionCommitContext commitContext = prefetchingQueue.generateSubscriptionCommitContext(); events.add( new SubscriptionEvent( - new SubscriptionPipeTsFileBatchEvents(this, referenceCount), tsFile, commitContext)); + new SubscriptionPipeTsFileBatchEvents(this, referenceCount), + tsFile.right, + commitContext)); } return events; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java index a1cbaa0d03a1..e6f2d44a589f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeTabletEventSorterTest.java @@ -19,17 +19,24 @@ package org.apache.iotdb.db.pipe.connector; -import org.apache.iotdb.db.pipe.connector.util.PipeTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTableModelTabletEventSorter; +import org.apache.iotdb.db.pipe.connector.util.sorter.PipeTreeModelTabletEventSorter; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -46,7 +53,7 @@ private static boolean checkSorted(final Tablet tablet) { } @Test - public void testDeduplicateAndSort() { + public void testTreeModelDeduplicateAndSort() { List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); @@ -82,7 +89,7 @@ public void testDeduplicateAndSort() { Assert.assertFalse(checkSorted(tablet)); - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); Assert.assertTrue(checkSorted(tablet)); @@ -103,7 +110,7 @@ public void testDeduplicateAndSort() { } @Test - public void testDeduplicate() { + public void testTreeModelDeduplicate() { List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); @@ -127,7 +134,7 @@ public void testDeduplicate() { Assert.assertTrue(checkSorted(tablet)); - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); Assert.assertTrue(checkSorted(tablet)); @@ -148,7 +155,7 @@ public void testDeduplicate() { } @Test - public void testSort() { + public void testTreeModelSort() { List schemaList = new ArrayList<>(); schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); @@ -196,7 +203,7 @@ public void testSort() { } } - new PipeTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); + new PipeTreeModelTabletEventSorter(tablet).deduplicateAndSortTimestampsIfNecessary(); Assert.assertTrue(checkSorted(tablet)); @@ -215,4 +222,193 @@ public void testSort() { } } } + + @Test + public void testTableModelDeduplicateAndSort() { + doTableModelTest(true, true); + } + + @Test + public void testTableModelDeduplicate() { + doTableModelTest(true, false); + } + + @Test + public void testTableModelSort() { + doTableModelTest(false, true); + } + + @Test + public void testTableModelDeduplicateAndSort1() { + doTableModelTest1(true, true); + } + + @Test + public void testTableModelDeduplicate1() { + doTableModelTest1(true, false); + } + + @Test + public void testTableModelSort1() { + doTableModelTest1(false, true); + } + + public void doTableModelTest(final boolean hasDuplicates, final boolean isUnSorted) { + final Tablet tablet = generateTablet("test", 10, hasDuplicates, isUnSorted); + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByDevIdTimestamp(); + for (int i = 1; i < tablet.getRowSize(); i++) { + long time = tablet.timestamps[i]; + Assert.assertTrue(time > tablet.timestamps[i - 1]); + Assert.assertEquals( + tablet.getValue(i, 0), + new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(tablet.getValue(i, 1), (long) i); + Assert.assertEquals(tablet.getValue(i, 2), i * 1.0f); + Assert.assertEquals( + tablet.getValue(i, 3), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(tablet.getValue(i, 4), (long) i); + Assert.assertEquals(tablet.getValue(i, 5), i); + Assert.assertEquals(tablet.getValue(i, 6), i * 0.1); + Assert.assertEquals(tablet.getValue(i, 7), getDate(i)); + Assert.assertEquals( + tablet.getValue(i, 8), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + } + } + + public void doTableModelTest1(final boolean hasDuplicates, final boolean isUnSorted) { + final Tablet tablet = generateTablet("test", 10, hasDuplicates, isUnSorted); + new PipeTableModelTabletEventSorter(tablet).sortAndDeduplicateByTimestampIfNecessary(); + for (int i = 1; i < tablet.getRowSize(); i++) { + long time = tablet.timestamps[i]; + Assert.assertTrue(time > tablet.timestamps[i - 1]); + Assert.assertEquals( + tablet.getValue(i, 0), + new Binary(String.valueOf(i / 100).getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(tablet.getValue(i, 1), (long) i); + Assert.assertEquals(tablet.getValue(i, 2), i * 1.0f); + Assert.assertEquals( + tablet.getValue(i, 3), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + Assert.assertEquals(tablet.getValue(i, 4), (long) i); + Assert.assertEquals(tablet.getValue(i, 5), i); + Assert.assertEquals(tablet.getValue(i, 6), i * 0.1); + Assert.assertEquals(tablet.getValue(i, 7), getDate(i)); + Assert.assertEquals( + tablet.getValue(i, 8), new Binary(String.valueOf(i).getBytes(StandardCharsets.UTF_8))); + } + } + + private Tablet generateTablet( + final String tableName, + final int deviceIDNum, + final boolean hasDuplicates, + final boolean isUnSorted) { + final List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s0", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.FLOAT)); + schemaList.add(new MeasurementSchema("s3", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s4", TSDataType.TIMESTAMP)); + schemaList.add(new MeasurementSchema("s5", TSDataType.INT32)); + schemaList.add(new MeasurementSchema("s6", TSDataType.DOUBLE)); + schemaList.add(new MeasurementSchema("s7", TSDataType.DATE)); + schemaList.add(new MeasurementSchema("s8", TSDataType.TEXT)); + + final List columnTypes = + Arrays.asList( + Tablet.ColumnCategory.TAG, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD, + Tablet.ColumnCategory.FIELD); + Tablet tablet = + new Tablet( + tableName, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + deviceIDNum * 1000); + tablet.initBitMaps(); + + // s2 float, s3 string, s4 timestamp, s5 int32, s6 double, s7 date, s8 text + int rowIndex = 0; + + for (long row = 0; row < deviceIDNum; row++) { + for (int i = 0; i < (isUnSorted ? 50 : 100); i++) { + + final long value; + if (isUnSorted) { + value = (row + 1) * 100 - i - 1; + } else { + value = (row) * 100 + i; + } + for (int j = 0; j < 10; j++) { + tablet.addTimestamp(rowIndex, value); + tablet.addValue( + "s0", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s1", rowIndex, value); + tablet.addValue("s2", rowIndex, (value * 1.0f)); + tablet.addValue( + "s3", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (int) value); + tablet.addValue("s6", rowIndex, value * 0.1); + tablet.addValue("s7", rowIndex, getDate((int) value)); + tablet.addValue( + "s8", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + rowIndex++; + tablet.setRowSize(rowIndex); + if (!hasDuplicates) { + break; + } + } + } + } + if (!isUnSorted) { + return tablet; + } + for (long row = 0; row < deviceIDNum; row++) { + for (int i = 50; i < 100; i++) { + + final long value; + value = (row + 1) * 100 - i - 1; + + for (int j = 0; j < 10; j++) { + tablet.addTimestamp(rowIndex, value); + tablet.addValue( + "s0", rowIndex, new Binary(String.valueOf(row).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s1", rowIndex, value); + tablet.addValue("s2", rowIndex, (value * 1.0f)); + tablet.addValue( + "s3", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + tablet.addValue("s4", rowIndex, value); + tablet.addValue("s5", rowIndex, (int) value); + tablet.addValue("s6", rowIndex, value * 0.1); + tablet.addValue("s7", rowIndex, getDate((int) value)); + tablet.addValue( + "s8", rowIndex, new Binary(String.valueOf(value).getBytes(StandardCharsets.UTF_8))); + rowIndex++; + tablet.setRowSize(rowIndex); + if (!hasDuplicates) { + break; + } + } + } + } + return tablet; + } + + public LocalDate getDate(final int value) { + Date date = new Date(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + return DateUtils.parseIntToLocalDate( + DateUtils.parseDateExpressionToInt(dateFormat.format(date))); + } catch (Exception e) { + return DateUtils.parseIntToLocalDate(DateUtils.parseDateExpressionToInt("1970-01-01")); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java index 0e1c0ad8f999..52321265280b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SubqueryTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.JoinNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import com.google.common.collect.ImmutableList; @@ -45,9 +46,12 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.filter; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.join; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.semiJoin; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.sort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; @@ -224,4 +228,98 @@ public void testUncorrelatedScalarSubqueryInWhereClauseWithEnforceSingleRowNode( JoinNode.JoinType.INNER, builder -> builder.left(tableScan1).right(enforceSingleRow(any()))))))); } + + @Test + public void testUncorrelatedInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new SymbolReference("expr"); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + + // Verify DistributionPlan + assertPlan( + planTester.getFragmentPlan(0), + output( + project( + filter( + filterPredicate, + semiJoin( + "s1", + "s1_6", + "expr", + mergeSort(exchange(), sort(tableScan1), exchange()), + mergeSort(exchange(), sort(tableScan2), exchange())))))); + + assertPlan(planTester.getFragmentPlan(1), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(2), sort(tableScan1)); + + assertPlan(planTester.getFragmentPlan(3), sort(tableScan2)); + + assertPlan(planTester.getFragmentPlan(4), sort(tableScan2)); + } + + @Test + public void testUncorrelatedNotInPredicateSubquery() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT s1 FROM table1 where s1 not in (select s1 from table1)"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + Expression filterPredicate = new NotExpression(new SymbolReference("expr")); + + PlanMatchPattern tableScan1 = + tableScan("testdb.table1", ImmutableList.of("s1"), ImmutableSet.of("s1")); + + PlanMatchPattern tableScan2 = tableScan("testdb.table1", ImmutableMap.of("s1_6", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──FilterNode + * └──SemiJoinNode + * |──SortNode + * | └──TableScanNode + * ├──SortNode + * │ └──TableScanNode + + */ + assertPlan( + logicalQueryPlan, + output( + project( + filter( + filterPredicate, + semiJoin("s1", "s1_6", "expr", sort(tableScan1), sort(tableScan2)))))); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 40e6abba5184..5c2ce8e2f806 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; @@ -461,6 +462,16 @@ public static PlanMatchPattern join( return builder.build(); } + public static PlanMatchPattern semiJoin( + String sourceSymbolAlias, + String filteringSymbolAlias, + String outputAlias, + PlanMatchPattern source, + PlanMatchPattern filtering) { + return node(SemiJoinNode.class, source, filtering) + .with(new SemiJoinMatcher(sourceSymbolAlias, filteringSymbolAlias, outputAlias)); + } + public static PlanMatchPattern streamSort(PlanMatchPattern source) { return node(StreamSortNode.class, source); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java new file mode 100644 index 000000000000..7ee9192254e7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/SemiJoinMatcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.assertions; + +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SemiJoinNode; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.NO_MATCH; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.MatchResult.match; + +final class SemiJoinMatcher implements Matcher { + private final String sourceSymbolAlias; + private final String filteringSymbolAlias; + private final String outputAlias; + + SemiJoinMatcher(String sourceSymbolAlias, String filteringSymbolAlias, String outputAlias) { + this.sourceSymbolAlias = requireNonNull(sourceSymbolAlias, "sourceSymbolAlias is null"); + this.filteringSymbolAlias = + requireNonNull(filteringSymbolAlias, "filteringSymbolAlias is null"); + this.outputAlias = requireNonNull(outputAlias, "outputAlias is null"); + } + + @Override + public boolean shapeMatches(PlanNode node) { + return node instanceof SemiJoinNode; + } + + @Override + public MatchResult detailMatches( + PlanNode node, SessionInfo sessionInfo, Metadata metadata, SymbolAliases symbolAliases) { + checkState( + shapeMatches(node), + "Plan testing framework error: shapeMatches returned false in detailMatches in %s", + this.getClass().getName()); + + SemiJoinNode semiJoinNode = (SemiJoinNode) node; + if (!(symbolAliases + .get(sourceSymbolAlias) + .equals(semiJoinNode.getSourceJoinSymbol().toSymbolReference()) + && symbolAliases + .get(filteringSymbolAlias) + .equals(semiJoinNode.getFilteringSourceJoinSymbol().toSymbolReference()))) { + return NO_MATCH; + } + + return match(outputAlias, semiJoinNode.getSemiJoinOutput().toSymbolReference()); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("filteringSymbolAlias", filteringSymbolAlias) + .add("sourceSymbolAlias", sourceSymbolAlias) + .add("outputAlias", outputAlias) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java index 6a86b3955f93..2cb60ed22562 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java @@ -146,7 +146,7 @@ public void testInsertionCompactionWithCachedDeviceInfoAndUnclosedResource() Phaser phaser = new Phaser(1); int submitTaskNum = - CompactionScheduler.scheduleInsertionCompaction(tsFileManager, 0, phaser, context); + CompactionScheduler.tryToSubmitInsertionCompactionTask(tsFileManager, 0, phaser, context); Assert.assertEquals(1, submitTaskNum); // perform insertion compaction phaser.awaitAdvanceInterruptibly(phaser.arrive()); @@ -169,7 +169,7 @@ public void testInsertionCompactionWithCachedDeviceInfoAndUnclosedResource() // unseq resource2 d2[10, 20] submitTaskNum = - CompactionScheduler.scheduleInsertionCompaction(tsFileManager, 0, phaser, context); + CompactionScheduler.tryToSubmitInsertionCompactionTask(tsFileManager, 0, phaser, context); Assert.assertEquals(0, submitTaskNum); Assert.assertTrue( TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(tsFileManager.getTsFileList(true))); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java index ddcb90f3ac8a..0e0516a46c06 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.cross; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.service.metrics.FileMetrics; @@ -452,7 +453,7 @@ public void testInsertionCompactionScheduleWithEmptySeqSpace2() } @Test - public void testInsertionCompactionScheduleWithMultiTimePartitions() + public void testInsertionCompactionScheduleWithMultiTimePartitions1() throws IOException, InterruptedException { TsFileResource unseqResource1 = generateSingleNonAlignedSeriesFileWithDevices( @@ -504,6 +505,60 @@ public void testInsertionCompactionScheduleWithMultiTimePartitions() TsFileResourceManager.getInstance().getPriorityQueueSize()); } + @Test + public void testInsertionCompactionScheduleWithMultiTimePartitions2() + throws IOException, InterruptedException { + IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + int innerCompactionCandidateFileNum = config.getInnerCompactionCandidateFileNum(); + config.setInnerCompactionCandidateFileNum(2); + try { + TsFileResource unseqResource1 = + generateSingleNonAlignedSeriesFileWithDevices( + "2-2-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new TimeRange(1, 4)}, false); + unseqResource1.setStatusForTest(TsFileResourceStatus.NORMAL); + + TsFileResource unseqResource2 = + generateSingleNonAlignedSeriesFileWithDevices( + "3-3-0-0.tsfile", new String[] {"d1"}, new TimeRange[] {new TimeRange(6, 9)}, false); + unseqResource2.setStatusForTest(TsFileResourceStatus.NORMAL); + createTimePartitionDirIfNotExist(2808L); + TsFileResource unseqResource3 = + generateSingleNonAlignedSeriesFileWithDevicesWithTimePartition( + "4-4-0-0.tsfile", + new String[] {"d1"}, + new TimeRange[] {new TimeRange(1698301490305L, 1698301490405L)}, + 2808L, + true); + TsFileResource unseqResource4 = + generateSingleNonAlignedSeriesFileWithDevicesWithTimePartition( + "5-5-0-0.tsfile", + new String[] {"d1"}, + new TimeRange[] {new TimeRange(1698301490306L, 1698301490406L)}, + 2808L, + true); + unseqResource3.setStatusForTest(TsFileResourceStatus.NORMAL); + unseqResources.add(unseqResource1); + unseqResources.add(unseqResource2); + seqResources.add(unseqResource3); + seqResources.add(unseqResource4); + + DataRegionForCompactionTest dataRegion = createDataRegion(); + TsFileManager tsFileManager = dataRegion.getTsFileManager(); + TsFileResourceManager.getInstance().registerSealedTsFileResource(unseqResource1); + TsFileResourceManager.getInstance().registerSealedTsFileResource(unseqResource2); + TsFileResourceManager.getInstance().registerSealedTsFileResource(unseqResource3); + TsFileResourceManager.getInstance().registerSealedTsFileResource(unseqResource4); + tsFileManager.getOrCreateUnsequenceListByTimePartition(0).keepOrderInsert(unseqResource1); + tsFileManager.getOrCreateUnsequenceListByTimePartition(0).keepOrderInsert(unseqResource2); + tsFileManager.getOrCreateSequenceListByTimePartition(2808).keepOrderInsert(unseqResource3); + tsFileManager.getOrCreateSequenceListByTimePartition(2808).keepOrderInsert(unseqResource4); + // 2 insertion task + 1 inner task + Assert.assertEquals(3, dataRegion.executeCompaction()); + } finally { + config.setInnerCompactionCandidateFileNum(innerCompactionCandidateFileNum); + } + } + @Test public void testInsertionCompactionUpdateFileMetrics() throws IOException { TsFileResource unseqResource1 = @@ -606,9 +661,10 @@ public DataRegionForCompactionTest(String databaseName, String id) { } public int executeInsertionCompaction() throws InterruptedException { - return super.executeInsertionCompaction( - new ArrayList<>(this.getTsFileManager().getTimePartitions()), - new CompactionScheduleContext()); + CompactionScheduleContext context = new CompactionScheduleContext(); + super.executeInsertionCompaction( + new ArrayList<>(this.getTsFileManager().getTimePartitions()), context); + return context.getSubmitCompactionTaskNum(); } } } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index a19e12713223..0f5d5d2060ba 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1457,10 +1457,10 @@ delete_wal_files_period_in_ms=20000 # The minimum size of wal files when throttle down in IoTConsensus # If this value is not set, it will be carefully chosen according to the available disk space. -# If this value is set smaller than 0, it will default to 50 * 1024 * 1024 * 1024 bytes (50GB). +# If this value is set smaller than 0, it will default to 200 * 1024 * 1024 * 1024 bytes (200GB). # effectiveMode: hot_reload # Datatype: long -wal_throttle_threshold_in_byte=53687091200 +wal_throttle_threshold_in_byte=214748364800 # Maximum wait time of write cache in IoTConsensus # If this value is less than or equal to 0, use the default value 10 * 1000 ms (10s) @@ -1504,7 +1504,7 @@ data_region_iot_max_memory_ratio_for_queue = 0.6 # values less than or equal to 0 means no limit # effectiveMode: hot_reload # Datatype: long -region_migration_speed_limit_bytes_per_second = 33554432 +region_migration_speed_limit_bytes_per_second = 50331648 #################### ### Blob Allocator Configuration diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java index 9fcda36998e8..0a9fa784e9f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/ColumnNotExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/ColumnNotExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java index d28c04e9e7ec..d0baa216b84f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableAlreadyExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableAlreadyExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java index 616a20d49bc3..c0caabe711a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/table/TableNotExistsException.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/table/TableNotExistsException.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.exception.metadata.table; +package org.apache.iotdb.commons.exception.table; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.rpc.TSStatusCode; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 997278010e9f..033fe6505824 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -19,6 +19,10 @@ package org.apache.iotdb.commons.pipe.agent.task.meta; +import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + import org.apache.tsfile.utils.PublicBAOS; import java.io.DataOutputStream; @@ -26,6 +30,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; public class PipeMeta { @@ -61,6 +66,41 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } + public boolean visibleUnder(final boolean isTableModel) { + final PipeParameters extractorParameters = getStaticMeta().getExtractorParameters(); + + // visible under all model when 'mode.double-living' is set to true + final boolean isDoubleLiving = + extractorParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + return true; + } + + final boolean isTreeDialect = + extractorParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); + final Boolean _isCaptureTree = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + final boolean isCaptureTree = Objects.nonNull(_isCaptureTree) ? _isCaptureTree : isTreeDialect; + final Boolean _isCaptureTable = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + final boolean isCaptureTable = + Objects.nonNull(_isCaptureTable) ? _isCaptureTable : !isTreeDialect; + + // visible under specific tree or table model <-> actually capture tree or table data + return isTableModel ? isCaptureTable : isCaptureTree; + } + public ByteBuffer serialize() throws IOException { final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index f8a8c9e55ac9..4009288dfc70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -85,6 +85,14 @@ public boolean containsPipeMeta(String pipeName) { return pipeNameToPipeMetaMap.containsKey(pipeName); } + public boolean containsPipeMeta(String pipeName, boolean isTableModel) { + final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); + if (Objects.isNull(pipeMeta)) { + return false; + } + return pipeMeta.visibleUnder(isTableModel); + } + public Iterable getPipeMetaList() { return pipeNameToPipeMetaMap.values(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index 1335b995a593..d5d34c269206 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -119,6 +119,9 @@ public class PipeExtractorConstant { public static final String EXTRACTOR_MODE_SNAPSHOT_KEY = "extractor.mode.snapshot"; public static final String SOURCE_MODE_SNAPSHOT_KEY = "source.mode.snapshot"; public static final boolean EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE = false; + public static final String EXTRACTOR_MODE_DOUBLE_LIVING_KEY = "extractor.mode.double-living"; + public static final String SOURCE_MODE_DOUBLE_LIVING_KEY = "source.mode.double-living"; + public static final boolean EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE = false; public static final String EXTRACTOR_START_TIME_KEY = "extractor.start-time"; public static final String SOURCE_START_TIME_KEY = "source.start-time"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java index 963595c62a62..00462cb5eb46 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java @@ -102,15 +102,22 @@ public String getTablePattern() { */ public static TablePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTableModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - !sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + final boolean isTableModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + !sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String databaseNamePattern = sourceParameters.getStringOrDefault( Arrays.asList( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 7ee8f6dd88db..f03f0a291868 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -70,15 +70,23 @@ public boolean isRoot() { */ public static TreePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTreeModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + + final boolean isTreeModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String path = sourceParameters.getStringByKeys(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java index 2d3a1615f00e..cecee61c7fae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java @@ -26,8 +26,10 @@ import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import java.util.Arrays; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE; @@ -85,6 +87,51 @@ public void validate(final PipeParameterValidator validator) throws Exception { .getStringOrDefault( Arrays.asList(EXTRACTOR_EXCLUSION_KEY, SOURCE_EXCLUSION_KEY), EXTRACTOR_EXCLUSION_DEFAULT_VALUE)); + + // Validate double living + validateDoubleLiving(validator.getParameters()); + } + + private void validateDoubleLiving(final PipeParameters parameters) { + final boolean isDoubleLiving = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (!isDoubleLiving) { + return; + } + + // check 'capture.tree' + final Boolean isCaptureTree = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + if (Objects.nonNull(isCaptureTree) && !isCaptureTree) { + throw new PipeParameterNotValidException( + "capture.tree can not be specified to false when double living is enabled"); + } + + // check 'capture.table' + final Boolean isCaptureTable = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + if (Objects.nonNull(isCaptureTable) && !isCaptureTable) { + throw new PipeParameterNotValidException( + "capture.table can not be specified to false when double living is enabled"); + } + + // check 'forwarding-pipe-requests' + final Boolean isForwardingPipeRequests = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY); + if (Objects.nonNull(isForwardingPipeRequests) && isForwardingPipeRequests) { + throw new PipeParameterNotValidException( + "forwarding-pipe-requests can not be specified to true when double living is enabled"); + } } @Override @@ -99,12 +146,22 @@ public void customize( taskID = pipeName + "_" + regionId + "_" + creationTime; pipeTaskMeta = environment.getPipeTaskMeta(); - isForwardingPipeRequests = + final boolean isDoubleLiving = parameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + isForwardingPipeRequests = false; + } else { + isForwardingPipeRequests = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + } } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java index 23e89ddb7d7b..336670f593f9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferHandshakeV1Req; import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferHandshakeV2Req; import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -112,7 +113,11 @@ protected TPipeTransferResp handleTransferHandshakeV1(final PipeTransferHandshak if (receiverFileDirWithIdSuffix.get() != null) { if (receiverFileDirWithIdSuffix.get().exists()) { try { - FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + return null; + }); LOGGER.info( "Receiver id = {}: Original receiver file dir {} was deleted.", receiverId.get(), @@ -440,7 +445,7 @@ private void deleteCurrentWritingFile() { private void deleteFile(final File file) { if (file.exists()) { try { - FileUtils.delete(file); + RetryUtils.retryOnException(() -> FileUtils.delete(file)); LOGGER.info( "Receiver id = {}: Original writing file {} was deleted.", receiverId.get(), @@ -749,7 +754,7 @@ public synchronized void handleExit() { if (writingFile != null) { try { - FileUtils.delete(writingFile); + RetryUtils.retryOnException(() -> FileUtils.delete(writingFile)); LOGGER.info( "Receiver id = {}: Handling exit: Writing file {} was deleted.", receiverId.get(), @@ -774,7 +779,11 @@ public synchronized void handleExit() { if (receiverFileDirWithIdSuffix.get() != null) { if (receiverFileDirWithIdSuffix.get().exists()) { try { - FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(receiverFileDirWithIdSuffix.get()); + return null; + }); LOGGER.info( "Receiver id = {}: Handling exit: Original receiver file dir {} was deleted.", receiverId.get(), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBReceiverAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBReceiverAgent.java index c09812f20036..7aaa66a8b752 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBReceiverAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBReceiverAgent.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.pipe.receiver; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -114,7 +115,11 @@ public final void handleClientExit(final String key) { public static void cleanPipeReceiverDir(final File receiverFileDir) { try { - FileUtils.deleteDirectory(receiverFileDir); + RetryUtils.retryOnException( + () -> { + FileUtils.deleteDirectory(receiverFileDir); + return null; + }); LOGGER.info("Clean pipe receiver dir {} successfully.", receiverFileDir); } catch (final Exception e) { LOGGER.warn("Clean pipe receiver dir {} failed.", receiverFileDir, e); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/snapshot/PipeSnapshotResourceManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/snapshot/PipeSnapshotResourceManager.java index 6b134d172056..24152a14da91 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/snapshot/PipeSnapshotResourceManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/resource/snapshot/PipeSnapshotResourceManager.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.pipe.resource.snapshot; import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.slf4j.Logger; @@ -75,7 +76,11 @@ public String increaseSnapshotReference(String snapshotPath) throws IOException } // Otherwise, copy the snapshot to pipe dir - FileUtils.copyFile(new File(snapshotPath), new File(copiedFilePath)); + RetryUtils.retryOnException( + () -> { + FileUtils.copyFile(new File(snapshotPath), new File(copiedFilePath)); + return null; + }); copiedSnapshotPath2ReferenceCountMap.put(copiedFilePath, new AtomicLong(1)); return copiedFilePath; } finally { @@ -140,7 +145,7 @@ public void decreaseSnapshotReference(String snapshotPath) { final long count = referenceCount.decrementAndGet(); if (count == 0) { copiedSnapshotPath2ReferenceCountMap.remove(snapshotPath); - FileUtils.deleteFileOrDirectory(new File(snapshotPath)); + FileUtils.deleteFileOrDirectoryWithRetry(new File(snapshotPath)); } } finally { lock.unlock(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index 99a7b4e75899..fbc0621caaae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -29,6 +29,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -41,7 +42,10 @@ import java.nio.file.StandardCopyOption; import java.text.CharacterIterator; import java.text.StringCharacterIterator; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Stack; public class FileUtils { private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); @@ -50,6 +54,29 @@ public class FileUtils { private FileUtils() {} + public static List listFilesRecursively(File dir, FileFilter fileFilter) { + List result = new ArrayList<>(); + Stack stack = new Stack<>(); + if (dir.exists()) { + stack.push(dir); + } + while (!stack.isEmpty()) { + File file = stack.pop(); + if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + stack.push(f); + } + } + } + if (fileFilter.accept(file)) { + result.add(file); + } + } + return result; + } + public static boolean deleteFileIfExist(File file) { try { Files.deleteIfExists(file.toPath()); @@ -86,6 +113,28 @@ public static void deleteFileOrDirectory(File file, boolean quietForNoSuchFile) } } + public static void deleteFileOrDirectoryWithRetry(File file) { + if (file.isDirectory()) { + File[] files = file.listFiles(); + if (files != null) { + for (File subfile : files) { + deleteFileOrDirectoryWithRetry(subfile); + } + } + } + try { + RetryUtils.retryOnException( + () -> { + Files.delete(file.toPath()); + return null; + }); + } catch (DirectoryNotEmptyException e) { + LOGGER.warn("{}: {}", e.getMessage(), Arrays.toString(file.list()), e); + } catch (Exception e) { + LOGGER.warn("{}: {}", e.getMessage(), file.getName(), e); + } + } + public static void deleteDirectoryAndEmptyParent(File folder) { deleteFileOrDirectory(folder); final File parentFolder = folder.getParentFile(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/RetryUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/RetryUtils.java new file mode 100644 index 000000000000..19a6456ec30a --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/RetryUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.utils; + +public class RetryUtils { + + public interface CallableWithException { + T call() throws E; + } + + public static final int MAX_RETRIES = 3; + + public static T retryOnException( + final CallableWithException callable) throws E { + int attempt = 0; + while (true) { + try { + return callable.call(); + } catch (Exception e) { + attempt++; + if (attempt >= MAX_RETRIES) { + throw e; + } + } + } + } + + private RetryUtils() { + // utility class + } +} diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 7d8d22a0a3aa..ebe673995523 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -751,11 +751,23 @@ struct TAlterPipeReq { 6: optional map extractorAttributes 7: optional bool isReplaceAllExtractorAttributes 8: optional bool ifExistsCondition + 9: optional bool isTableModel +} + +struct TStartPipeReq { + 1: required string pipeName + 2: optional bool isTableModel +} + +struct TStopPipeReq { + 1: required string pipeName + 2: optional bool isTableModel } struct TDropPipeReq { 1: required string pipeName 2: optional bool ifExistsCondition + 3: optional bool isTableModel } // Deprecated, restored for compatibility @@ -768,6 +780,7 @@ struct TPipeSinkInfo { struct TShowPipeReq { 1: optional string pipeName 2: optional bool whereClause + 3: optional bool isTableModel } struct TShowPipeResp { @@ -1071,12 +1084,27 @@ struct TShowTableResp { 2: optional list tableInfoList } +struct TShowTable4InformationSchemaResp { + 1: required common.TSStatus status + 2: optional map> databaseTableInfoMap +} + struct TDescTableResp { 1: required common.TSStatus status 2: optional binary tableInfo 3: optional set preDeletedColumns } +struct TDescTable4InformationSchemaResp { + 1: required common.TSStatus status + 2: optional map> tableColumnInfoMap +} + +struct TTableColumnInfo { + 1: required binary tableInfo + 2: optional set preDeletedColumns +} + struct TFetchTableResp { 1: required common.TSStatus status 2: optional binary tableInfoMap @@ -1665,9 +1693,15 @@ service IConfigNodeRPCService { /** Start Pipe */ common.TSStatus startPipe(string pipeName) + /** Start Pipe */ + common.TSStatus startPipeExtended(TStartPipeReq req) + /** Stop Pipe */ common.TSStatus stopPipe(string pipeName) + /** Stop Pipe */ + common.TSStatus stopPipeExtended(TStopPipeReq req) + /** Drop Pipe */ common.TSStatus dropPipe(string pipeName) @@ -1825,8 +1859,12 @@ service IConfigNodeRPCService { TShowTableResp showTables(string database, bool isDetails) + TShowTable4InformationSchemaResp showTables4InformationSchema() + TDescTableResp describeTable(string database, string tableName, bool isDetails) + TDescTable4InformationSchemaResp descTables4InformationSchema() + TFetchTableResp fetchTables(map> fetchTableMap) TDeleteTableDeviceResp deleteDevice(TDeleteTableDeviceReq req)