diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java new file mode 100644 index 000000000000..07cdb16b34a8 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java @@ -0,0 +1,310 @@ +package org.apache.iotdb.tool.common; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.HashMap; +import java.util.Map; + +public class Constants { + + // common + public static final int CODE_OK = 0; + public static final int CODE_ERROR = 1; + + public static final String HOST_ARGS = "h"; + public static final String HOST_NAME = "host"; + public static final String HOST_DESC = "Host Name (optional)"; + public static final String HOST_DEFAULT_VALUE = "127.0.0.1"; + + public static final String HELP_ARGS = "help"; + public static final String HELP_DESC = "Display help information"; + + public static final String PORT_ARGS = "p"; + public static final String PORT_NAME = "port"; + public static final String PORT_DESC = "Port (optional)"; + public static final String PORT_DEFAULT_VALUE = "6667"; + + public static final String PW_ARGS = "pw"; + public static final String PW_NAME = "password"; + public static final String PW_DESC = "Password (optional)"; + public static final String PW_DEFAULT_VALUE = "root"; + + public static final String USERNAME_ARGS = "u"; + public static final String USERNAME_NAME = "username"; + public static final String USERNAME_DESC = "Username (optional)"; + public static final String USERNAME_DEFAULT_VALUE = "root"; + + public static final String FILE_TYPE_ARGS = "ft"; + public static final String FILE_TYPE_NAME = "file_type"; + public static final String FILE_TYPE_ARGS_NAME = "format"; + public static final String FILE_TYPE_DESC = + "File type?You can choose tsfile)、csv) or sql).(required)"; + public static final String FILE_TYPE_DESC_EXPORT = + "Export file type ?You can choose tsfile)、csv) or sql).(required)"; + public static final String FILE_TYPE_DESC_IMPORT = + "Types of imported files: csv, sql, tsfile.(required)"; + + public static final String TIME_FORMAT_ARGS = "tf"; + public static final String TIME_FORMAT_NAME = "time_format"; + public static final String TIME_FORMAT_DESC = + "Output time Format in csv file. " + + "You can choose 1) timestamp, number, long 2) ISO8601, default 3) " + + "user-defined pattern like yyyy-MM-dd HH:mm:ss, default ISO8601.\n OutPut timestamp in sql file, No matter what time format is set(optional)"; + + public static final String TIME_ZONE_ARGS = "tz"; + public static final String TIME_ZONE_NAME = "timezone"; + public static final String TIME_ZONE_DESC = "Time Zone eg. +08:00 or -01:00 .(optional)"; + + public static final String TIMEOUT_ARGS = "timeout"; + public static final String TIMEOUT_NAME = "query_timeout"; + public static final String TIMEOUT_DESC = "Timeout for session query.(optional)"; + + public static final String ALIGNED_ARGS = "aligned"; + public static final String ALIGNED_NAME = "use_aligned"; + public static final String ALIGNED_ARGS_NAME_EXPORT = "export aligned insert sql"; + public static final String ALIGNED_ARGS_NAME_IMPORT = "use the aligned interface"; + public static final String ALIGNED_EXPORT_DESC = "Whether export to sql of aligned.(optional)"; + public static final String ALIGNED_IMPORT_DESC = + "Whether to use the interface of aligned.(optional)"; + + public static final String SQL_DIALECT_ARGS = "sql_dialect"; + public static final String SQL_DIALECT_DESC = + "Currently supports tree and table model, default tree. (optional)"; + public static final String SQL_DIALECT_VALUE_TREE = "tree"; + public static final String SQL_DIALECT_VALUE_TABLE = "table"; + + public static final String DB_ARGS = "db"; + public static final String DB_NAME = "database"; + public static final String DB_DESC = + "The database to be exported,Only takes effect when sql_dialect is of type table.(optional)"; + + public static final String TABLE_ARGS = "table"; + public static final String TABLE_DESC = + "The table to be exported,only takes effect when sql_dialect is of type table"; + public static final String TABLE_DESC_EXPORT = + TABLE_DESC + + ".If the '- q' parameter is specified, this parameter does not take effect. If the export type is tsfile or sql, this parameter is required. (optional)"; + public static final String TABLE_DESC_IMPORT = TABLE_DESC + " and file_type is csv. (optional)"; + + public static final String DATATYPE_BOOLEAN = "boolean"; + public static final String DATATYPE_INT = "int"; + public static final String DATATYPE_LONG = "long"; + public static final String DATATYPE_FLOAT = "float"; + public static final String DATATYPE_DOUBLE = "double"; + public static final String DATATYPE_TIMESTAMP = "timestamp"; + public static final String DATATYPE_DATE = "date"; + public static final String DATATYPE_BLOB = "blob"; + public static final String DATATYPE_NAN = "NaN"; + public static final String DATATYPE_TEXT = "text"; + public static final String DATATYPE_STRING = "string"; + public static final String DATATYPE_NULL = "null"; + public static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); + + static { + TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); + TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT); + TYPE_INFER_KEY_DICT.put(DATATYPE_LONG, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); + TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); + TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); + TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); + TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE); + TYPE_INFER_KEY_DICT.put(DATATYPE_STRING, TSDataType.STRING); + } + + public static final Map TYPE_INFER_VALUE_DICT = new HashMap<>(); + + static { + TYPE_INFER_VALUE_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); + TYPE_INFER_VALUE_DICT.put(DATATYPE_INT, TSDataType.INT32); + TYPE_INFER_VALUE_DICT.put(DATATYPE_LONG, TSDataType.INT64); + TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); + TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); + TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.DATE); + TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); + TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT); + TYPE_INFER_VALUE_DICT.put(DATATYPE_STRING, TSDataType.STRING); + } + + public static final int MAX_HELP_CONSOLE_WIDTH = 92; + + public static final String CSV_SUFFIXS = "csv"; + public static final String TXT_SUFFIXS = "txt"; + public static final String SQL_SUFFIXS = "sql"; + public static final String TSFILE_SUFFIXS = "tsfile"; + + public static final String TSFILEDB_CLI_DIVIDE = "-------------------"; + public static final String COLON = ": "; + public static final String MINUS = "-"; + public static String timestampPrecision = "ms"; + public static Boolean aligned = false; + public static String fileType = null; + + // export constants + public static final String EXPORT_CLI_PREFIX = "Export Data"; + + public static final String EXPORT_CLI_HEAD = + "Please obtain help information for the corresponding data type based on different parameters, for example:\n" + + "./export_data.sh -help tsfile\n" + + "./export_data.sh -help sql\n" + + "./export_data.sh -help csv"; + + public static final String START_TIME_ARGS = "start_time"; + public static final String START_TIME_DESC = "The start time to be exported (optional)"; + + public static final String END_TIME_ARGS = "end_time"; + public static final String END_TIME_DESC = "The end time to be exported. (optional)"; + + public static final String TARGET_DIR_ARGS = "t"; + public static final String TARGET_DIR_NAME = "target"; + public static final String TARGET_DIR_ARGS_NAME = "target_directory"; + public static final String TARGET_DIR_DESC = "Target file directory (required)"; + + public static final String QUERY_COMMAND_ARGS = "q"; + public static final String QUERY_COMMAND_NAME = "query"; + public static final String QUERY_COMMAND_ARGS_NAME = "query_command"; + public static final String QUERY_COMMAND_DESC = + "The query command that you want to execute.If sql-dialect is of type table The 'q' parameter is only applicable to export types of CSV, and is not available for other types.If the '- q' parameter is not empty, then the parameters' creatTime ',' EndTime 'and' table 'are not effective.(optional)"; + + public static final String TARGET_FILE_ARGS = "pfn"; + public static final String TARGET_FILE_NAME = "prefix_file_name"; + public static final String TARGET_FILE_DESC = "Export file name .(optional)"; + + public static final String DATA_TYPE_ARGS = "dt"; + public static final String DATA_TYPE_NAME = "datatype"; + public static final String DATA_TYPE_DESC = + "Will the data type of timeseries be printed in the head line of the CSV file?" + + '\n' + + "You can choose true) or false) . (optional)"; + + public static final String LINES_PER_FILE_ARGS = "lpf"; + public static final String LINES_PER_FILE_NAME = "lines_per_file"; + public static final String LINES_PER_FILE_DESC = "Lines per dump file.(optional)"; + + public static final String DUMP_FILE_NAME_DEFAULT = "dump"; + public static String targetFile = DUMP_FILE_NAME_DEFAULT; + + public static final String queryTableParamRequired = + "Either '-q' or '-table' is required when 'sql-dialect' is' table '"; + public static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; + public static final String TARGET_DATABASE_NOT_EXIST_MSG = + "The target database %s does not exist"; + public static final String TARGET_TABLE_NOT_EXIST_MSG = "The target table %s does not exist"; + + public static final String[] TIME_FORMAT = + new String[] {"default", "long", "number", "timestamp"}; + + public static final long memoryThreshold = 10 * 1024 * 1024; + + public static final String[] STRING_TIME_FORMAT = + new String[] { + "yyyy-MM-dd HH:mm:ss.SSSX", + "yyyy/MM/dd HH:mm:ss.SSSX", + "yyyy.MM.dd HH:mm:ss.SSSX", + "yyyy-MM-dd HH:mm:ssX", + "yyyy/MM/dd HH:mm:ssX", + "yyyy.MM.dd HH:mm:ssX", + "yyyy-MM-dd HH:mm:ss.SSSz", + "yyyy/MM/dd HH:mm:ss.SSSz", + "yyyy.MM.dd HH:mm:ss.SSSz", + "yyyy-MM-dd HH:mm:ssz", + "yyyy/MM/dd HH:mm:ssz", + "yyyy.MM.dd HH:mm:ssz", + "yyyy-MM-dd HH:mm:ss.SSS", + "yyyy/MM/dd HH:mm:ss.SSS", + "yyyy.MM.dd HH:mm:ss.SSS", + "yyyy-MM-dd HH:mm:ss", + "yyyy/MM/dd HH:mm:ss", + "yyyy.MM.dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ss.SSSX", + "yyyy/MM/dd'T'HH:mm:ss.SSSX", + "yyyy.MM.dd'T'HH:mm:ss.SSSX", + "yyyy-MM-dd'T'HH:mm:ssX", + "yyyy/MM/dd'T'HH:mm:ssX", + "yyyy.MM.dd'T'HH:mm:ssX", + "yyyy-MM-dd'T'HH:mm:ss.SSSz", + "yyyy/MM/dd'T'HH:mm:ss.SSSz", + "yyyy.MM.dd'T'HH:mm:ss.SSSz", + "yyyy-MM-dd'T'HH:mm:ssz", + "yyyy/MM/dd'T'HH:mm:ssz", + "yyyy.MM.dd'T'HH:mm:ssz", + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy/MM/dd'T'HH:mm:ss.SSS", + "yyyy.MM.dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss", + "yyyy/MM/dd'T'HH:mm:ss", + "yyyy.MM.dd'T'HH:mm:ss" + }; + + // import constants + public static final String IMPORT_CLI_PREFIX = "Import Data"; + + public static final String IMPORT_CLI_HEAD = + "Please obtain help information for the corresponding data type based on different parameters, for example:\n" + + "./import_data.sh -help tsfile\n" + + "./import_data.sh -help sql\n" + + "./import_data.sh -help csv"; + + public static final String FILE_ARGS = "s"; + public static final String FILE_NAME = "source"; + public static final String FILE_DESC = + "The local directory path of the script file (folder) to be loaded. (required)"; + + public static final String ON_SUCCESS_ARGS = "os"; + public static final String ON_SUCCESS_NAME = "on_success"; + public static final String ON_SUCCESS_DESC = + "When loading tsfile successfully, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)"; + + public static final String SUCCESS_DIR_ARGS = "sd"; + public static final String SUCCESS_DIR_NAME = "success_dir"; + public static final String SUCCESS_DIR_DESC = + "The target folder when 'os' is 'mv' or 'cp'.(optional)"; + + public static final String FAIL_DIR_ARGS = "fd"; + public static final String FAIL_DIR_NAME = "fail_dir"; + public static final String FAIL_DIR_CSV_DESC = + "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH.(optional)"; + public static final String FAIL_DIR_SQL_DESC = + "Specifying a directory to save failed file, default YOUR_SQL_FILE_PATH.(optional)"; + public static final String FAIL_DIR_TSFILE_DESC = + "The target folder when 'of' is 'mv' or 'cp'.(optional)"; + + public static final String ON_FAIL_ARGS = "of"; + public static final String ON_FAIL_NAME = "on_fail"; + public static final String ON_FAIL_DESC = + "When loading tsfile fail, do operation on tsfile (and its .resource and .mods files), " + + "optional parameters are none, mv, cp, delete. (required)"; + + public static final String THREAD_NUM_ARGS = "tn"; + public static final String THREAD_NUM_NAME = "thread_num"; + public static final String THREAD_NUM_DESC = + "The number of threads used to import tsfile, default is 8.(optional)"; + + public static final String BATCH_POINT_SIZE_ARGS = "batch"; + public static final String BATCH_POINT_SIZE_NAME = "batch_size"; + public static final String BATCH_POINT_SIZE_ARGS_NAME = "batch_size"; + public static final String BATCH_POINT_SIZE_DESC = "100000 (optional)"; + + public static final String TIMESTAMP_PRECISION_ARGS = "tp"; + public static final String TIMESTAMP_PRECISION_NAME = "timestamp_precision"; + public static final String TIMESTAMP_PRECISION_ARGS_NAME = "timestamp precision (ms/us/ns)"; + public static final String TIMESTAMP_PRECISION_DESC = "Timestamp precision (ms/us/ns).(optional)"; + + public static final String TYPE_INFER_ARGS = "ti"; + public static final String TYPE_INFER_NAME = "type_infer"; + public static final String TYPE_INFER_DESC = + "Define type info by option:\"boolean=text,int=long, ... (optional)"; + + public static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; + public static final String LINES_PER_FAILED_FILE_ARGS_NAME = "lines_per_failed_file"; + public static final String LINES_PER_FAILED_FILE_DESC = "Lines per failed file.(optional)"; + + public static String successDir = "success/"; + public static String failDir = "fail/"; + + public static String timeColumn = "Time"; + public static String deviceColumn = "Device"; +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java new file mode 100644 index 000000000000..f9cabfc59ef7 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/OptionsUtil.java @@ -0,0 +1,896 @@ +package org.apache.iotdb.tool.common; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +public class OptionsUtil extends Constants { + + private static boolean isImport = true; + + public static void setIsImport(boolean isImport) { + OptionsUtil.isImport = isImport; + } + + public static Options createHelpOptions() { + final Options options = new Options(); + Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg().desc(HELP_DESC).build(); + options.addOption(opHelp); + + Option opFileType = + Option.builder(FILE_TYPE_ARGS) + .longOpt(FILE_TYPE_NAME) + .argName(FILE_TYPE_ARGS_NAME) + .hasArg() + .desc(FILE_TYPE_DESC) + .build(); + options.addOption(opFileType); + + Option opSqlDialect = + Option.builder(SQL_DIALECT_ARGS) + .longOpt(SQL_DIALECT_ARGS) + .argName(SQL_DIALECT_ARGS) + .hasArg() + .desc(SQL_DIALECT_DESC) + .build(); + options.addOption(opSqlDialect); + + return options; + } + + public static Options createCommonOptions() { + Options options = new Options(); + + Option opFileType = + Option.builder(FILE_TYPE_ARGS) + .longOpt(FILE_TYPE_NAME) + .argName(FILE_TYPE_ARGS_NAME) + .required() + .hasArg() + .desc(isImport ? FILE_TYPE_DESC_IMPORT : FILE_TYPE_DESC_EXPORT) + .build(); + options.addOption(opFileType); + + Option opSqlDialect = + Option.builder(SQL_DIALECT_ARGS) + .longOpt(SQL_DIALECT_ARGS) + .argName(SQL_DIALECT_ARGS) + .hasArg() + .desc(SQL_DIALECT_DESC) + .build(); + options.addOption(opSqlDialect); + + Option opHost = + Option.builder(HOST_ARGS) + .longOpt(HOST_NAME) + .argName(HOST_NAME) + .hasArg() + .desc(HOST_DESC) + .build(); + options.addOption(opHost); + + Option opPort = + Option.builder(PORT_ARGS) + .longOpt(PORT_NAME) + .argName(PORT_NAME) + .hasArg() + .desc(PORT_DESC) + .build(); + options.addOption(opPort); + + Option opUsername = + Option.builder(USERNAME_ARGS) + .longOpt(USERNAME_NAME) + .argName(USERNAME_NAME) + .hasArg() + .desc(USERNAME_DESC) + .build(); + options.addOption(opUsername); + + Option opPassword = + Option.builder(PW_ARGS) + .longOpt(PW_NAME) + .optionalArg(true) + .argName(PW_NAME) + .hasArg() + .desc(PW_DESC) + .build(); + options.addOption(opPassword); + + return options; + } + + public static Options createTreeImportCommonOptions() { + return createCommonOptions(); + } + + public static Options createTableImportCommonOptions() { + Options options = createCommonOptions(); + + Option opDatabase = + Option.builder(DB_ARGS) + .longOpt(DB_NAME) + .argName(DB_ARGS) + .hasArg() + .required() + .desc(DB_DESC) + .build(); + options.addOption(opDatabase); + + return options; + } + + public static Options createTreeExportCommonOptions() { + Options options = createCommonOptions(); + + Option opFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_NAME) + .argName(TARGET_DIR_ARGS_NAME) + .hasArg() + .desc(TARGET_DIR_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_NAME) + .argName(TARGET_FILE_NAME) + .hasArg() + .desc(TARGET_FILE_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opQuery = + Option.builder(QUERY_COMMAND_ARGS) + .longOpt(QUERY_COMMAND_NAME) + .argName(QUERY_COMMAND_ARGS_NAME) + .hasArg() + .desc(QUERY_COMMAND_DESC) + .build(); + options.addOption(opQuery); + + Option opTimeOut = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_NAME) + .argName(TIMEOUT_NAME) + .hasArg() + .desc(TIMEOUT_DESC) + .build(); + options.addOption(opTimeOut); + return options; + } + + public static Options createTableExportCommonOptions() { + final Options options = createCommonOptions(); + + Option opFile = + Option.builder(TARGET_DIR_ARGS) + .required() + .longOpt(TARGET_DIR_NAME) + .argName(TARGET_DIR_ARGS_NAME) + .hasArg() + .desc(TARGET_DIR_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(TARGET_FILE_ARGS) + .longOpt(TARGET_FILE_NAME) + .argName(TARGET_FILE_NAME) + .hasArg() + .desc(TARGET_FILE_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opQuery = + Option.builder(QUERY_COMMAND_ARGS) + .longOpt(QUERY_COMMAND_NAME) + .argName(QUERY_COMMAND_ARGS_NAME) + .hasArg() + .desc(QUERY_COMMAND_DESC) + .build(); + options.addOption(opQuery); + + Option opTimeOut = + Option.builder(TIMEOUT_ARGS) + .longOpt(TIMEOUT_NAME) + .argName(TIMEOUT_NAME) + .hasArg() + .desc(TIMEOUT_DESC) + .build(); + options.addOption(opTimeOut); + + Option opDatabase = + Option.builder(DB_ARGS) + .longOpt(DB_NAME) + .argName(DB_ARGS) + .hasArg() + .required() + .desc(DB_DESC) + .build(); + options.addOption(opDatabase); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opStartTime = + Option.builder(START_TIME_ARGS) + .longOpt(START_TIME_ARGS) + .argName(START_TIME_ARGS) + .hasArg() + .desc(START_TIME_DESC) + .build(); + options.addOption(opStartTime); + + Option opEndTime = + Option.builder(END_TIME_ARGS) + .longOpt(END_TIME_ARGS) + .argName(END_TIME_ARGS) + .hasArg() + .desc(END_TIME_DESC) + .build(); + options.addOption(opEndTime); + + return options; + } + + public static Options createTableExportCsvOptions() { + Options options = createTableExportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createTableExportTsFileSqlOptions() { + Options options = createTableExportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .required() + .desc(TABLE_DESC_EXPORT) + .build(); + options.addOption(opTable); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportCsvOptions() { + Options options = createTreeExportCommonOptions(); + + Option opDataType = + Option.builder(DATA_TYPE_ARGS) + .longOpt(DATA_TYPE_NAME) + .argName(DATA_TYPE_NAME) + .hasArg() + .desc(DATA_TYPE_DESC) + .build(); + options.addOption(opDataType); + + Option opLinesPerFile = + Option.builder(LINES_PER_FILE_ARGS) + .longOpt(LINES_PER_FILE_NAME) + .argName(LINES_PER_FILE_NAME) + .hasArg() + .desc(LINES_PER_FILE_DESC) + .build(); + options.addOption(opLinesPerFile); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportSqlOptions() { + Options options = createTreeExportCommonOptions(); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_EXPORT) + .hasArgs() + .desc(ALIGNED_EXPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opTimeFormat = + Option.builder(TIME_FORMAT_ARGS) + .longOpt(TIME_FORMAT_NAME) + .argName(TIME_FORMAT_NAME) + .hasArg() + .desc(TIME_FORMAT_DESC) + .build(); + options.addOption(opTimeFormat); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + return options; + } + + public static Options createExportTsFileOptions() { + Options options = createTreeExportCommonOptions(); + return options; + } + + public static Options createImportCsvOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .required() + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_CSV_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_IMPORT) + .hasArg() + .desc(ALIGNED_IMPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opTypeInfer = + Option.builder(TYPE_INFER_ARGS) + .longOpt(TYPE_INFER_NAME) + .argName(TYPE_INFER_NAME) + .numberOfArgs(5) + .hasArgs() + .valueSeparator(',') + .desc(TYPE_INFER_DESC) + .build(); + options.addOption(opTypeInfer); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createImportSqlOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_SQL_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArgs() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createImportTsFileOptions() { + Options options = createTreeImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(ON_SUCCESS_ARGS) + .longOpt(ON_SUCCESS_NAME) + .argName(ON_SUCCESS_NAME) + .required() + .hasArg() + .desc(ON_SUCCESS_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opSuccessDir = + Option.builder(SUCCESS_DIR_ARGS) + .longOpt(SUCCESS_DIR_NAME) + .argName(SUCCESS_DIR_NAME) + .hasArg() + .desc(SUCCESS_DIR_DESC) + .build(); + options.addOption(opSuccessDir); + + Option opOnFail = + Option.builder(ON_FAIL_ARGS) + .longOpt(ON_FAIL_NAME) + .argName(ON_FAIL_NAME) + .required() + .hasArg() + .desc(ON_FAIL_DESC) + .build(); + options.addOption(opOnFail); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_TSFILE_DESC) + .build(); + options.addOption(opFailDir); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + return options; + } + + public static Options createTableImportCsvOptions() { + Options options = createTableImportCommonOptions(); + + Option opTable = + Option.builder(TABLE_ARGS) + .longOpt(TABLE_ARGS) + .argName(TABLE_ARGS) + .hasArg() + .required() + .desc(TABLE_DESC_IMPORT) + .build(); + options.addOption(opTable); + + Option opFile = + Option.builder(FILE_ARGS) + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .required() + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_CSV_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opAligned = + Option.builder(ALIGNED_ARGS) + .longOpt(ALIGNED_NAME) + .argName(ALIGNED_ARGS_NAME_IMPORT) + .hasArg() + .desc(ALIGNED_IMPORT_DESC) + .build(); + options.addOption(opAligned); + + Option opTypeInfer = + Option.builder(TYPE_INFER_ARGS) + .longOpt(TYPE_INFER_NAME) + .argName(TYPE_INFER_NAME) + .numberOfArgs(5) + .hasArgs() + .valueSeparator(',') + .desc(TYPE_INFER_DESC) + .build(); + options.addOption(opTypeInfer); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_ARGS_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createTableImportSqlOptions() { + Options options = createTableImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_SQL_DESC) + .build(); + options.addOption(opFailDir); + + Option opFailedLinesPerFile = + Option.builder(LINES_PER_FAILED_FILE_ARGS) + .argName(LINES_PER_FAILED_FILE_ARGS_NAME) + .hasArgs() + .desc(LINES_PER_FAILED_FILE_DESC) + .build(); + options.addOption(opFailedLinesPerFile); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opBatchPointSize = + Option.builder(BATCH_POINT_SIZE_ARGS) + .longOpt(BATCH_POINT_SIZE_NAME) + .argName(BATCH_POINT_SIZE_NAME) + .hasArg() + .desc(BATCH_POINT_SIZE_DESC) + .build(); + options.addOption(opBatchPointSize); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArgs() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + return options; + } + + public static Options createTableImportTsFileOptions() { + Options options = createTableImportCommonOptions(); + + Option opFile = + Option.builder(FILE_ARGS) + .required() + .longOpt(FILE_NAME) + .argName(FILE_NAME) + .hasArg() + .desc(FILE_DESC) + .build(); + options.addOption(opFile); + + Option opOnSuccess = + Option.builder(ON_SUCCESS_ARGS) + .longOpt(ON_SUCCESS_NAME) + .argName(ON_SUCCESS_NAME) + .required() + .hasArg() + .desc(ON_SUCCESS_DESC) + .build(); + options.addOption(opOnSuccess); + + Option opSuccessDir = + Option.builder(SUCCESS_DIR_ARGS) + .longOpt(SUCCESS_DIR_NAME) + .argName(SUCCESS_DIR_NAME) + .hasArg() + .desc(SUCCESS_DIR_DESC) + .build(); + options.addOption(opSuccessDir); + + Option opOnFail = + Option.builder(ON_FAIL_ARGS) + .longOpt(ON_FAIL_NAME) + .argName(ON_FAIL_NAME) + .required() + .hasArg() + .desc(ON_FAIL_DESC) + .build(); + options.addOption(opOnFail); + + Option opFailDir = + Option.builder(FAIL_DIR_ARGS) + .longOpt(FAIL_DIR_NAME) + .argName(FAIL_DIR_NAME) + .hasArg() + .desc(FAIL_DIR_TSFILE_DESC) + .build(); + options.addOption(opFailDir); + + Option opThreadNum = + Option.builder(THREAD_NUM_ARGS) + .longOpt(THREAD_NUM_NAME) + .argName(THREAD_NUM_NAME) + .hasArg() + .desc(THREAD_NUM_DESC) + .build(); + options.addOption(opThreadNum); + + Option opTimeZone = + Option.builder(TIME_ZONE_ARGS) + .longOpt(TIME_ZONE_NAME) + .argName(TIME_ZONE_NAME) + .hasArg() + .desc(TIME_ZONE_DESC) + .build(); + options.addOption(opTimeZone); + + Option opTimestampPrecision = + Option.builder(TIMESTAMP_PRECISION_ARGS) + .longOpt(TIMESTAMP_PRECISION_NAME) + .argName(TIMESTAMP_PRECISION_ARGS_NAME) + .hasArg() + .desc(TIMESTAMP_PRECISION_DESC) + .build(); + + options.addOption(opTimestampPrecision); + return options; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java index 286a755756b2..1f95c7e3ff95 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -27,15 +27,13 @@ import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.isession.SessionDataSet; -import org.apache.iotdb.isession.pool.SessionDataSetWrapper; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.tsfile.ImportTsFile; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -50,6 +48,7 @@ import org.apache.tsfile.read.common.Field; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,98 +74,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.iotdb.tool.common.Constants.*; import static org.apache.tsfile.enums.TSDataType.STRING; import static org.apache.tsfile.enums.TSDataType.TEXT; public abstract class AbstractDataTool { - protected static final String FILE_TYPE_ARGS = "ft"; - protected static final String FILE_TYPE_NAME = "file_type"; - protected static final String FILE_TYPE_ARGS_NAME = "format"; - - protected static final String HOST_ARGS = "h"; - protected static final String HOST_NAME = "host"; - protected static final String HOST_DEFAULT_VALUE = "127.0.0.1"; - - protected static final String HELP_ARGS = "help"; - - protected static final String PORT_ARGS = "p"; - protected static final String PORT_NAME = "port"; - protected static final String PORT_DEFAULT_VALUE = "6667"; - - protected static final String PW_ARGS = "pw"; - protected static final String PW_NAME = "password"; - protected static final String PW_DEFAULT_VALUE = "root"; - - protected static final String USERNAME_ARGS = "u"; - protected static final String USERNAME_NAME = "username"; - protected static final String USERNAME_DEFAULT_VALUE = "root"; - - protected static final String TIME_FORMAT_ARGS = "tf"; - protected static final String TIME_FORMAT_NAME = "time_format"; - - protected static final String TIME_ZONE_ARGS = "tz"; - protected static final String TIME_ZONE_NAME = "timezone"; - protected static final String TIMEOUT_ARGS = "timeout"; - protected static final String TIMEOUT_NAME = "query_timeout"; - protected static final int MAX_HELP_CONSOLE_WIDTH = 92; - protected static final String[] TIME_FORMAT = - new String[] {"default", "long", "number", "timestamp"}; - protected static final String[] STRING_TIME_FORMAT = - new String[] { - "yyyy-MM-dd HH:mm:ss.SSSX", - "yyyy/MM/dd HH:mm:ss.SSSX", - "yyyy.MM.dd HH:mm:ss.SSSX", - "yyyy-MM-dd HH:mm:ssX", - "yyyy/MM/dd HH:mm:ssX", - "yyyy.MM.dd HH:mm:ssX", - "yyyy-MM-dd HH:mm:ss.SSSz", - "yyyy/MM/dd HH:mm:ss.SSSz", - "yyyy.MM.dd HH:mm:ss.SSSz", - "yyyy-MM-dd HH:mm:ssz", - "yyyy/MM/dd HH:mm:ssz", - "yyyy.MM.dd HH:mm:ssz", - "yyyy-MM-dd HH:mm:ss.SSS", - "yyyy/MM/dd HH:mm:ss.SSS", - "yyyy.MM.dd HH:mm:ss.SSS", - "yyyy-MM-dd HH:mm:ss", - "yyyy/MM/dd HH:mm:ss", - "yyyy.MM.dd HH:mm:ss", - "yyyy-MM-dd'T'HH:mm:ss.SSSX", - "yyyy/MM/dd'T'HH:mm:ss.SSSX", - "yyyy.MM.dd'T'HH:mm:ss.SSSX", - "yyyy-MM-dd'T'HH:mm:ssX", - "yyyy/MM/dd'T'HH:mm:ssX", - "yyyy.MM.dd'T'HH:mm:ssX", - "yyyy-MM-dd'T'HH:mm:ss.SSSz", - "yyyy/MM/dd'T'HH:mm:ss.SSSz", - "yyyy.MM.dd'T'HH:mm:ss.SSSz", - "yyyy-MM-dd'T'HH:mm:ssz", - "yyyy/MM/dd'T'HH:mm:ssz", - "yyyy.MM.dd'T'HH:mm:ssz", - "yyyy-MM-dd'T'HH:mm:ss.SSS", - "yyyy/MM/dd'T'HH:mm:ss.SSS", - "yyyy.MM.dd'T'HH:mm:ss.SSS", - "yyyy-MM-dd'T'HH:mm:ss", - "yyyy/MM/dd'T'HH:mm:ss", - "yyyy.MM.dd'T'HH:mm:ss" - }; - protected static final String INSERT_CSV_MEET_ERROR_MSG = "Meet error when insert csv because "; - protected static final String CSV_SUFFIXS = "csv"; - protected static final String TXT_SUFFIXS = "txt"; - protected static final String SQL_SUFFIXS = "sql"; - protected static final String TSFILE_SUFFIXS = "tsfile"; - protected static final String TSFILEDB_CLI_DIVIDE = "-------------------"; - protected static final String COLON = ": "; - protected static final String MINUS = "-"; - protected static String failedFileDirectory = null; - protected static String timeColumn = "Time"; - protected static String deviceColumn = "Device"; - protected static int linesPerFailedFile = 10000; - protected static String timestampPrecision = "ms"; - protected static final int CODE_OK = 0; - protected static final int CODE_ERROR = 1; - protected static String host; protected static String port; protected static String username; @@ -176,53 +89,29 @@ public abstract class AbstractDataTool { protected static String timeFormat; protected static String exportType; protected static Boolean aligned; - protected static Session session; - protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); - - protected static final String DATATYPE_BOOLEAN = "boolean"; - protected static final String DATATYPE_INT = "int"; - protected static final String DATATYPE_LONG = "long"; - protected static final String DATATYPE_FLOAT = "float"; - protected static final String DATATYPE_DOUBLE = "double"; - protected static final String DATATYPE_TIMESTAMP = "timestamp"; - protected static final String DATATYPE_DATE = "date"; - protected static final String DATATYPE_BLOB = "blob"; - protected static final String DATATYPE_NAN = "NaN"; - protected static final String DATATYPE_TEXT = "text"; - protected static final String DATATYPE_STRING = "string"; - - protected static final String DATATYPE_NULL = "null"; + protected static String table; + protected static String database; + protected static String startTime; + protected static String endTime; + protected static int threadNum = 8; + protected static String targetPath; + protected static long timeout = -1; + protected static String queryCommand; + protected static String targetDirectory; + protected static Boolean needDataTypePrinted; + protected static int linesPerFile = 10000; + protected static boolean isRemoteLoad = true; + protected static boolean sqlDialectTree = true; protected static int batchPointSize = 100_000; - - protected static final Map TYPE_INFER_KEY_DICT = new HashMap<>(); - - static { - TYPE_INFER_KEY_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); - TYPE_INFER_KEY_DICT.put(DATATYPE_INT, TSDataType.FLOAT); - TYPE_INFER_KEY_DICT.put(DATATYPE_LONG, TSDataType.DOUBLE); - TYPE_INFER_KEY_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); - TYPE_INFER_KEY_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); - TYPE_INFER_KEY_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); - TYPE_INFER_KEY_DICT.put(DATATYPE_DATE, TSDataType.TIMESTAMP); - TYPE_INFER_KEY_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); - TYPE_INFER_KEY_DICT.put(DATATYPE_NAN, TSDataType.DOUBLE); - TYPE_INFER_KEY_DICT.put(DATATYPE_STRING, TSDataType.STRING); - } - - protected static final Map TYPE_INFER_VALUE_DICT = new HashMap<>(); - - static { - TYPE_INFER_VALUE_DICT.put(DATATYPE_BOOLEAN, TSDataType.BOOLEAN); - TYPE_INFER_VALUE_DICT.put(DATATYPE_INT, TSDataType.INT32); - TYPE_INFER_VALUE_DICT.put(DATATYPE_LONG, TSDataType.INT64); - TYPE_INFER_VALUE_DICT.put(DATATYPE_FLOAT, TSDataType.FLOAT); - TYPE_INFER_VALUE_DICT.put(DATATYPE_DOUBLE, TSDataType.DOUBLE); - TYPE_INFER_VALUE_DICT.put(DATATYPE_TIMESTAMP, TSDataType.TIMESTAMP); - TYPE_INFER_VALUE_DICT.put(DATATYPE_DATE, TSDataType.DATE); - TYPE_INFER_VALUE_DICT.put(DATATYPE_BLOB, TSDataType.BLOB); - TYPE_INFER_VALUE_DICT.put(DATATYPE_TEXT, TSDataType.TEXT); - TYPE_INFER_VALUE_DICT.put(DATATYPE_STRING, TSDataType.STRING); - } + protected static int linesPerFailedFile = 10000; + protected static String timestampPrecision = "ms"; + protected static String failedFileDirectory = null; + protected static ImportTsFile.Operation successOperation; + protected static ImportTsFile.Operation failOperation; + protected static final LongAdder loadFileFailedNum = new LongAdder(); + protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); + protected static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); + protected static final LongAdder processingLoadSuccessfulFileSuccessfulNum = new LongAdder(); private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDataTool.class); @@ -245,13 +134,6 @@ protected static String checkRequiredArg( return str; } - protected static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { - if (timeZoneID != null) { - session.setTimeZone(timeZoneID); - } - zoneId = ZoneId.of(session.getTimeZone()); - } - protected static void parseBasicParams(CommandLine commandLine) throws ArgsErrorException { host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, HOST_DEFAULT_VALUE); port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, PORT_DEFAULT_VALUE); @@ -313,231 +195,6 @@ protected static boolean checkTimeFormat() { return false; } - protected static Options createImportOptions() { - Options options = new Options(); - Option opFileType = - Option.builder(FILE_TYPE_ARGS) - .longOpt(FILE_TYPE_NAME) - .argName(FILE_TYPE_ARGS_NAME) - .required() - .hasArg() - .desc("Types of imported files: CSV, SQL, TSfile (required)") - .build(); - options.addOption(opFileType); - return createNewOptions(options); - } - - protected static Options createExportOptions() { - Options options = new Options(); - Option opFileType = - Option.builder(FILE_TYPE_ARGS) - .longOpt(FILE_TYPE_NAME) - .argName(FILE_TYPE_ARGS_NAME) - .required() - .hasArg() - .desc("Export file type ?You can choose tsfile)、csv) or sql) . (required)") - .build(); - options.addOption(opFileType); - return createNewOptions(options); - } - - protected static Options createNewOptions(Options options) { - Option opHost = - Option.builder(HOST_ARGS) - .longOpt(HOST_NAME) - .argName(HOST_NAME) - .hasArg() - .desc("Host Name (optional)") - .build(); - options.addOption(opHost); - - Option opPort = - Option.builder(PORT_ARGS) - .longOpt(PORT_NAME) - .argName(PORT_NAME) - .hasArg() - .desc("Port (optional)") - .build(); - options.addOption(opPort); - - Option opUsername = - Option.builder(USERNAME_ARGS) - .longOpt(USERNAME_NAME) - .argName(USERNAME_NAME) - .hasArg() - .desc("Username (optional)") - .build(); - options.addOption(opUsername); - - Option opPassword = - Option.builder(PW_ARGS) - .longOpt(PW_NAME) - .optionalArg(true) - .argName(PW_NAME) - .hasArg() - .desc("Password (optional)") - .build(); - options.addOption(opPassword); - return options; - } - - /** - * if the data is aligned by device, the data will be written by this method. - * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param failedFilePath the directory to save the failed files - */ - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - protected static void writeDataAlignedByDevice( - SessionPool sessionPool, - List headerNames, - Stream records, - String failedFilePath) - throws IllegalPathException { - HashMap headerTypeMap = new HashMap<>(); - HashMap headerNameMap = new HashMap<>(); - parseHeaders(headerNames, null, headerTypeMap, headerNameMap); - - AtomicReference deviceName = new AtomicReference<>(null); - - HashSet typeQueriedDevice = new HashSet<>(); - - // the data that interface need - List times = new ArrayList<>(); - List> typesList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - List> measurementsList = new ArrayList<>(); - - AtomicInteger pointSize = new AtomicInteger(0); - - ArrayList> failedRecords = new ArrayList<>(); - - records.forEach( - recordObj -> { - // only run in first record - if (deviceName.get() == null) { - deviceName.set(recordObj.get(1)); - } else if (!Objects.equals(deviceName.get(), recordObj.get(1))) { - // if device changed - writeAndEmptyDataSet( - sessionPool, deviceName.get(), times, typesList, valuesList, measurementsList, 3); - deviceName.set(recordObj.get(1)); - pointSize.set(0); - } else if (pointSize.get() >= batchPointSize) { - // insert a batch - writeAndEmptyDataSet( - sessionPool, deviceName.get(), times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - // the data of the record - ArrayList types = new ArrayList<>(); - ArrayList values = new ArrayList<>(); - ArrayList measurements = new ArrayList<>(); - - AtomicReference isFail = new AtomicReference<>(false); - - // read data from record - for (Map.Entry headerNameEntry : headerNameMap.entrySet()) { - // headerNameWithoutType is equal to headerName if the CSV column do not have data type. - String headerNameWithoutType = headerNameEntry.getKey(); - String headerName = headerNameEntry.getValue(); - String value = recordObj.get(headerName); - if (!"".equals(value)) { - TSDataType type; - // Get the data type directly if the CSV column have data type. - if (!headerTypeMap.containsKey(headerNameWithoutType)) { - boolean hasResult = false; - // query the data type in iotdb - if (!typeQueriedDevice.contains(deviceName.get())) { - if (headerTypeMap.isEmpty()) { - Set devices = new HashSet<>(); - devices.add(deviceName.get()); - queryType(sessionPool, devices, headerTypeMap, deviceColumn); - } - typeQueriedDevice.add(deviceName.get()); - } - type = typeInfer(value); - if (type != null) { - headerTypeMap.put(headerNameWithoutType, type); - } else { - ioTPrinter.printf( - "Line '%s', column '%s': '%s' unknown type%n", - recordObj.getRecordNumber(), headerNameWithoutType, value); - isFail.set(true); - } - } - type = headerTypeMap.get(headerNameWithoutType); - if (type != null) { - Object valueTrans = typeTrans(value, type); - if (valueTrans == null) { - isFail.set(true); - ioTPrinter.printf( - "Line '%s', column '%s': '%s' can't convert to '%s'%n", - recordObj.getRecordNumber(), headerNameWithoutType, value, type); - } else { - values.add(valueTrans); - measurements.add(headerNameWithoutType); - types.add(type); - pointSize.getAndIncrement(); - } - } - } - } - if (Boolean.TRUE.equals(isFail.get())) { - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - } - if (!measurements.isEmpty()) { - times.add(parseTimestamp(recordObj.get(timeColumn))); - typesList.add(types); - valuesList.add(values); - measurementsList.add(measurements); - } - }); - if (!times.isEmpty()) { - writeAndEmptyDataSet( - sessionPool, deviceName.get(), times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - if (!failedRecords.isEmpty()) { - writeFailedLinesFile(headerNames, failedFilePath, failedRecords); - } - // ioTPrinter.println("Import completely!"); - } - - private static void writeAndEmptyDataSet( - SessionPool sessionPool, - String device, - List times, - List> typesList, - List> valuesList, - List> measurementsList, - int retryTime) { - try { - if (Boolean.FALSE.equals(aligned)) { - sessionPool.insertRecordsOfOneDevice( - device, times, measurementsList, typesList, valuesList); - } else { - sessionPool.insertAlignedRecordsOfOneDevice( - device, times, measurementsList, typesList, valuesList); - } - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - writeAndEmptyDataSet( - sessionPool, device, times, typesList, valuesList, measurementsList, --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - System.exit(1); - } finally { - times.clear(); - typesList.clear(); - valuesList.clear(); - measurementsList.clear(); - } - } - private static void writeAndEmptyDataSet( Session session, List deviceIds, @@ -579,80 +236,8 @@ private static void writeAndEmptyDataSet( } } - private static void writeAndEmptyDataSet( - Session session, - String device, - List times, - List> typesList, - List> valuesList, - List> measurementsList, - int retryTime) { - try { - if (Boolean.FALSE.equals(aligned)) { - session.insertRecordsOfOneDevice(device, times, measurementsList, typesList, valuesList); - } else { - session.insertAlignedRecordsOfOneDevice( - device, times, measurementsList, typesList, valuesList); - } - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - try { - session.open(); - } catch (IoTDBConnectionException ex) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - } - writeAndEmptyDataSet( - session, device, times, typesList, valuesList, measurementsList, --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } finally { - times.clear(); - typesList.clear(); - valuesList.clear(); - measurementsList.clear(); - } - } - - private static void writeAndEmptyDataSet( - SessionPool sessionPool, - List deviceIds, - List times, - List> typesList, - List> valuesList, - List> measurementsList, - int retryTime) { - try { - if (Boolean.FALSE.equals(aligned)) { - sessionPool.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); - } else { - sessionPool.insertAlignedRecords(deviceIds, times, measurementsList, typesList, valuesList); - } - } catch (IoTDBConnectionException e) { - if (retryTime > 0) { - writeAndEmptyDataSet( - sessionPool, deviceIds, times, typesList, valuesList, measurementsList, --retryTime); - } - } catch (StatementExecutionException e) { - ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); - System.exit(1); - } finally { - deviceIds.clear(); - times.clear(); - typesList.clear(); - valuesList.clear(); - measurementsList.clear(); - } - } - - private static void writeFailedLinesFile( - List headerNames, String failedFilePath, ArrayList> failedRecords) { + protected static void writeFailedLinesFile( + List headerNames, String failedFilePath, List> failedRecords) { int fileIndex = 0; int from = 0; int failedRecordsSize = failedRecords.size(); @@ -675,7 +260,7 @@ private static void writeFailedLinesFile( * @param strValue * @return */ - private static TSDataType typeInfer(String strValue) { + protected static TSDataType typeInfer(String strValue) { if (strValue.contains("\"")) { return strValue.length() <= 512 + 2 ? STRING : TEXT; } @@ -734,7 +319,7 @@ private static boolean isConvertFloatPrecisionLack(String s) { * @param type * @return */ - private static Object typeTrans(String value, TSDataType type) { + protected static Object typeTrans(String value, TSDataType type) { try { switch (type) { case TEXT: @@ -779,7 +364,7 @@ private static byte[] parseHexStringToByteArray(String hexString) { return bytes; } - private static long parseTimestamp(String str) { + protected static long parseTimestamp(String str) { long timestamp; try { timestamp = Long.parseLong(str); @@ -790,149 +375,12 @@ private static long parseTimestamp(String str) { } /** - * query data type of timeseries from IoTDB - * - * @param deviceNames - * @param headerTypeMap - * @param alignedType - * @throws IoTDBConnectionException - * @throws StatementExecutionException - */ - private static void queryType( - SessionPool sessionPool, - Set deviceNames, - HashMap headerTypeMap, - String alignedType) { - for (String deviceName : deviceNames) { - String sql = "show timeseries " + deviceName + ".*"; - try (SessionDataSetWrapper sessionDataSetWrapper = sessionPool.executeQueryStatement(sql)) { - int tsIndex = - sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); - int dtIndex = sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); - while (sessionDataSetWrapper.hasNext()) { - RowRecord rowRecord = sessionDataSetWrapper.next(); - List fields = rowRecord.getFields(); - String timeseries = fields.get(tsIndex).getStringValue(); - String dataType = fields.get(dtIndex).getStringValue(); - if (Objects.equals(alignedType, "Time")) { - headerTypeMap.put(timeseries, getType(dataType)); - } else if (Objects.equals(alignedType, deviceColumn)) { - String[] split = PathUtils.splitPathToDetachedNodes(timeseries); - String measurement = split[split.length - 1]; - headerTypeMap.put(measurement, getType(dataType)); - } - } - } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { - ioTPrinter.println( - "Meet error when query the type of timeseries because " + e.getMessage()); - System.exit(1); - } - } - } - - /** - * query data type of timeseries from IoTDB - * - * @param deviceNames - * @param headerTypeMap - * @param alignedType - * @throws IoTDBConnectionException - * @throws StatementExecutionException - */ - private static void queryType( - Session session, - Set deviceNames, - HashMap headerTypeMap, - String alignedType) { - for (String deviceName : deviceNames) { - String sql = "show timeseries " + deviceName + ".*"; - SessionDataSet sessionDataSet = null; - try { - sessionDataSet = session.executeQueryStatement(sql); - int tsIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); - int dtIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); - while (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - String timeseries = fields.get(tsIndex).getStringValue(); - String dataType = fields.get(dtIndex).getStringValue(); - if (Objects.equals(alignedType, "Time")) { - headerTypeMap.put(timeseries, getType(dataType)); - } else if (Objects.equals(alignedType, deviceColumn)) { - String[] split = PathUtils.splitPathToDetachedNodes(timeseries); - String measurement = split[split.length - 1]; - headerTypeMap.put(measurement, getType(dataType)); - } - } - } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { - ioTPrinter.println( - "Meet error when query the type of timeseries because " + e.getMessage()); - try { - session.close(); - } catch (IoTDBConnectionException ex) { - // do nothing - } - System.exit(1); - } - } - } - - /** - * parse deviceNames, measurementNames(aligned by time), headerType from headers - * - * @param headerNames - * @param deviceAndMeasurementNames - * @param headerTypeMap - * @param headerNameMap - */ - @SuppressWarnings( - "squid:S135") // ignore for loops should not contain more than a single "break" or "continue" - // statement - private static void parseHeaders( - List headerNames, - @Nullable HashMap> deviceAndMeasurementNames, - HashMap headerTypeMap, - HashMap headerNameMap) - throws IllegalPathException { - String regex = "(?<=\\()\\S+(?=\\))"; - Pattern pattern = Pattern.compile(regex); - for (String headerName : headerNames) { - if ("Time".equalsIgnoreCase(filterBomHeader(headerName))) { - timeColumn = headerName; - continue; - } else if ("Device".equalsIgnoreCase(headerName)) { - deviceColumn = headerName; - continue; - } - Matcher matcher = pattern.matcher(headerName); - String type; - String headerNameWithoutType; - if (matcher.find()) { - type = matcher.group(); - headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); - headerNameMap.put(headerNameWithoutType, headerName); - headerTypeMap.put(headerNameWithoutType, getType(type)); - } else { - headerNameWithoutType = headerName; - headerNameMap.put(headerName, headerName); - } - String[] split = PathUtils.splitPathToDetachedNodes(headerNameWithoutType); - String measurementName = split[split.length - 1]; - String deviceName = StringUtils.join(Arrays.copyOfRange(split, 0, split.length - 1), '.'); - if (deviceAndMeasurementNames != null) { - deviceAndMeasurementNames.putIfAbsent(deviceName, new ArrayList<>()); - deviceAndMeasurementNames.get(deviceName).add(measurementName); - } - } - } - - /** - * return the TSDataType + * return the TSDataType * * @param typeStr * @return */ - private static TSDataType getType(String typeStr) { + protected static TSDataType getType(String typeStr) { try { return TSDataType.valueOf(typeStr); } catch (Exception e) { @@ -941,117 +389,20 @@ private static TSDataType getType(String typeStr) { } /** - * if the data is aligned by time, the data will be written by this method. + * return the ColumnCategory * - * @param headerNames the header names of CSV file - * @param records the records of CSV file - * @param failedFilePath the directory to save the failed files + * @param typeStr + * @return */ - @SuppressWarnings("squid:S3776") - protected static void writeDataAlignedByTime( - SessionPool sessionPool, - List headerNames, - Stream records, - String failedFilePath) - throws IllegalPathException { - HashMap> deviceAndMeasurementNames = new HashMap<>(); - HashMap headerTypeMap = new HashMap<>(); - HashMap headerNameMap = new HashMap<>(); - parseHeaders(headerNames, deviceAndMeasurementNames, headerTypeMap, headerNameMap); - - Set devices = deviceAndMeasurementNames.keySet(); - if (headerTypeMap.isEmpty()) { - queryType(sessionPool, devices, headerTypeMap, "Time"); - } - - List deviceIds = new ArrayList<>(); - List times = new ArrayList<>(); - List> measurementsList = new ArrayList<>(); - List> typesList = new ArrayList<>(); - List> valuesList = new ArrayList<>(); - - AtomicReference hasStarted = new AtomicReference<>(false); - AtomicInteger pointSize = new AtomicInteger(0); - - ArrayList> failedRecords = new ArrayList<>(); - - records.forEach( - recordObj -> { - if (Boolean.FALSE.equals(hasStarted.get())) { - hasStarted.set(true); - } else if (pointSize.get() >= batchPointSize) { - writeAndEmptyDataSet( - sessionPool, deviceIds, times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - boolean isFail = false; - - for (Map.Entry> entry : deviceAndMeasurementNames.entrySet()) { - String deviceId = entry.getKey(); - List measurementNames = entry.getValue(); - ArrayList types = new ArrayList<>(); - ArrayList values = new ArrayList<>(); - ArrayList measurements = new ArrayList<>(); - for (String measurement : measurementNames) { - String header = deviceId + "." + measurement; - String value = recordObj.get(headerNameMap.get(header)); - if (!"".equals(value)) { - TSDataType type; - if (!headerTypeMap.containsKey(header)) { - type = typeInfer(value); - if (type != null) { - headerTypeMap.put(header, type); - } else { - ioTPrinter.printf( - "Line '%s', column '%s': '%s' unknown type%n", - recordObj.getRecordNumber(), header, value); - isFail = true; - } - } - type = headerTypeMap.get(header); - if (type != null) { - Object valueTrans = typeTrans(value, type); - if (valueTrans == null) { - isFail = true; - ioTPrinter.printf( - "Line '%s', column '%s': '%s' can't convert to '%s'%n", - recordObj.getRecordNumber(), header, value, type); - } else { - measurements.add(header.replace(deviceId + '.', "")); - types.add(type); - values.add(valueTrans); - pointSize.getAndIncrement(); - } - } - } - } - if (!measurements.isEmpty()) { - times.add(parseTimestamp(recordObj.get(timeColumn))); - deviceIds.add(deviceId); - typesList.add(types); - valuesList.add(values); - measurementsList.add(measurements); - } - } - if (isFail) { - failedRecords.add(recordObj.stream().collect(Collectors.toList())); - } - }); - if (!deviceIds.isEmpty()) { - writeAndEmptyDataSet( - sessionPool, deviceIds, times, typesList, valuesList, measurementsList, 3); - pointSize.set(0); - } - - if (!failedRecords.isEmpty()) { - writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + protected static Tablet.ColumnCategory getColumnCategory(String typeStr) { + if (StringUtils.isNotBlank(typeStr)) { + try { + return Tablet.ColumnCategory.valueOf(typeStr); + } catch (Exception e) { + return null; + } } - // if (Boolean.TRUE.equals(hasStarted.get())) { - // ioTPrinter.println("Import completely!"); - // } else { - // ioTPrinter.println("No records!"); - // } + return null; } /** @@ -1081,7 +432,7 @@ protected static void writeDataAlignedByDevice( AtomicInteger pointSize = new AtomicInteger(0); - ArrayList> failedRecords = new ArrayList<>(); + List> failedRecords = new ArrayList<>(); records.forEach( recordObj -> { @@ -1286,6 +637,135 @@ protected static void writeDataAlignedByTime( } } + private static void writeAndEmptyDataSet( + Session session, + String device, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + session.insertRecordsOfOneDevice(device, times, measurementsList, typesList, valuesList); + } else { + session.insertAlignedRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + try { + session.open(); + } catch (IoTDBConnectionException ex) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } + writeAndEmptyDataSet( + session, device, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } finally { + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + /** + * query data type of timeseries from IoTDB + * + * @param deviceNames + * @param headerTypeMap + * @param alignedType + * @throws IoTDBConnectionException + * @throws StatementExecutionException + */ + private static void queryType( + Session session, + Set deviceNames, + HashMap headerTypeMap, + String alignedType) { + for (String deviceName : deviceNames) { + String sql = "show timeseries " + deviceName + ".*"; + SessionDataSet sessionDataSet = null; + try { + sessionDataSet = session.executeQueryStatement(sql); + int tsIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); + int dtIndex = sessionDataSet.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + String timeseries = fields.get(tsIndex).getStringValue(); + String dataType = fields.get(dtIndex).getStringValue(); + if (Objects.equals(alignedType, "Time")) { + headerTypeMap.put(timeseries, getType(dataType)); + } else if (Objects.equals(alignedType, deviceColumn)) { + String[] split = PathUtils.splitPathToDetachedNodes(timeseries); + String measurement = split[split.length - 1]; + headerTypeMap.put(measurement, getType(dataType)); + } + } + } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { + ioTPrinter.println( + "Meet error when query the type of timeseries because " + e.getMessage()); + try { + session.close(); + } catch (IoTDBConnectionException ex) { + // do nothing + } + System.exit(1); + } + } + } + + @SuppressWarnings( + "squid:S135") // ignore for loops should not contain more than a single "break" or "continue" + // statement + protected static void parseHeaders( + List headerNames, + @Nullable HashMap> deviceAndMeasurementNames, + HashMap headerTypeMap, + HashMap headerNameMap) + throws IllegalPathException { + String regex = "(?<=\\()\\S+(?=\\))"; + Pattern pattern = Pattern.compile(regex); + for (String headerName : headerNames) { + if ("Time".equalsIgnoreCase(filterBomHeader(headerName))) { + timeColumn = headerName; + continue; + } else if ("Device".equalsIgnoreCase(headerName)) { + deviceColumn = headerName; + continue; + } + Matcher matcher = pattern.matcher(headerName); + String type; + String headerNameWithoutType; + if (matcher.find()) { + type = matcher.group(); + headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); + headerNameMap.put(headerNameWithoutType, headerName); + headerTypeMap.put(headerNameWithoutType, getType(type)); + } else { + headerNameWithoutType = headerName; + headerNameMap.put(headerName, headerName); + } + String[] split = PathUtils.splitPathToDetachedNodes(headerNameWithoutType); + String measurementName = split[split.length - 1]; + String deviceName = StringUtils.join(Arrays.copyOfRange(split, 0, split.length - 1), '.'); + if (deviceAndMeasurementNames != null) { + deviceAndMeasurementNames.putIfAbsent(deviceName, new ArrayList<>()); + deviceAndMeasurementNames.get(deviceName).add(measurementName); + } + } + } + protected static String filterBomHeader(String s) { byte[] bom = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; byte[] bytes = Arrays.copyOf(s.getBytes(), 3); @@ -1295,27 +775,6 @@ protected static String filterBomHeader(String s) { return s; } - protected static Options createHelpOptions() { - final Options options = new Options(); - Option opHelp = - Option.builder(HELP_ARGS) - .longOpt(HELP_ARGS) - .hasArg() - .desc("Display help information") - .build(); - options.addOption(opHelp); - - Option opFileType = - Option.builder(FILE_TYPE_ARGS) - .longOpt(FILE_TYPE_NAME) - .argName(FILE_TYPE_ARGS_NAME) - .hasArg() - .desc("Export file type ?You can choose tsfile)、csv) or sql) . (required)") - .build(); - options.addOption(opFileType); - return options; - } - /** * read data from the CSV file * diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java new file mode 100644 index 000000000000..1bf3a23e3e85 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractExportData.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.thrift.TException; + +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.iotdb.tool.common.Constants.*; + +public abstract class AbstractExportData extends AbstractDataTool { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + public abstract void init() + throws IoTDBConnectionException, StatementExecutionException, TException; + + public abstract void exportBySql(String sql, int index); + + protected static void legalCheck(String sql) { + String aggregatePattern = + "\\b(count|sum|avg|extreme|max_value|min_value|first_value|last_value|max_time|min_time|stddev|stddev_pop|stddev_samp|variance|var_pop|var_samp|max_by|min_by)\\b\\s*\\("; + Pattern pattern = Pattern.compile(aggregatePattern, Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(sql); + if (matcher.find()) { + ioTPrinter.println("The sql you entered is invalid, please don't use aggregate query."); + System.exit(CODE_ERROR); + } + } + + protected static String timeTrans(Long time) { + switch (timeFormat) { + case "default": + return RpcUtils.parseLongToDateWithPrecision( + DateTimeFormatter.ISO_OFFSET_DATE_TIME, time, zoneId, timestampPrecision); + case "timestamp": + case "long": + case "number": + return String.valueOf(time); + default: + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), zoneId) + .format(DateTimeFormatter.ofPattern(timeFormat)); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java new file mode 100644 index 000000000000..b3be0502f163 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractImportData.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import java.io.File; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.apache.iotdb.tool.common.Constants.*; + +public abstract class AbstractImportData extends AbstractDataTool implements Runnable { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + + public abstract void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException; + + @Override + public void run() { + String filePath; + try { + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + while ((filePath = ImportTsFileScanTool.pollFromQueue()) != null) { + File file = new File(filePath); + if (file.getName().endsWith(TSFILE_SUFFIXS)) { + importFromTsFile(file); + } + } + } else { + while ((filePath = ImportDataScanTool.pollFromQueue()) != null) { + File file = new File(filePath); + if (file.getName().endsWith(SQL_SUFFIXS)) { + importFromSqlFile(file); + } else { + importFromCsvFile(file); + } + } + } + } catch (Exception e) { + ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); + } + } + + protected abstract Runnable getAsyncImportRunnable(); + + protected class ThreadManager { + public void asyncImportDataFiles() { + List list = new ArrayList<>(threadNum); + for (int i = 0; i < threadNum; i++) { + Thread thread = new Thread(getAsyncImportRunnable()); + thread.start(); + list.add(thread); + } + list.forEach( + thread -> { + try { + thread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + ioTPrinter.println("ImportData thread join interrupted: " + e.getMessage()); + } + }); + ioTPrinter.println("Import completely!"); + } + } + + public static void init(AbstractImportData instance) { + instance.new ThreadManager().asyncImportDataFiles(); + } + + protected abstract void importFromSqlFile(File file); + + protected abstract void importFromTsFile(File file); + + protected abstract void importFromCsvFile(File file); + + protected void processSuccessFile(String file) { + loadFileSuccessfulNum.increment(); + if (fileType.equalsIgnoreCase(TSFILE_SUFFIXS)) { + try { + processingFile(file, true); + processingLoadSuccessfulFileSuccessfulNum.increment(); + ioTPrinter.println("Processed success file [ " + file + " ] successfully!"); + } catch (final Exception processSuccessException) { + ioTPrinter.println( + "Failed to process success file [ " + + file + + " ]: " + + processSuccessException.getMessage()); + } + } + } + + protected void processingFile(final String file, final boolean isSuccess) { + final String relativePath = file.substring(ImportTsFileScanTool.getSourceFullPathLength() + 1); + final Path sourcePath = Paths.get(file); + + final String target = + isSuccess + ? successDir + : failDir + File.separator + relativePath.replace(File.separator, "_"); + final Path targetPath = Paths.get(target); + + final String RESOURCE = ".resource"; + Path sourceResourcePath = Paths.get(sourcePath + RESOURCE); + sourceResourcePath = Files.exists(sourceResourcePath) ? sourceResourcePath : null; + final Path targetResourcePath = Paths.get(target + RESOURCE); + + final String MODS = ".mods"; + Path sourceModsPath = Paths.get(sourcePath + MODS); + sourceModsPath = Files.exists(sourceModsPath) ? sourceModsPath : null; + final Path targetModsPath = Paths.get(target + MODS); + + switch (isSuccess ? successOperation : failOperation) { + case DELETE: + { + try { + Files.deleteIfExists(sourcePath); + if (null != sourceResourcePath) { + Files.deleteIfExists(sourceResourcePath); + } + if (null != sourceModsPath) { + Files.deleteIfExists(sourceModsPath); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to delete file: %s", e.getMessage())); + } + break; + } + case CP: + { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to copy file: %s", e.getMessage())); + } + break; + } + case HARDLINK: + { + try { + Files.createLink(targetPath, sourcePath); + } catch (FileAlreadyExistsException e) { + ioTPrinter.println("Hardlink already exists: " + e.getMessage()); + } catch (final Exception e) { + try { + Files.copy(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception copyException) { + ioTPrinter.println( + String.format("Failed to copy file: %s", copyException.getMessage())); + } + } + + try { + if (null != sourceResourcePath) { + Files.copy( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.copy(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println( + String.format("Failed to copy resource or mods file: %s", e.getMessage())); + } + break; + } + case MV: + { + try { + Files.move(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING); + if (null != sourceResourcePath) { + Files.move( + sourceResourcePath, targetResourcePath, StandardCopyOption.REPLACE_EXISTING); + } + if (null != sourceModsPath) { + Files.move(sourceModsPath, targetModsPath, StandardCopyOption.REPLACE_EXISTING); + } + } catch (final Exception e) { + ioTPrinter.println(String.format("Failed to move file: %s", e.getMessage())); + } + break; + } + default: + break; + } + } + + protected void processFailFile(final String filePath, final Exception e) { + try { + if (Objects.nonNull(e.getMessage()) && e.getMessage().contains("memory")) { + ioTPrinter.println( + "Rejecting file [ " + filePath + " ] due to memory constraints, will retry later."); + ImportTsFileScanTool.putToQueue(filePath); + return; + } + + loadFileFailedNum.increment(); + ioTPrinter.println("Failed to import [ " + filePath + " ] file: " + e.getMessage()); + + try { + processingFile(filePath, false); + processingLoadFailedFileSuccessfulNum.increment(); + ioTPrinter.println("Processed fail file [ " + filePath + " ] successfully!"); + } catch (final Exception processFailException) { + ioTPrinter.println( + "Failed to process fail file [ " + + filePath + + " ]: " + + processFailException.getMessage()); + } + } catch (final InterruptedException e1) { + ioTPrinter.println("Unexpected error occurred: " + e1.getMessage()); + Thread.currentThread().interrupt(); + } catch (final Exception e1) { + ioTPrinter.println("Unexpected error occurred: " + e1.getMessage()); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AsyncImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AsyncImportData.java deleted file mode 100644 index 6500ef670834..000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AsyncImportData.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool.data; - -import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.pool.SessionPool; - -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.lang3.ObjectUtils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -public class AsyncImportData extends AbstractDataTool implements Runnable { - - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static SessionPool sessionPool; - - protected static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { - if (timeZoneID != null) { - sessionPool.setTimeZone(timeZoneID); - zoneId = sessionPool.getZoneId(); - } - } - - @Override - public void run() { - String filePath; - try { - while ((filePath = ImportDataScanTool.pollFromQueue()) != null) { - File file = new File(filePath); - if (file.getName().endsWith(SQL_SUFFIXS)) { - importFromSqlFile(file); - } else { - importFromSingleFile(file); - } - } - } catch (Exception e) { - ioTPrinter.println("Unexpected error occurred: " + e.getMessage()); - } - } - - protected static void processSuccessFile() { - loadFileSuccessfulNum.increment(); - } - - @SuppressWarnings("java:S2259") - private static void importFromSqlFile(File file) { - ArrayList> failedRecords = new ArrayList<>(); - String failedFilePath; - if (failedFileDirectory == null) { - failedFilePath = file.getAbsolutePath() + ".failed"; - } else { - failedFilePath = failedFileDirectory + file.getName() + ".failed"; - } - try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { - String sql; - while ((sql = br.readLine()) != null) { - try { - sessionPool.executeNonQueryStatement(sql); - } catch (IoTDBConnectionException | StatementExecutionException e) { - failedRecords.add(Arrays.asList(sql)); - } - } - processSuccessFile(); - } catch (IOException e) { - ioTPrinter.println("SQL file read exception because: " + e.getMessage()); - } - if (!failedRecords.isEmpty()) { - FileWriter writer = null; - try { - writer = new FileWriter(failedFilePath); - for (List failedRecord : failedRecords) { - writer.write(failedRecord.get(0).toString() + "\n"); - } - } catch (IOException e) { - ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); - } finally { - if (ObjectUtils.isNotEmpty(writer)) { - try { - writer.flush(); - writer.close(); - } catch (IOException e) { - ; - } - } - } - } - } - - private static void importFromSingleFile(File file) { - if (file.getName().endsWith(CSV_SUFFIXS) || file.getName().endsWith(TXT_SUFFIXS)) { - try { - CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); - List headerNames = csvRecords.getHeaderNames(); - Stream records = csvRecords.stream(); - if (headerNames.isEmpty()) { - ioTPrinter.println("Empty file!"); - return; - } - if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { - ioTPrinter.println("The first field of header must be `Time`!"); - return; - } - String failedFilePath; - if (failedFileDirectory == null) { - failedFilePath = file.getAbsolutePath() + ".failed"; - } else { - failedFilePath = failedFileDirectory + file.getName() + ".failed"; - } - if (!deviceColumn.equalsIgnoreCase(headerNames.get(1))) { - writeDataAlignedByTime(sessionPool, headerNames, records, failedFilePath); - } else { - writeDataAlignedByDevice(sessionPool, headerNames, records, failedFilePath); - } - processSuccessFile(); - } catch (IOException | IllegalPathException e) { - ioTPrinter.println("CSV file read exception because: " + e.getMessage()); - } - } else { - ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); - } - } - - public static void setSessionPool(SessionPool sessionPool) { - AsyncImportData.sessionPool = sessionPool; - } - - public static void setAligned(Boolean aligned) { - AsyncImportData.aligned = aligned; - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java index 0f60c38659c6..9e75013cb6e0 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportData.java @@ -24,49 +24,26 @@ import org.apache.iotdb.cli.utils.IoTPrinter; import org.apache.iotdb.cli.utils.JlineUtils; import org.apache.iotdb.exception.ArgsErrorException; -import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.Session; -import org.apache.iotdb.tool.tsfile.ExportTsFile; +import org.apache.iotdb.tool.common.OptionsUtil; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.thrift.TException; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.read.common.Field; -import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.common.RowRecord; -import org.apache.tsfile.utils.BytesUtils; import org.jline.reader.LineReader; -import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; -import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; +import static org.apache.iotdb.tool.common.Constants.*; /** * Export CSV file. @@ -75,57 +52,7 @@ */ public class ExportData extends AbstractDataTool { - private static final String TARGET_DIR_ARGS = "t"; - private static final String TARGET_DIR_NAME = "target"; - private static final String TARGET_DIR_ARGS_NAME = "target_directory"; - - private static final String TARGET_FILE_ARGS = "pfn"; - private static final String TARGET_FILE_NAME = "prefix_file_name"; - - private static final String SQL_FILE_ARGS = "s"; - private static final String SQL_FILE_NAME = "sourceSqlFile"; - - private static final String DATA_TYPE_ARGS = "dt"; - private static final String DATA_TYPE_NAME = "datatype"; - - private static final String QUERY_COMMAND_ARGS = "q"; - private static final String QUERY_COMMAND_NAME = "query"; - private static final String QUERY_COMMAND_ARGS_NAME = "query_command"; - - private static final String EXPORT_SQL_TYPE_NAME = "sql"; - - private static final String ALIGNED_ARGS = "aligned"; - private static final String ALIGNED_NAME = "export_aligned"; - private static final String ALIGNED_ARGS_NAME = "export aligned insert sql"; - private static final String LINES_PER_FILE_ARGS = "lpf"; - private static final String LINES_PER_FILE_NAME = "lines_per_file"; - - private static final String TSFILEDB_CLI_PREFIX = "Export Data"; - - private static final String DUMP_FILE_NAME_DEFAULT = "dump"; - private static String targetFile = DUMP_FILE_NAME_DEFAULT; - private static Session session; - private static String targetDirectory; - - private static Boolean needDataTypePrinted; - - private static String queryCommand; - - private static String timestampPrecision; - - private static int linesPerFile = 10000; - - private static long timeout = -1; - - private static Boolean aligned = false; - private static String fileType = null; - private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static final String TSFILEDB_CLI_HEAD = - "Please obtain help information for the corresponding data type based on different parameters, for example:\n" - + "./export_data.sh -help tsfile\n" - + "./export_data.sh -help sql\n" - + "./export_data.sh -help csv"; @SuppressWarnings({ "squid:S3776", @@ -133,10 +60,11 @@ public class ExportData extends AbstractDataTool { }) // Suppress high Cognitive Complexity warning, ignore try-with-resources /* main function of export csv tool. */ public static void main(String[] args) { - Options helpOptions = createHelpOptions(); - Options tsFileOptions = createTsFileOptions(); - Options csvOptions = createCsvOptions(); - Options sqlOptions = createSqlOptions(); + OptionsUtil.setIsImport(false); + Options helpOptions = OptionsUtil.createHelpOptions(); + Options tsFileOptions = OptionsUtil.createExportTsFileOptions(); + Options csvOptions = OptionsUtil.createExportCsvOptions(); + Options sqlOptions = OptionsUtil.createExportSqlOptions(); HelpFormatter hf = new HelpFormatter(); CommandLine commandLine = null; CommandLineParser parser = new DefaultParser(); @@ -145,18 +73,34 @@ public static void main(String[] args) { if (args == null || args.length == 0) { printHelpOptions( - TSFILEDB_CLI_HEAD, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); System.exit(CODE_ERROR); } try { commandLine = parser.parse(helpOptions, args, true); } catch (ParseException e) { printHelpOptions( - TSFILEDB_CLI_HEAD, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); System.exit(CODE_ERROR); } final List argList = Arrays.asList(args); int helpIndex = argList.indexOf(MINUS + HELP_ARGS); + int sql_dialect = argList.indexOf(MINUS + SQL_DIALECT_ARGS); // -sql_dialect + if (sql_dialect >= 0 + && !SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(argList.get(sql_dialect + 1))) { + final String sqlDialectValue = argList.get(sql_dialect + 1); + if (SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + csvOptions = OptionsUtil.createTableExportCsvOptions(); + tsFileOptions = OptionsUtil.createTableExportTsFileSqlOptions(); + sqlOptions = OptionsUtil.createTableExportTsFileSqlOptions(); + } else { + ioTPrinter.println(String.format("sql_dialect %s is not support", sqlDialectValue)); + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + } int ftIndex = argList.indexOf(MINUS + FILE_TYPE_ARGS); if (ftIndex < 0) { ftIndex = argList.indexOf(MINUS + FILE_TYPE_NAME); @@ -165,31 +109,19 @@ public static void main(String[] args) { fileType = argList.get(helpIndex + 1); if (StringUtils.isNotBlank(fileType)) { if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, null, null, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, csvOptions, null, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, null, csvOptions, null, false); } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, null, sqlOptions, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); } else { ioTPrinter.println(String.format("File type %s is not support", fileType)); printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); } } else { printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); } System.exit(CODE_ERROR); } else if (ftIndex >= 0) { @@ -198,11 +130,11 @@ public static void main(String[] args) { if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { try { commandLine = parser.parse(tsFileOptions, args, true); - ExportTsFile exportTsFile = new ExportTsFile(commandLine); - exportTsFile.exportTsfile(CODE_OK); + // ExportTsFile exportTsFile = new ExportTsFile(commandLine); + // exportTsFile.exportTsfile(CODE_OK); } catch (ParseException e) { ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, null, null, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); System.exit(CODE_ERROR); } } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { @@ -210,7 +142,7 @@ public static void main(String[] args) { commandLine = parser.parse(csvOptions, args, true); } catch (ParseException e) { ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, csvOptions, null, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, null, csvOptions, null, false); System.exit(CODE_ERROR); } } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { @@ -218,30 +150,18 @@ public static void main(String[] args) { commandLine = parser.parse(sqlOptions, args, true); } catch (ParseException e) { ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, null, sqlOptions, false); + printHelpOptions(null, EXPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); System.exit(CODE_ERROR); } } else { ioTPrinter.println(String.format("File type %s is not support", fileType)); printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); System.exit(CODE_ERROR); } } else { printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); + EXPORT_CLI_HEAD, EXPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); System.exit(CODE_ERROR); } } else { @@ -257,35 +177,30 @@ public static void main(String[] args) { if (!checkTimeFormat()) { System.exit(CODE_ERROR); } - session = new Session(host, Integer.parseInt(port), username, password); - session.open(false); - timestampPrecision = session.getTimestampPrecision(); - setTimeZone(); - - if (queryCommand == null) { - String sqlFile = commandLine.getOptionValue(SQL_FILE_ARGS); - String sql; - - if (sqlFile == null) { - LineReader lineReader = - JlineUtils.getLineReader( - new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), - username, - host, - port); - sql = lineReader.readLine(TSFILEDB_CLI_PREFIX + "> please input query: "); - ioTPrinter.println(sql); - String[] values = sql.trim().split(";"); - for (int i = 0; i < values.length; i++) { - dumpResult(values[i], i); - } - } else { - dumpFromSqlFile(sqlFile); + AbstractExportData exportData; + // check timePrecision by session + exportData = new ExportDataTree(); + exportData.init(); + if (!sqlDialectTree) { + exportData = new ExportDataTable(); + exportData.init(); + } + if (sqlDialectTree && queryCommand == null) { + LineReader lineReader = + JlineUtils.getLineReader( + new CliContext(System.in, System.out, System.err, ExitType.EXCEPTION), + username, + host, + port); + String sql = lineReader.readLine(EXPORT_CLI_PREFIX + "> please input query: "); + ioTPrinter.println(sql); + String[] values = sql.trim().split(";"); + for (int i = 0; i < values.length; i++) { + exportData.exportBySql(values[i], i); } } else { - dumpResult(queryCommand, 0); + exportData.exportBySql(queryCommand, 0); } - } catch (IOException e) { ioTPrinter.println("Failed to operate on file, because " + e.getMessage()); exitCode = CODE_ERROR; @@ -299,221 +214,10 @@ public static void main(String[] args) { ioTPrinter.println( "Can not get the timestamp precision from server because " + e.getMessage()); exitCode = CODE_ERROR; - } finally { - if (session != null) { - try { - session.close(); - } catch (IoTDBConnectionException e) { - exitCode = CODE_ERROR; - ioTPrinter.println( - "Encounter an error when closing session, error is: " + e.getMessage()); - } - } } System.exit(exitCode); } - private static Options createTsFileOptions() { - Options options = createExportOptions(); - - Option opFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .longOpt(TARGET_DIR_NAME) - .argName(TARGET_DIR_ARGS_NAME) - .hasArg() - .desc("Target File Directory (required)") - .build(); - options.addOption(opFile); - - Option opOnSuccess = - Option.builder(TARGET_FILE_ARGS) - .longOpt(TARGET_FILE_NAME) - .argName(TARGET_FILE_NAME) - .hasArg() - .desc("Export file name (optional)") - .build(); - options.addOption(opOnSuccess); - - Option opQuery = - Option.builder(QUERY_COMMAND_ARGS) - .longOpt(QUERY_COMMAND_NAME) - .argName(QUERY_COMMAND_ARGS_NAME) - .hasArg() - .desc("The query command that you want to execute. (optional)") - .build(); - options.addOption(opQuery); - - Option opTimeOut = - Option.builder(TIMEOUT_ARGS) - .longOpt(TIMEOUT_NAME) - .argName(TIMEOUT_NAME) - .hasArg() - .desc("Timeout for session query (optional)") - .build(); - options.addOption(opTimeOut); - - return options; - } - - private static Options createCsvOptions() { - Options options = createExportOptions(); - - Option opFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .longOpt(TARGET_DIR_NAME) - .argName(TARGET_DIR_ARGS_NAME) - .hasArg() - .desc("Target File Directory (required)") - .build(); - options.addOption(opFile); - - Option opOnSuccess = - Option.builder(TARGET_FILE_ARGS) - .longOpt(TARGET_FILE_NAME) - .argName(TARGET_FILE_NAME) - .hasArg() - .desc("Export file name (optional)") - .build(); - options.addOption(opOnSuccess); - - Option opDataType = - Option.builder(DATA_TYPE_ARGS) - .longOpt(DATA_TYPE_NAME) - .argName(DATA_TYPE_NAME) - .hasArg() - .desc( - "Will the data type of timeseries be printed in the head line of the CSV file?" - + '\n' - + "You can choose true) or false) . (optional)") - .build(); - options.addOption(opDataType); - - Option opLinesPerFile = - Option.builder(LINES_PER_FILE_ARGS) - .longOpt(LINES_PER_FILE_NAME) - .argName(LINES_PER_FILE_NAME) - .hasArg() - .desc("Lines per dump file.(optional)") - .build(); - options.addOption(opLinesPerFile); - - Option opTimeFormat = - Option.builder(TIME_FORMAT_ARGS) - .longOpt(TIME_FORMAT_NAME) - .argName(TIME_FORMAT_NAME) - .hasArg() - .desc( - "Output time Format in csv file. " - + "You can choose 1) timestamp, number, long 2) ISO8601, default 3) " - + "user-defined pattern like yyyy-MM-dd HH:mm:ss, default ISO8601.\n OutPut timestamp in sql file, No matter what time format is set(optional)") - .build(); - options.addOption(opTimeFormat); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .longOpt(TIMEOUT_NAME) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opQuery = - Option.builder(QUERY_COMMAND_ARGS) - .longOpt(QUERY_COMMAND_NAME) - .argName(QUERY_COMMAND_ARGS_NAME) - .hasArg() - .desc("The query command that you want to execute. (optional)") - .build(); - options.addOption(opQuery); - - Option opTimeOut = - Option.builder(TIMEOUT_ARGS) - .longOpt(TIMEOUT_NAME) - .argName(TIMEOUT_NAME) - .hasArg() - .desc("Timeout for session query (optional)") - .build(); - options.addOption(opTimeOut); - - return options; - } - - private static Options createSqlOptions() { - Options options = createExportOptions(); - - Option opFile = - Option.builder(TARGET_DIR_ARGS) - .required() - .longOpt(TARGET_DIR_NAME) - .argName(TARGET_DIR_ARGS_NAME) - .hasArg() - .desc("Target File Directory (required)") - .build(); - options.addOption(opFile); - - Option opOnSuccess = - Option.builder(TARGET_FILE_ARGS) - .longOpt(TARGET_FILE_NAME) - .argName(TARGET_FILE_NAME) - .hasArg() - .desc("Export file name (optional)") - .build(); - options.addOption(opOnSuccess); - - Option opAligned = - Option.builder(ALIGNED_ARGS) - .longOpt(ALIGNED_NAME) - .argName(ALIGNED_ARGS_NAME) - .hasArgs() - .desc("Whether export to sql of aligned (optional)") - .build(); - options.addOption(opAligned); - - Option opTimeFormat = - Option.builder(TIME_FORMAT_ARGS) - .longOpt(TIME_FORMAT_NAME) - .argName(TIME_FORMAT_NAME) - .hasArg() - .desc( - "Output time Format in csv file. " - + "You can choose 1) timestamp, number, long 2) ISO8601, default 3) " - + "user-defined pattern like yyyy-MM-dd HH:mm:ss, default ISO8601.\n OutPut timestamp in sql file, No matter what time format is set(optional)") - .build(); - options.addOption(opTimeFormat); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .longOpt(TIME_ZONE_NAME) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opQuery = - Option.builder(QUERY_COMMAND_ARGS) - .longOpt(QUERY_COMMAND_NAME) - .argName(QUERY_COMMAND_ARGS_NAME) - .hasArg() - .desc("The query command that you want to execute. (optional)") - .build(); - options.addOption(opQuery); - - Option opTimeOut = - Option.builder(TIMEOUT_ARGS) - .longOpt(TIMEOUT_NAME) - .argName(TIMEOUT_NAME) - .hasArg() - .desc("Timeout for session query (optional)") - .build(); - options.addOption(opTimeOut); - - return options; - } - private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { targetDirectory = checkRequiredArg(TARGET_DIR_ARGS, TARGET_DIR_NAME, commandLine, null); targetFile = commandLine.getOptionValue(TARGET_FILE_ARGS); @@ -549,277 +253,22 @@ private static void parseSpecialParams(CommandLine commandLine) throws ArgsError if (commandLine.getOptionValue(ALIGNED_ARGS) != null) { aligned = Boolean.valueOf(commandLine.getOptionValue(ALIGNED_ARGS)); } - } - - protected static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { - if (timeZoneID != null) { - session.setTimeZone(timeZoneID); - } - zoneId = ZoneId.of(session.getTimeZone()); - } - - /** - * This method will be called, if the query commands are written in a sql file. - * - * @param filePath sql file path - * @throws IOException exception - */ - private static void dumpFromSqlFile(String filePath) throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - String sql; - int index = 0; - while ((sql = reader.readLine()) != null) { - dumpResult(sql, index); - index++; - } - } - } - - /** - * Dump files from database to CSV file. - * - * @param sql export the result of executing the sql - * @param index used to create dump file name - */ - private static void dumpResult(String sql, int index) { - if (EXPORT_SQL_TYPE_NAME.equalsIgnoreCase(exportType)) { - legalCheck(sql); - } - final String path = targetDirectory + targetFile + index; - try { - SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout); - List headers = new ArrayList<>(); - List names = sessionDataSet.getColumnNames(); - List types = sessionDataSet.getColumnTypes(); - if (EXPORT_SQL_TYPE_NAME.equalsIgnoreCase(exportType)) { - writeSqlFile(sessionDataSet, path, names, linesPerFile); - } else { - if (Boolean.TRUE.equals(needDataTypePrinted)) { - for (int i = 0; i < names.size(); i++) { - if (!"Time".equals(names.get(i)) && !"Device".equals(names.get(i))) { - headers.add(String.format("%s(%s)", names.get(i), types.get(i))); - } else { - headers.add(names.get(i)); - } - } - } else { - headers.addAll(names); - } - writeCsvFile(sessionDataSet, path, headers, linesPerFile); - } - sessionDataSet.closeOperationHandle(); - ioTPrinter.println("Export completely!"); - } catch (StatementExecutionException | IoTDBConnectionException | IOException e) { - ioTPrinter.println("Cannot dump result because: " + e.getMessage()); - } - } - - private static void legalCheck(String sql) { - String aggregatePattern = - "\\b(count|sum|avg|extreme|max_value|min_value|first_value|last_value|max_time|min_time|stddev|stddev_pop|stddev_samp|variance|var_pop|var_samp|max_by|min_by)\\b\\s*\\("; - Pattern pattern = Pattern.compile(aggregatePattern, Pattern.CASE_INSENSITIVE); - Matcher matcher = pattern.matcher(sql.toUpperCase(Locale.ROOT)); - if (matcher.find()) { - ioTPrinter.println("The sql you entered is invalid, please don't use aggregate query."); - } - } - - public static String timeTrans(Long time) { - switch (timeFormat) { - case "default": - return RpcUtils.parseLongToDateWithPrecision( - DateTimeFormatter.ISO_OFFSET_DATE_TIME, time, zoneId, timestampPrecision); - case "timestamp": - case "long": - case "number": - return String.valueOf(time); - default: - return ZonedDateTime.ofInstant(Instant.ofEpochMilli(time), zoneId) - .format(DateTimeFormatter.ofPattern(timeFormat)); - } - } - - @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning - public static void writeCsvFile( - SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) - throws IOException, IoTDBConnectionException, StatementExecutionException { - int fileIndex = 0; - boolean hasNext = true; - while (hasNext) { - int i = 0; - final String finalFilePath = filePath + "_" + fileIndex + ".csv"; - final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); - csvPrinterWrapper.printRecord(headers); - while (i++ < linesPerFile) { - if (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - if ("Time".equals(headers.get(0))) { - csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); - } - rowRecord - .getFields() - .forEach( - field -> { - String fieldStringValue = field.getStringValue(); - if (!"null".equals(field.getStringValue())) { - final TSDataType dataType = field.getDataType(); - if ((dataType == TSDataType.TEXT || dataType == TSDataType.STRING) - && !fieldStringValue.startsWith("root.")) { - fieldStringValue = "\"" + fieldStringValue + "\""; - } else if (dataType == TSDataType.BLOB) { - final byte[] v = field.getBinaryV().getValues(); - if (v == null) { - fieldStringValue = null; - } else { - fieldStringValue = BytesUtils.parseBlobByteArrayToString(v); - } - } else if (dataType == TSDataType.DATE) { - final LocalDate dateV = field.getDateV(); - if (dateV == null) { - fieldStringValue = null; - } else { - fieldStringValue = dateV.toString(); - } - } - csvPrinterWrapper.print(fieldStringValue); - } else { - csvPrinterWrapper.print(""); - } - }); - csvPrinterWrapper.println(); - } else { - hasNext = false; - break; - } - } - fileIndex++; - csvPrinterWrapper.flush(); - csvPrinterWrapper.close(); + if (commandLine.getOptionValue(DB_ARGS) != null) { + database = commandLine.getOptionValue(DB_ARGS).toLowerCase(); } - } - - public static void writeSqlFile( - SessionDataSet sessionDataSet, String filePath, List headers, int linesPerFile) - throws IOException, IoTDBConnectionException, StatementExecutionException { - int fileIndex = 0; - String deviceName = null; - boolean writeNull = false; - List seriesList = new ArrayList<>(headers); - if (CollectionUtils.isEmpty(headers) || headers.size() <= 1) { - writeNull = true; + if (commandLine.getOptionValue(TABLE_ARGS) != null) { + table = commandLine.getOptionValue(TABLE_ARGS).toLowerCase(); } else { - if (headers.contains("Device")) { - seriesList.remove("Time"); - seriesList.remove("Device"); - } else { - Path path = new Path(seriesList.get(1), true); - deviceName = path.getDeviceString(); - seriesList.remove("Time"); - for (int i = 0; i < seriesList.size(); i++) { - String series = seriesList.get(i); - path = new Path(series, true); - seriesList.set(i, path.getMeasurement()); - } + if (!sqlDialectTree && StringUtils.isBlank(queryCommand)) { + ioTPrinter.println(queryTableParamRequired); + System.exit(CODE_ERROR); } } - boolean hasNext = true; - while (hasNext) { - int i = 0; - final String finalFilePath = filePath + "_" + fileIndex + ".sql"; - try (FileWriter writer = new FileWriter(finalFilePath)) { - if (writeNull) { - break; - } - while (i++ < linesPerFile) { - if (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - List headersTemp = new ArrayList<>(seriesList); - List timeseries = new ArrayList<>(); - if (headers.contains("Device")) { - deviceName = fields.get(0).toString(); - if (deviceName.startsWith(SYSTEM_DATABASE + ".")) { - continue; - } - for (String header : headersTemp) { - timeseries.add(deviceName + "." + header); - } - } else { - if (headers.get(1).startsWith(SYSTEM_DATABASE + ".")) { - continue; - } - timeseries.addAll(headers); - timeseries.remove(0); - } - String sqlMiddle = null; - if (Boolean.TRUE.equals(aligned)) { - sqlMiddle = " ALIGNED VALUES (" + rowRecord.getTimestamp() + ","; - } else { - sqlMiddle = " VALUES (" + rowRecord.getTimestamp() + ","; - } - List values = new ArrayList<>(); - if (headers.contains("Device")) { - fields.remove(0); - } - for (int index = 0; index < fields.size(); index++) { - RowRecord next = - session - .executeQueryStatement("SHOW TIMESERIES " + timeseries.get(index), timeout) - .next(); - if (ObjectUtils.isNotEmpty(next)) { - List timeseriesList = next.getFields(); - String value = fields.get(index).toString(); - if (value.equals("null")) { - headersTemp.remove(seriesList.get(index)); - continue; - } - final String dataType = timeseriesList.get(3).getStringValue(); - if (TSDataType.TEXT.name().equalsIgnoreCase(dataType) - || TSDataType.STRING.name().equalsIgnoreCase(dataType)) { - values.add("\'" + value + "\'"); - } else if (TSDataType.BLOB.name().equalsIgnoreCase(dataType)) { - final byte[] v = fields.get(index).getBinaryV().getValues(); - if (v == null) { - values.add(null); - } else { - values.add( - BytesUtils.parseBlobByteArrayToString(v).replaceFirst("0x", "X'") + "'"); - } - } else if (TSDataType.DATE.name().equalsIgnoreCase(dataType)) { - final LocalDate dateV = fields.get(index).getDateV(); - if (dateV == null) { - values.add(null); - } else { - values.add("'" + dateV.toString() + "'"); - } - } else { - values.add(value); - } - } else { - headersTemp.remove(seriesList.get(index)); - continue; - } - } - if (CollectionUtils.isNotEmpty(headersTemp)) { - writer.write( - "INSERT INTO " - + deviceName - + "(TIMESTAMP," - + String.join(",", headersTemp) - + ")" - + sqlMiddle - + String.join(",", values) - + ");\n"); - } - - } else { - hasNext = false; - break; - } - } - fileIndex++; - writer.flush(); - } + if (commandLine.getOptionValue(START_TIME_ARGS) != null) { + startTime = commandLine.getOptionValue(START_TIME_ARGS); + } + if (commandLine.getOptionValue(END_TIME_ARGS) != null) { + endTime = commandLine.getOptionValue(END_TIME_ARGS); } } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java new file mode 100644 index 000000000000..fc32755741c5 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTable.java @@ -0,0 +1,326 @@ +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.TableSessionBuilder; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.iotdb.tool.common.Constants.*; + +public class ExportDataTable extends AbstractExportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSession tableSession; + + @Override + public void init() throws IoTDBConnectionException, StatementExecutionException { + tableSession = + new TableSessionBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .username(username) + .password(password) + .database(database) + .build(); + SessionDataSet sessionDataSet = tableSession.executeQueryStatement("show databases", timeout); + List databases = new ArrayList<>(); + while (sessionDataSet.hasNext()) { + databases.add(sessionDataSet.next().getField(0).getStringValue()); + } + if (CollectionUtils.isEmpty(databases) || !databases.contains(database)) { + ioTPrinter.println(String.format(TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + if (StringUtils.isNotBlank(table)) { + sessionDataSet = tableSession.executeQueryStatement("show tables", timeout); + List tables = new ArrayList<>(); + while (sessionDataSet.hasNext()) { + tables.add(sessionDataSet.next().getField(0).getStringValue()); + } + if (CollectionUtils.isEmpty(tables) || !tables.contains(table)) { + ioTPrinter.println(String.format(TARGET_TABLE_NOT_EXIST_MSG, table)); + System.exit(1); + } + } + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + sessionDataSet.close(); + } + } + + @Override + public void exportBySql(String sql, int index) { + if (StringUtils.isNotBlank(sql)) { + if (SQL_SUFFIXS.equalsIgnoreCase(exportType) || TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + legalCheck(sql); + } + } else { + StringBuilder sqlBuilder = new StringBuilder("select * from ").append(table); + if (StringUtils.isNotBlank(startTime) || StringUtils.isNotBlank(endTime)) { + sqlBuilder.append(" where "); + if (StringUtils.isNotBlank(startTime)) { + sqlBuilder.append("time >= ").append(startTime); + } + if (StringUtils.isNotBlank(startTime) && StringUtils.isNotBlank(endTime)) { + sqlBuilder.append(" and "); + } + if (StringUtils.isNotBlank(endTime)) { + sqlBuilder.append("time <= ").append(endTime); + } + } + sql = sqlBuilder.toString(); + } + String path = targetDirectory + targetFile + index; + System.out.println("导出条件sql:" + sql); + try (SessionDataSet sessionDataSet = tableSession.executeQueryStatement(sql, timeout)) { + if (SQL_SUFFIXS.equalsIgnoreCase(exportType)) { + exportToSqlFile(sessionDataSet, path); + } else if (TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + long start = System.currentTimeMillis(); + boolean isComplete = exportToTsFile(sessionDataSet, path + ".tsfile"); + if (isComplete) { + long end = System.currentTimeMillis(); + ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); + } + } else { + exportToCsvFile(sessionDataSet, path); + } + sessionDataSet.closeOperationHandle(); + ioTPrinter.println("Export completely!"); + } catch (StatementExecutionException + | IoTDBConnectionException + | IOException + | WriteProcessException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + + private void exportToSqlFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + StringBuilder sqlBuilder; + List headers = sessionDataSet.getColumnNames(); + String prevSql = "insert into " + table + "(" + StringUtils.join(headers, ",") + ") values("; + final List columnTypes = sessionDataSet.getColumnTypes(); + boolean hasNext = true; + int fileIndex = 0; + while (hasNext) { + final String finalFilePath = filePath + "_" + fileIndex + ".sql"; + int countLine = 0; + try (FileWriter writer = new FileWriter(finalFilePath)) { + if (CollectionUtils.isEmpty(headers) || headers.size() <= 1) { + break; + } + while (countLine++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + sqlBuilder = new StringBuilder(); + List fields = rowRecord.getFields(); + sqlBuilder.append(prevSql); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sqlBuilder.append(","); + } + final TSDataType type = getType(columnTypes.get(i)); + if (TSDataType.TEXT.equals(type) || TSDataType.STRING.equals(type)) { + sqlBuilder.append("\'" + fields.get(i).getObjectValue(type) + "\'"); + } else { + sqlBuilder.append(fields.get(i).getObjectValue(type)); + } + } + sqlBuilder.append(");\n"); + writer.write(sqlBuilder.toString()); + } else { + hasNext = false; + break; + } + } + fileIndex++; + writer.flush(); + } + } + } + + private Boolean exportToTsFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, + IoTDBConnectionException, + StatementExecutionException, + WriteProcessException { + List columnNamesRaw = sessionDataSet.getColumnNames(); + List columnTypesRaw = + sessionDataSet.getColumnTypes().stream().map(t -> getType(t)).collect(Collectors.toList()); + File f = FSFactoryProducer.getFSFactory().getFile(filePath); + if (f.exists()) { + Files.delete(f.toPath()); + } + boolean isEmpty = false; + Map deviceColumnIndices = new HashMap<>(); + List columnSchemas = collectSchemas(columnNamesRaw, deviceColumnIndices); + try (ITsFileWriter tsFileWriter = + new TsFileWriterBuilder() + .file(f) + .tableSchema(new org.apache.tsfile.file.metadata.TableSchema(table, columnSchemas)) + .memoryThreshold(memoryThreshold) + .build()) { + List columnNames = new ArrayList<>(columnNamesRaw); + List columnTypes = new ArrayList<>(columnTypesRaw); + int timeIndex = columnNamesRaw.indexOf(timeColumn.toLowerCase()); + if (timeIndex >= 0) { + columnNames.remove(timeIndex); + columnTypes.remove(timeIndex); + } + Tablet tablet = new Tablet(columnNames, columnTypes); + if (ObjectUtils.isNotEmpty(tablet)) { + writeWithTablets(sessionDataSet, tablet, deviceColumnIndices, tsFileWriter); + } else { + isEmpty = true; + } + } + if (isEmpty) { + ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); + return false; + } + return true; + } + + private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + boolean hasNext = true; + while (hasNext) { + int i = 0; + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + csvPrinterWrapper.printRecord(headers); + while (i++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + if (rowRecord.getTimestamp() != 0) { + csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); + } + rowRecord + .getFields() + .forEach( + field -> { + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + if ((field.getDataType() == TSDataType.TEXT + || field.getDataType() == TSDataType.STRING)) { + fieldStringValue = "\"" + fieldStringValue + "\""; + } + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + } else { + hasNext = false; + break; + } + } + fileIndex++; + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + } + } + + private static void writeWithTablets( + SessionDataSet sessionDataSet, + Tablet tablet, + Map deviceColumnIndices, + ITsFileWriter tsFileWriter) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp( + rowIndex, fields.get(deviceColumnIndices.get(timeColumn.toLowerCase())).getLongV()); + List schemas = tablet.getSchemas(); + for (int i = 0; i < schemas.size(); i++) { + IMeasurementSchema measurementSchema = schemas.get(i); + // -1 for time not in fields + final String measurementName = measurementSchema.getMeasurementName(); + if (timeColumn.equalsIgnoreCase(measurementName)) { + continue; + } + Object value = + fields + .get(deviceColumnIndices.get(measurementName)) + .getObjectValue(measurementSchema.getType()); + tablet.addValue(measurementName, rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(tsFileWriter, tablet); + tablet.initBitMaps(); + tablet.reset(); + } + } + if (tablet.getRowSize() != 0) { + writeToTsFile(tsFileWriter, tablet); + } + } + + private static void writeToTsFile(ITsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + tsFileWriter.write(tablet); + } + + private List collectSchemas( + List columnNames, Map deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + List columnSchemas = new ArrayList<>(); + SessionDataSet sessionDataSet = tableSession.executeQueryStatement("describe " + table); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + final String columnName = rowRecord.getField(0).getStringValue(); + if (timeColumn.equalsIgnoreCase(columnName)) { + continue; + } + columnSchemas.add( + new ColumnSchemaBuilder() + .name(columnName) + .dataType(getType(rowRecord.getField(1).getStringValue())) + .category(getColumnCategory(rowRecord.getField(2).getStringValue())) + .build()); + } + for (int i = 0; i < columnNames.size(); i++) { + deviceColumnIndices.put(columnNames.get(i), i); + } + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + sessionDataSet.close(); + } + return columnSchemas; + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java new file mode 100644 index 000000000000..675e77c606b0 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ExportDataTree.java @@ -0,0 +1,406 @@ +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.Session; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; +import static org.apache.iotdb.tool.common.Constants.*; + +public class ExportDataTree extends AbstractExportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static Session session; + + @Override + public void init() throws IoTDBConnectionException, StatementExecutionException, TException { + session = new Session(host, Integer.parseInt(port), username, password); + session.open(false); + timestampPrecision = session.getTimestampPrecision(); + if (timeZoneID != null) { + session.setTimeZone(timeZoneID); + } + zoneId = ZoneId.of(session.getTimeZone()); + } + + @Override + public void exportBySql(String sql, int index) { + if (SQL_SUFFIXS.equalsIgnoreCase(exportType) || TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + legalCheck(sql); + } + final String path = targetDirectory + targetFile + index; + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout)) { + if (SQL_SUFFIXS.equalsIgnoreCase(exportType)) { + exportToSqlFile(sessionDataSet, path); + } else if (TSFILE_SUFFIXS.equalsIgnoreCase(exportType)) { + long start = System.currentTimeMillis(); + boolean isComplete = exportToTsFile(sessionDataSet, path + ".tsfile"); + if (isComplete) { + long end = System.currentTimeMillis(); + ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); + } + } else { + List headers = new ArrayList<>(); + List names = sessionDataSet.getColumnNames(); + List types = sessionDataSet.getColumnTypes(); + if (Boolean.TRUE.equals(needDataTypePrinted)) { + for (int i = 0; i < names.size(); i++) { + if (!"Time".equals(names.get(i)) && !"Device".equals(names.get(i))) { + headers.add(String.format("%s(%s)", names.get(i), types.get(i))); + } else { + headers.add(names.get(i)); + } + } + } else { + headers.addAll(names); + } + exportToCsvFile(sessionDataSet, path); + } + sessionDataSet.closeOperationHandle(); + ioTPrinter.println("Export completely!"); + } catch (StatementExecutionException + | IoTDBConnectionException + | IOException + | WriteProcessException e) { + ioTPrinter.println("Cannot dump result because: " + e.getMessage()); + } + } + + private void exportToSqlFile(SessionDataSet sessionDataSet, String filePath) + throws IoTDBConnectionException, StatementExecutionException, IOException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + String deviceName = null; + boolean writeNull = false; + List seriesList = new ArrayList<>(headers); + if (CollectionUtils.isEmpty(headers) || headers.size() <= 1) { + writeNull = true; + } else { + if (headers.contains("Device")) { + seriesList.remove("Time"); + seriesList.remove("Device"); + } else { + Path path = new Path(seriesList.get(1), true); + deviceName = path.getDeviceString(); + seriesList.remove("Time"); + for (int i = 0; i < seriesList.size(); i++) { + String series = seriesList.get(i); + path = new Path(series, true); + seriesList.set(i, path.getMeasurement()); + } + } + } + boolean hasNext = true; + while (hasNext) { + int i = 0; + final String finalFilePath = filePath + "_" + fileIndex + ".sql"; + try (FileWriter writer = new FileWriter(finalFilePath)) { + if (writeNull) { + break; + } + while (i++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + List headersTemp = new ArrayList<>(seriesList); + List timeseries = new ArrayList<>(); + if (headers.contains("Device")) { + deviceName = fields.get(0).toString(); + if (deviceName.startsWith(SYSTEM_DATABASE + ".")) { + continue; + } + for (String header : headersTemp) { + timeseries.add(deviceName + "." + header); + } + } else { + if (headers.get(1).startsWith(SYSTEM_DATABASE + ".")) { + continue; + } + timeseries.addAll(headers); + timeseries.remove(0); + } + String sqlMiddle = null; + if (Boolean.TRUE.equals(aligned)) { + sqlMiddle = " ALIGNED VALUES (" + rowRecord.getTimestamp() + ","; + } else { + sqlMiddle = " VALUES (" + rowRecord.getTimestamp() + ","; + } + List values = new ArrayList<>(); + if (headers.contains("Device")) { + fields.remove(0); + } + for (int index = 0; index < fields.size(); index++) { + RowRecord next = + session + .executeQueryStatement("SHOW TIMESERIES " + timeseries.get(index), timeout) + .next(); + if (ObjectUtils.isNotEmpty(next)) { + List timeseriesList = next.getFields(); + String value = fields.get(index).toString(); + if (value.equals("null")) { + headersTemp.remove(seriesList.get(index)); + continue; + } + if ("TEXT".equalsIgnoreCase(timeseriesList.get(3).getStringValue())) { + values.add("\"" + value + "\""); + } else { + values.add(value); + } + } else { + headersTemp.remove(seriesList.get(index)); + continue; + } + } + if (CollectionUtils.isNotEmpty(headersTemp)) { + writer.write( + "INSERT INTO " + + deviceName + + "(TIMESTAMP," + + String.join(",", headersTemp) + + ")" + + sqlMiddle + + String.join(",", values) + + ");\n"); + } + + } else { + hasNext = false; + break; + } + } + fileIndex++; + writer.flush(); + } + } + } + + private static Boolean exportToTsFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, + IoTDBConnectionException, + StatementExecutionException, + WriteProcessException { + List columnNames = sessionDataSet.getColumnNames(); + List columnTypes = sessionDataSet.getColumnTypes(); + File f = FSFactoryProducer.getFSFactory().getFile(filePath); + if (f.exists()) { + Files.delete(f.toPath()); + } + boolean isEmpty = false; + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); + + collectSchemas( + columnNames, columnTypes, deviceSchemaMap, alignedDevices, deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); + + if (!tabletList.isEmpty()) { + writeWithTablets( + sessionDataSet, tabletList, alignedDevices, tsFileWriter, deviceColumnIndices); + tsFileWriter.flush(); + } else { + isEmpty = true; + } + } + if (isEmpty) { + ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); + return false; + } + return true; + } + + private void exportToCsvFile(SessionDataSet sessionDataSet, String filePath) + throws IOException, IoTDBConnectionException, StatementExecutionException { + List headers = sessionDataSet.getColumnNames(); + int fileIndex = 0; + boolean hasNext = true; + while (hasNext) { + int i = 0; + final String finalFilePath = filePath + "_" + fileIndex + ".csv"; + final CSVPrinterWrapper csvPrinterWrapper = new CSVPrinterWrapper(finalFilePath); + csvPrinterWrapper.printRecord(headers); + while (i++ < linesPerFile) { + if (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + if (rowRecord.getTimestamp() != 0) { + csvPrinterWrapper.print(timeTrans(rowRecord.getTimestamp())); + } + rowRecord + .getFields() + .forEach( + field -> { + String fieldStringValue = field.getStringValue(); + if (!"null".equals(field.getStringValue())) { + if ((field.getDataType() == TSDataType.TEXT + || field.getDataType() == TSDataType.STRING) + && !fieldStringValue.startsWith("root.")) { + fieldStringValue = "\"" + fieldStringValue + "\""; + } + csvPrinterWrapper.print(fieldStringValue); + } else { + csvPrinterWrapper.print(""); + } + }); + csvPrinterWrapper.println(); + } else { + hasNext = false; + break; + } + } + fileIndex++; + csvPrinterWrapper.flush(); + csvPrinterWrapper.close(); + } + } + + private static void writeWithTablets( + SessionDataSet sessionDataSet, + List tabletList, + Set alignedDevices, + TsFileWriter tsFileWriter, + Map> deviceColumnIndices) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + + for (Tablet tablet : tabletList) { + String deviceId = tablet.getDeviceId(); + List columnIndices = deviceColumnIndices.get(deviceId); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + List schemas = tablet.getSchemas(); + + for (int i = 0, columnIndicesSize = columnIndices.size(); i < columnIndicesSize; i++) { + Integer columnIndex = columnIndices.get(i); + IMeasurementSchema measurementSchema = schemas.get(i); + // -1 for time not in fields + Object value = fields.get(columnIndex - 1).getObjectValue(measurementSchema.getType()); + // if (value == null) { + // tablet.bitMaps[i].mark(rowIndex); + // } + tablet.addValue(measurementSchema.getMeasurementName(), rowIndex, value); + } + + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + // tablet.initBitMaps(); + tablet.reset(); + } + } + } + + for (Tablet tablet : tabletList) { + if (tablet.getRowSize() != 0) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + } + } + } + + private static void writeToTsFile( + Set deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) + throws IOException, WriteProcessException { + if (deviceFilterSet.contains(tablet.getDeviceId())) { + tsFileWriter.writeAligned(tablet); + } else { + tsFileWriter.writeTree(tablet); + } + } + + private static List constructTablets( + Map> deviceSchemaMap, + Set alignedDevices, + TsFileWriter tsFileWriter) + throws WriteProcessException { + List tabletList = new ArrayList<>(deviceSchemaMap.size()); + for (Map.Entry> stringListEntry : deviceSchemaMap.entrySet()) { + String deviceId = stringListEntry.getKey(); + List schemaList = stringListEntry.getValue(); + Tablet tablet = new Tablet(deviceId, schemaList); + tablet.initBitMaps(); + Path path = new Path(tablet.getDeviceId()); + if (alignedDevices.contains(tablet.getDeviceId())) { + tsFileWriter.registerAlignedTimeseries(path, schemaList); + } else { + tsFileWriter.registerTimeseries(path, schemaList); + } + tabletList.add(tablet); + } + return tabletList; + } + + private static void collectSchemas( + List columnNames, + List columnTypes, + Map> deviceSchemaMap, + Set alignedDevices, + Map> deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < columnNames.size(); i++) { + String column = columnNames.get(i); + if (!column.startsWith("root.")) { + continue; + } + TSDataType tsDataType = getType(columnTypes.get(i)); + Path path = new Path(column, true); + String deviceId = path.getDeviceString(); + // query whether the device is aligned or not + try (SessionDataSet deviceDataSet = + session.executeQueryStatement("show devices " + deviceId, timeout)) { + List deviceList = deviceDataSet.next().getFields(); + if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { + alignedDevices.add(deviceId); + } + } + + // query timeseries metadata + MeasurementSchema measurementSchema = + new MeasurementSchema(path.getMeasurement(), tsDataType); + List seriesList = + session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + measurementSchema.setEncoding(TSEncoding.valueOf(seriesList.get(4).getStringValue())); + measurementSchema.setCompressionType( + CompressionType.valueOf(seriesList.get(5).getStringValue())); + + deviceSchemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); + deviceColumnIndices.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(i); + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java index 753713b9d1f6..cf07faa23310 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java @@ -21,18 +21,18 @@ import org.apache.iotdb.cli.utils.IoTPrinter; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.exception.ArgsErrorException; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; -import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.OptionsUtil; import org.apache.iotdb.tool.tsfile.ImportTsFile; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.csv.CSVParser; @@ -46,305 +46,155 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Stream; -public class ImportData extends AbstractDataTool { - - private static final String FILE_ARGS = "s"; - private static final String FILE_NAME = "source"; - - private static final String ON_SUCCESS_ARGS = "os"; - private static final String ON_SUCCESS_NAME = "on_success"; - - private static final String SUCCESS_DIR_ARGS = "sd"; - private static final String SUCCESS_DIR_NAME = "success_dir"; - - private static final String FAIL_DIR_ARGS = "fd"; - private static final String FAIL_DIR_NAME = "fail_dir"; - - private static final String ON_FAIL_ARGS = "of"; - private static final String ON_FAIL_NAME = "on_fail"; - - private static final String THREAD_NUM_ARGS = "tn"; - private static final String THREAD_NUM_NAME = "thread_num"; - - private static final String BATCH_POINT_SIZE_ARGS = "batch"; - private static final String BATCH_POINT_SIZE_NAME = "batch_size"; - private static final String BATCH_POINT_SIZE_ARGS_NAME = "batch_size"; - - private static final String ALIGNED_ARGS = "aligned"; - private static final String ALIGNED_NAME = "use_aligned"; - private static final String ALIGNED_ARGS_NAME = "use the aligned interface"; +import static org.apache.iotdb.tool.common.Constants.*; - private static final String TIMESTAMP_PRECISION_ARGS = "tp"; - private static final String TIMESTAMP_PRECISION_NAME = "timestamp_precision"; - private static final String TIMESTAMP_PRECISION_ARGS_NAME = "timestamp precision (ms/us/ns)"; - - private static final String TYPE_INFER_ARGS = "ti"; - private static final String TYPE_INFER_NAME = "type_infer"; - - private static final String LINES_PER_FAILED_FILE_ARGS = "lpf"; - private static final String LINES_PER_FAILED_FILE_ARGS_NAME = "lines_per_failed_file"; - - private static final String TSFILEDB_CLI_PREFIX = "Import Data"; - private static final String TSFILEDB_CLI_HEAD = - "Please obtain help information for the corresponding data type based on different parameters, for example:\n" - + "./import_data.sh -help tsfile\n" - + "./import_data.sh -help sql\n" - + "./import_data.sh -help csv"; - - private static String targetPath; - private static String fileType = null; - private static Boolean aligned = false; - private static int threadNum = 8; - private static SessionPool sessionPool; +public class ImportData extends AbstractDataTool { private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static Session session; - private static Options createTsFileOptions() { - Options options = createImportOptions(); - - Option opFile = - Option.builder(FILE_ARGS) - .required() - .longOpt(FILE_NAME) - .argName(FILE_NAME) - .hasArg() - .desc("The local directory path of the script file (folder) to be loaded. (required)") - .build(); - options.addOption(opFile); - - Option opOnSuccess = - Option.builder(ON_SUCCESS_ARGS) - .longOpt(ON_SUCCESS_NAME) - .argName(ON_SUCCESS_NAME) - .required() - .hasArg() - .desc( - "When loading tsfile successfully, do operation on tsfile (and its .resource and .mods files), " - + "optional parameters are none, mv, cp, delete. (required)") - .build(); - options.addOption(opOnSuccess); - - Option opSuccessDir = - Option.builder(SUCCESS_DIR_ARGS) - .longOpt(SUCCESS_DIR_NAME) - .argName(SUCCESS_DIR_NAME) - .hasArg() - .desc("The target folder when 'os' is 'mv' or 'cp'.(optional)") - .build(); - options.addOption(opSuccessDir); - - Option opOnFail = - Option.builder(ON_FAIL_ARGS) - .longOpt(ON_FAIL_NAME) - .argName(ON_FAIL_NAME) - .required() - .hasArg() - .desc( - "When loading tsfile fail, do operation on tsfile (and its .resource and .mods files), " - + "optional parameters are none, mv, cp, delete. (required)") - .build(); - options.addOption(opOnFail); - - Option opFailDir = - Option.builder(FAIL_DIR_ARGS) - .longOpt(FAIL_DIR_NAME) - .argName(FAIL_DIR_NAME) - .hasArg() - .desc("The target folder when 'of' is 'mv' or 'cp'.(optional)") - .build(); - options.addOption(opFailDir); - - Option opThreadNum = - Option.builder(THREAD_NUM_ARGS) - .longOpt(THREAD_NUM_NAME) - .argName(THREAD_NUM_NAME) - .hasArg() - .desc("The number of threads used to import tsfile, default is 8.(optional)") - .build(); - options.addOption(opThreadNum); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .longOpt(TIME_ZONE_NAME) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opTimestampPrecision = - Option.builder(TIMESTAMP_PRECISION_ARGS) - .longOpt(TIMESTAMP_PRECISION_NAME) - .argName(TIMESTAMP_PRECISION_ARGS_NAME) - .hasArg() - .desc("Timestamp precision (ms/us/ns) (optional)") - .build(); - - options.addOption(opTimestampPrecision); - return options; - } - - private static Options createCsvOptions() { - Options options = createImportOptions(); - - Option opFile = - Option.builder(FILE_ARGS) - .longOpt(FILE_NAME) - .argName(FILE_NAME) - .required() - .hasArg() - .desc("The local directory path of the script file (folder) to be loaded. (required)") - .build(); - options.addOption(opFile); - - Option opFailDir = - Option.builder(FAIL_DIR_ARGS) - .longOpt(FAIL_DIR_NAME) - .argName(FAIL_DIR_NAME) - .hasArg() - .desc( - "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)") - .build(); - options.addOption(opFailDir); - - Option opFailedLinesPerFile = - Option.builder(LINES_PER_FAILED_FILE_ARGS) - .longOpt(LINES_PER_FAILED_FILE_ARGS_NAME) - .argName(LINES_PER_FAILED_FILE_ARGS_NAME) - .hasArgs() - .desc("Lines per failed file (optional)") - .build(); - options.addOption(opFailedLinesPerFile); - - Option opAligned = - Option.builder(ALIGNED_ARGS) - .longOpt(ALIGNED_NAME) - .argName(ALIGNED_ARGS_NAME) - .hasArg() - .desc("Whether to use the interface of aligned (optional)") - .build(); - options.addOption(opAligned); - - Option opTypeInfer = - Option.builder(TYPE_INFER_ARGS) - .longOpt(TYPE_INFER_NAME) - .argName(TYPE_INFER_NAME) - .numberOfArgs(5) - .hasArgs() - .valueSeparator(',') - .desc("Define type info by option:\"boolean=text,int=long, ... (optional)") - .build(); - options.addOption(opTypeInfer); - - Option opTimestampPrecision = - Option.builder(TIMESTAMP_PRECISION_ARGS) - .longOpt(TIMESTAMP_PRECISION_NAME) - .argName(TIMESTAMP_PRECISION_ARGS_NAME) - .hasArg() - .desc("Timestamp precision (ms/us/ns) (optional)") - .build(); - - options.addOption(opTimestampPrecision); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .longOpt(TIME_ZONE_NAME) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); - - Option opBatchPointSize = - Option.builder(BATCH_POINT_SIZE_ARGS) - .longOpt(BATCH_POINT_SIZE_NAME) - .argName(BATCH_POINT_SIZE_ARGS_NAME) - .hasArg() - .desc("100000 (optional)") - .build(); - options.addOption(opBatchPointSize); - - Option opThreadNum = - Option.builder(THREAD_NUM_ARGS) - .longOpt(THREAD_NUM_NAME) - .argName(THREAD_NUM_NAME) - .hasArg() - .desc("The number of threads used to import tsfile, default is 8. (optional)") - .build(); - options.addOption(opThreadNum); - return options; - } - - private static Options createSqlOptions() { - Options options = createImportOptions(); - - Option opFile = - Option.builder(FILE_ARGS) - .required() - .longOpt(FILE_NAME) - .argName(FILE_NAME) - .hasArg() - .desc("The local directory path of the script file (folder) to be loaded. (required)") - .build(); - options.addOption(opFile); - - Option opFailDir = - Option.builder(FAIL_DIR_ARGS) - .longOpt(FAIL_DIR_NAME) - .argName(FAIL_DIR_NAME) - .hasArg() - .desc( - "Specifying a directory to save failed file, default YOUR_CSV_FILE_PATH (optional)") - .build(); - options.addOption(opFailDir); - - Option opFailedLinesPerFile = - Option.builder(LINES_PER_FAILED_FILE_ARGS) - .argName(LINES_PER_FAILED_FILE_ARGS_NAME) - .hasArgs() - .desc("Lines per failed file (optional)") - .build(); - options.addOption(opFailedLinesPerFile); - - Option opTimeZone = - Option.builder(TIME_ZONE_ARGS) - .longOpt(TIME_ZONE_NAME) - .argName(TIME_ZONE_NAME) - .hasArg() - .desc("Time Zone eg. +08:00 or -01:00 (optional)") - .build(); - options.addOption(opTimeZone); + public static void main(String[] args) throws IoTDBConnectionException { + Options helpOptions = OptionsUtil.createHelpOptions(); + Options tsFileOptions = OptionsUtil.createImportTsFileOptions(); + Options csvOptions = OptionsUtil.createImportCsvOptions(); + Options sqlOptions = OptionsUtil.createImportSqlOptions(); + HelpFormatter hf = new HelpFormatter(); + hf.setOptionComparator(null); + hf.setWidth(MAX_HELP_CONSOLE_WIDTH); + CommandLine commandLine = null; + CommandLineParser parser = new DefaultParser(); - Option opBatchPointSize = - Option.builder(BATCH_POINT_SIZE_ARGS) - .longOpt(BATCH_POINT_SIZE_NAME) - .argName(BATCH_POINT_SIZE_NAME) - .hasArg() - .desc("100000 (optional)") - .build(); - options.addOption(opBatchPointSize); + if (args == null || args.length == 0) { + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + try { + commandLine = parser.parse(helpOptions, args, true); + } catch (org.apache.commons.cli.ParseException e) { + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + final List argList = Arrays.asList(args); + int helpIndex = argList.indexOf(MINUS + HELP_ARGS); + int sql_dialect = argList.indexOf(MINUS + SQL_DIALECT_ARGS); // -sql_dialect + if (sql_dialect >= 0 + && !SQL_DIALECT_VALUE_TREE.equalsIgnoreCase(argList.get(sql_dialect + 1))) { + final String sqlDialectValue = argList.get(sql_dialect + 1); + if (SQL_DIALECT_VALUE_TABLE.equalsIgnoreCase(sqlDialectValue)) { + sqlDialectTree = false; + tsFileOptions = OptionsUtil.createTableImportTsFileOptions(); + csvOptions = OptionsUtil.createTableImportCsvOptions(); + sqlOptions = OptionsUtil.createTableImportSqlOptions(); + } else { + ioTPrinter.println(String.format("sql_dialect %s is not support", sqlDialectValue)); + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + } + int ftIndex = argList.indexOf(MINUS + FILE_TYPE_ARGS); // -ft + if (ftIndex < 0) { + ftIndex = argList.indexOf(MINUS + FILE_TYPE_NAME); // -file_type + } + if (helpIndex >= 0) { + fileType = argList.get(helpIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + } + } else { + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + } + System.exit(CODE_ERROR); + } else if (ftIndex >= 0) { + fileType = argList.get(ftIndex + 1); + if (StringUtils.isNotBlank(fileType)) { + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(tsFileOptions, args); + // ImportTsFile.importData(commandLine); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, tsFileOptions, null, null, false); + System.exit(CODE_ERROR); + } + } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(csvOptions, args); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, null, csvOptions, null, false); + System.exit(CODE_ERROR); + } + } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { + try { + commandLine = parser.parse(sqlOptions, args); + } catch (ParseException e) { + ioTPrinter.println("Parse error: " + e.getMessage()); + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, null, null, sqlOptions, false); + System.exit(CODE_ERROR); + } + } else { + ioTPrinter.println(String.format("File type %s is not support", fileType)); + printHelpOptions( + IMPORT_CLI_HEAD, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + } else { + ioTPrinter.println( + String.format( + "Invalid args: Required values for option '%s' not provided", FILE_TYPE_NAME)); + System.exit(CODE_ERROR); + } + } else { + ioTPrinter.println( + String.format( + "Invalid args: Required values for option '%s' not provided", FILE_TYPE_NAME)); + System.exit(CODE_ERROR); + } - Option opThreadNum = - Option.builder(THREAD_NUM_ARGS) - .longOpt(THREAD_NUM_NAME) - .argName(THREAD_NUM_NAME) - .hasArgs() - .desc("The number of threads used to import tsfile, default is 8. (optional)") - .build(); - options.addOption(opThreadNum); - return options; + try { + parseBasicParams(commandLine); + String filename = commandLine.getOptionValue(FILE_ARGS); + if (filename == null) { + ioTPrinter.println(IMPORT_CLI_HEAD); + printHelpOptions(null, IMPORT_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); + System.exit(CODE_ERROR); + } + parseSpecialParams(commandLine); + } catch (ArgsErrorException e) { + ioTPrinter.println("Args error: " + e.getMessage()); + System.exit(CODE_ERROR); + } catch (Exception e) { + ioTPrinter.println("Encounter an error, because: " + e.getMessage()); + System.exit(CODE_ERROR); + } + int resultCode = importFromTargetPathAsync(); + System.exit(resultCode); } - /** - * parse optional params - * - * @param commandLine - */ private static void parseSpecialParams(CommandLine commandLine) throws ArgsErrorException { timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS); targetPath = commandLine.getOptionValue(FILE_ARGS); @@ -390,6 +240,93 @@ private static void parseSpecialParams(CommandLine commandLine) throws ArgsError if (commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS) != null) { linesPerFailedFile = Integer.parseInt(commandLine.getOptionValue(LINES_PER_FAILED_FILE_ARGS)); } + if (commandLine.getOptionValue(DB_ARGS) != null) { + database = commandLine.getOptionValue(DB_ARGS); + } + if (commandLine.getOptionValue(TABLE_ARGS) != null) { + table = commandLine.getOptionValue(TABLE_ARGS); + } + if (commandLine.getOptionValue(START_TIME_ARGS) != null) { + startTime = commandLine.getOptionValue(START_TIME_ARGS); + } + if (commandLine.getOptionValue(END_TIME_ARGS) != null) { + endTime = commandLine.getOptionValue(END_TIME_ARGS); + } + try { + isRemoteLoad = !NodeUrlUtils.containsLocalAddress(Collections.singletonList(host)); + if (!sqlDialectTree && isRemoteLoad) { + ioTPrinter.println( + "host: " + host + " is remote load,only local load is supported in table model"); + } + } catch (UnknownHostException e) { + ioTPrinter.println( + "Unknown host: " + host + ". Exception: " + e.getMessage() + ". Will use local load."); + } + final String os = commandLine.getOptionValue(ON_SUCCESS_ARGS); + final String onSuccess = StringUtils.isNotBlank(os) ? os.trim().toLowerCase() : null; + final String of = commandLine.getOptionValue(ON_FAIL_ARGS); + final String onFail = StringUtils.isNotBlank(of) ? of.trim().toLowerCase() : null; + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType) + && (!ImportTsFile.Operation.isValidOperation(onSuccess) + || !ImportTsFile.Operation.isValidOperation(onFail))) { + ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(CODE_ERROR); + boolean isSuccessDirEqualsSourceDir = false; + if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onSuccess)) { + File dir = createSuccessDir(commandLine); + isSuccessDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); + } + + boolean isFailDirEqualsSourceDir = false; + if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onFail)) { + File dir = createFailDir(commandLine); + isFailDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); + } + + successOperation = + ImportTsFile.Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFile.Operation.getOperation(onFail, isFailDirEqualsSourceDir); + } + } + + public static boolean isFileStoreEquals(String pathString, File dir) { + try { + return Objects.equals( + Files.getFileStore(Paths.get(pathString)), Files.getFileStore(dir.toPath())); + } catch (IOException e) { + ioTPrinter.println("IOException when checking file store: " + e.getMessage()); + return false; + } + } + + public static File createSuccessDir(CommandLine commandLine) { + if (commandLine.getOptionValue(SUCCESS_DIR_ARGS) != null) { + successDir = commandLine.getOptionValue(SUCCESS_DIR_ARGS); + } + File file = new File(successDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + ioTPrinter.println(String.format("Failed to create %s %s", SUCCESS_DIR_NAME, successDir)); + System.exit(CODE_ERROR); + } + } + return file; + } + + public static File createFailDir(CommandLine commandLine) { + if (commandLine.getOptionValue(FAIL_DIR_ARGS) != null) { + failDir = commandLine.getOptionValue(FAIL_DIR_ARGS); + } + File file = new File(failDir); + if (!file.isDirectory()) { + if (!file.mkdirs()) { + ioTPrinter.println(String.format("Failed to create %s %s", FAIL_DIR_NAME, failDir)); + System.exit(CODE_ERROR); + } + } + return file; } private static void applyTypeInferArgs(String key, String value) throws ArgsErrorException { @@ -437,188 +374,22 @@ private static void applyTypeInferArgs(String key, String value) throws ArgsErro TYPE_INFER_KEY_DICT.put(key, TYPE_INFER_VALUE_DICT.get(value)); } - public static void main(String[] args) throws IoTDBConnectionException { - Options helpOptions = createHelpOptions(); - Options tsFileOptions = createTsFileOptions(); - Options csvOptions = createCsvOptions(); - Options sqlOptions = createSqlOptions(); - HelpFormatter hf = new HelpFormatter(); - hf.setOptionComparator(null); - hf.setWidth(MAX_HELP_CONSOLE_WIDTH); - CommandLine commandLine = null; - CommandLineParser parser = new DefaultParser(); - - if (args == null || args.length == 0) { - printHelpOptions( - TSFILEDB_CLI_HEAD, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); - System.exit(CODE_ERROR); - } + private static int importFromTargetPathAsync() { try { - commandLine = parser.parse(helpOptions, args, true); - } catch (org.apache.commons.cli.ParseException e) { - printHelpOptions( - TSFILEDB_CLI_HEAD, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); - System.exit(CODE_ERROR); - } - final List argList = Arrays.asList(args); - int helpIndex = argList.indexOf(MINUS + HELP_ARGS); - int ftIndex = argList.indexOf(MINUS + FILE_TYPE_ARGS); - if (ftIndex < 0) { - ftIndex = argList.indexOf(MINUS + FILE_TYPE_NAME); - } - if (helpIndex >= 0) { - fileType = argList.get(helpIndex + 1); - if (StringUtils.isNotBlank(fileType)) { - if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, null, null, false); - } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, csvOptions, null, false); - } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, null, sqlOptions, false); - } else { - ioTPrinter.println(String.format("File type %s is not support", fileType)); - printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); - } + AbstractImportData importData; + if (sqlDialectTree) { + importData = new ImportDataTree(); } else { - printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); + importData = new ImportDataTable(); } - System.exit(CODE_ERROR); - } else if (ftIndex >= 0) { - fileType = argList.get(ftIndex + 1); - if (StringUtils.isNotBlank(fileType)) { - if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { - try { - commandLine = parser.parse(tsFileOptions, args); - ImportTsFile.importData(commandLine); - } catch (ParseException e) { - ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, null, null, false); - System.exit(CODE_ERROR); - } - } else if (CSV_SUFFIXS.equalsIgnoreCase(fileType)) { - try { - commandLine = parser.parse(csvOptions, args); - } catch (ParseException e) { - ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, csvOptions, null, false); - System.exit(CODE_ERROR); - } - } else if (SQL_SUFFIXS.equalsIgnoreCase(fileType)) { - try { - commandLine = parser.parse(sqlOptions, args); - } catch (ParseException e) { - ioTPrinter.println("Parse error: " + e.getMessage()); - printHelpOptions(null, TSFILEDB_CLI_PREFIX, hf, null, null, sqlOptions, false); - System.exit(CODE_ERROR); - } - } else { - ioTPrinter.println(String.format("File type %s is not support", fileType)); - printHelpOptions( - TSFILEDB_CLI_HEAD, - TSFILEDB_CLI_PREFIX, - hf, - tsFileOptions, - csvOptions, - sqlOptions, - true); - System.exit(CODE_ERROR); - } - } else { - ioTPrinter.println( - String.format( - "Invalid args: Required values for option '%s' not provided", FILE_TYPE_NAME)); - System.exit(CODE_ERROR); - } - } else { - ioTPrinter.println( - String.format( - "Invalid args: Required values for option '%s' not provided", FILE_TYPE_NAME)); - System.exit(CODE_ERROR); - } - - try { - parseBasicParams(commandLine); - String filename = commandLine.getOptionValue(FILE_ARGS); - if (filename == null) { - ioTPrinter.println(TSFILEDB_CLI_HEAD); - printHelpOptions( - null, TSFILEDB_CLI_PREFIX, hf, tsFileOptions, csvOptions, sqlOptions, true); - System.exit(CODE_ERROR); - } - parseSpecialParams(commandLine); - } catch (ArgsErrorException e) { - ioTPrinter.println("Args error: " + e.getMessage()); - System.exit(CODE_ERROR); - } catch (Exception e) { - ioTPrinter.println("Encounter an error, because: " + e.getMessage()); - System.exit(CODE_ERROR); - } - final int resultCode = importFromTargetPathAsync(); - if (ImportDataScanTool.getTsFileQueueSize() <= 0) { - System.exit(CODE_OK); - } - asyncImportDataFiles(); - System.exit(resultCode); - } - - private static void asyncImportDataFiles() { - List list = new ArrayList<>(threadNum); - for (int i = 0; i < threadNum; i++) { - final Thread thread = new Thread(new AsyncImportData()); - thread.start(); - list.add(thread); - } - list.forEach( - thread -> { - try { - thread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - ioTPrinter.println("ImportData thread join interrupted: " + e.getMessage()); - } - }); - ioTPrinter.println("Import completely!"); - } - - private static int importFromTargetPathAsync() { - try { - sessionPool = - new SessionPool.Builder() - .host(host) - .port(Integer.parseInt(port)) - .user(username) - .password(password) - .maxSize(threadNum + 1) - .enableCompression(false) - .enableRedirection(false) - .enableAutoFetch(false) - .build(); - sessionPool.setEnableQueryRedirection(false); - AsyncImportData.setAligned(aligned); - AsyncImportData.setSessionPool(sessionPool); - AsyncImportData.setTimeZone(); + importData.init(); ImportDataScanTool.setSourceFullPath(targetPath); final File file = new File(targetPath); if (!file.isFile() && !file.isDirectory()) { ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); System.exit(CODE_ERROR); } - ImportDataScanTool.traverseAndCollectFiles(); - asyncImportDataFiles(); + AbstractImportData.init(importData); return CODE_OK; } catch (InterruptedException e) { ioTPrinter.println(String.format("Import tsfile fail: %s", e.getMessage())); @@ -632,7 +403,8 @@ private static int importFromTargetPathAsync() { /** * Specifying a CSV file or a directory including CSV files that you want to import. This method - * can be offered to console cli to implement importing CSV file by command. + * can be offered to console cli to implement importing CSV file by command. Available for Cli + * import * * @param host * @param port @@ -690,8 +462,15 @@ public static int importFromTargetPath( return CODE_OK; } + private static void setTimeZone() throws IoTDBConnectionException, StatementExecutionException { + if (timeZoneID != null) { + session.setTimeZone(timeZoneID); + } + zoneId = ZoneId.of(session.getTimeZone()); + } + /** - * import the CSV file and load headers and records. + * import the CSV file and load headers and records. Available for Cli import * * @param file the File object of the CSV file that you want to import. */ @@ -728,6 +507,12 @@ private static void importFromSingleFile(Session session, File file) { } } + /** + * import the SQL file and load headers and records. Available for Cli import + * + * @param session + * @param file + */ @SuppressWarnings("java:S2259") private static void importFromSqlFile(Session session, File file) { ArrayList> failedRecords = new ArrayList<>(); diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java index eaed0f5a4b09..8a5f5ce6a238 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataScanTool.java @@ -56,8 +56,4 @@ public static void putToQueue(final String filePath) throws InterruptedException public static void setSourceFullPath(final String sourceFullPath) { ImportDataScanTool.sourceFullPath = sourceFullPath; } - - public static int getTsFileQueueSize() { - return ImportDataScanTool.dataQueue.size(); - } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java new file mode 100644 index 000000000000..1ae1214b920b --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTable.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.iotdb.tool.common.Constants.*; + +public class ImportDataTable extends AbstractImportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static ITableSessionPool sessionPool; + private static Map dataTypes = new HashMap<>(); + private static Map columnCategory = new HashMap<>(); + + public void init() throws InterruptedException { + sessionPool = + new TableSessionPoolBuilder() + .nodeUrls(Collections.singletonList(host + ":" + port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .database(database) + .build(); + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(CODE_ERROR); + } + // checkDataBase + SessionDataSet sessionDataSet = null; + ITableSession session = null; + try { + List databases = new ArrayList<>(); + session = sessionPool.getSession(); + sessionDataSet = session.executeQueryStatement("show databases"); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + databases.add(rowRecord.getField(0).getStringValue()); + } + if (!databases.contains(database)) { + ioTPrinter.println(String.format(TARGET_DATABASE_NOT_EXIST_MSG, database)); + System.exit(1); + } + if (CSV_SUFFIXS.equals(fileType)) { + if (StringUtils.isNotBlank(table)) { + sessionDataSet = session.executeQueryStatement("show tables"); + List tables = new ArrayList<>(); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + tables.add(rowRecord.getField(0).getStringValue()); + } + if (!tables.contains(table)) { + ioTPrinter.println(String.format(TARGET_TABLE_NOT_EXIST_MSG, table)); + System.exit(1); + } + sessionDataSet = session.executeQueryStatement("describe " + table); + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + final String columnName = rowRecord.getField(0).getStringValue(); + final String category = rowRecord.getField(2).getStringValue(); + if (!timeColumn.equalsIgnoreCase(category)) { + dataTypes.put(columnName, getType(rowRecord.getField(1).getStringValue())); + columnCategory.put(columnName, getColumnCategory(category)); + } + } + } else { + ioTPrinter.println(String.format(TARGET_TABLE_NOT_EXIST_MSG, null)); + System.exit(1); + } + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } catch (IoTDBConnectionException e) { + throw new RuntimeException(e); + } finally { + if (ObjectUtils.isNotEmpty(sessionDataSet)) { + try { + sessionDataSet.close(); + } catch (Exception e) { + ; + } + } + if (ObjectUtils.isNotEmpty(session)) { + try { + session.close(); + } catch (IoTDBConnectionException e) { + ; + } + } + } + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + ImportTsFileScanTool.traverseAndCollectFiles(); + ImportTsFileScanTool.addNoResourceOrModsToQueue(); + } else { + + ImportDataScanTool.traverseAndCollectFiles(); + } + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportDataTable(); // 返回子类1的Runnable对象 + } + + protected static void processSuccessFile() { + loadFileSuccessfulNum.increment(); + } + + @SuppressWarnings("java:S2259") + protected void importFromSqlFile(File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + ITableSession session = null; + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String sql; + while ((sql = br.readLine()) != null) { + try { + sql = sql.replace(";", ""); + session = sessionPool.getSession(); + session.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Arrays.asList(sql)); + } finally { + if (ObjectUtils.isNotEmpty(session)) { + try { + session.close(); + } catch (IoTDBConnectionException e) { + ; + } + } + } + } + processSuccessFile(); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + FileWriter writer = null; + try { + writer = new FileWriter(failedFilePath); + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + "\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } finally { + if (ObjectUtils.isNotEmpty(writer)) { + try { + writer.flush(); + writer.close(); + } catch (IOException e) { + ; + } + } + } + } + } + + protected void importFromTsFile(File file) { + final String sql = "load '" + file + "' onSuccess=none "; + try { + sessionPool.getSession().executeNonQueryStatement(sql); + processSuccessFile(file.getPath()); + } catch (final Exception e) { + processFailFile(file.getPath(), e); + } + } + + protected void importFromCsvFile(File file) { + if (file.getName().endsWith(CSV_SUFFIXS) || file.getName().endsWith(TXT_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println("Empty file!"); + return; + } + if (!timeColumn.toLowerCase().equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { + ioTPrinter.println("The first field of header must be `time`!"); + return; + } + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + writeData(headerNames, records.collect(Collectors.toList()), failedFilePath); + processSuccessFile(); + } catch (IOException e) { + ioTPrinter.println("CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); + } + } + + protected void writeData( + List headerNames, List records, String failedFilePath) { + Map headerTypeMap = new HashMap<>(); + Map headerNameMap = new HashMap<>(); + parseHeaders(headerNames, headerTypeMap, headerNameMap); + queryType(headerTypeMap); + List> failedRecords = new ArrayList<>(); + List schemas = new ArrayList<>(); + headerNames.stream() + .forEach( + t -> { + if (dataTypes.keySet().contains(t)) { + schemas.add(new MeasurementSchema(t, dataTypes.get(t))); + } + }); + List headNames = new LinkedList<>(dataTypes.keySet()); + List columnTypes = new LinkedList<>(dataTypes.values()); + List columnCategorys = new LinkedList<>(columnCategory.values()); + Tablet tablet = new Tablet(table, headNames, columnTypes, columnCategorys); + for (CSVRecord recordObj : records) { + boolean isFail = false; + final int rowSize = tablet.getRowSize(); + long rowTimeStamp = parseTimestamp(recordObj.get(0)); + for (String headerName : headerNameMap.keySet()) { + String value = recordObj.get(headerNameMap.get(headerName)); + if (!"".equals(value)) { + TSDataType type; + if (timeColumn.equalsIgnoreCase(headerName)) { + // time列 + tablet.addTimestamp(rowSize, rowTimeStamp); + continue; + } else if (!headerTypeMap.containsKey(headerName)) { + // 未定义列 + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(headerName, type); + int newIndex = headerNames.indexOf(headerName); + if (newIndex >= columnTypes.size()) { + headNames.add(headerName); + columnTypes.add(type); + columnCategorys.add(Tablet.ColumnCategory.FIELD); + } else { + headNames.add(headerName); + columnTypes.add(newIndex, type); + columnCategorys.add(newIndex, Tablet.ColumnCategory.FIELD); + } + writeAndEmptyDataSet(tablet, 3); + tablet = new Tablet(table, headNames, columnTypes, columnCategorys); + tablet.addTimestamp(rowSize, rowTimeStamp); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), headerName, value); + isFail = true; + } + } + type = headerTypeMap.get(headerName); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail = true; + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), headerName, value, type); + } else { + tablet.addValue(headerName, rowSize, valueTrans); + } + } + } + } + if (tablet.getRowSize() >= batchPointSize) { + writeAndEmptyDataSet(tablet, 3); + tablet.reset(); + } + if (isFail) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + } + if (tablet.values.length > 0) { + writeAndEmptyDataSet(tablet, 3); + } + + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + } + + private static void writeAndEmptyDataSet(Tablet tablet, int retryTime) { + try { + sessionPool.getSession().insert(tablet); + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet(tablet, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } + } + + private void parseHeaders( + List headerNames, + Map headerTypeMap, + Map headerNameMap) { + String regex = "(?<=\\()\\S+(?=\\))"; + Pattern pattern = Pattern.compile(regex); + for (String headerName : headerNames) { + headerName = headerName.toLowerCase(); + Matcher matcher = pattern.matcher(headerName); + String type; + String headerNameWithoutType; + if (matcher.find()) { + type = matcher.group(); + headerNameWithoutType = headerName.replace("(" + type + ")", "").replaceAll("\\s+", ""); + headerNameMap.put(headerNameWithoutType, headerName); + headerTypeMap.put(headerNameWithoutType, getType(type)); + } else { + headerNameMap.put(headerName, headerName); + } + } + } + + private void queryType(Map dataType) { + if (MapUtils.isEmpty(dataType)) { + dataType.putAll(dataTypes); + } else { + List noMatch = new ArrayList<>(); + for (String headName : dataType.keySet()) { + if (dataTypes.keySet().contains(headName) + && !dataTypes.get(headName).equals(dataType.get(headName))) { + noMatch.add(headName); + } + } + if (CollectionUtils.isNotEmpty(noMatch)) { + StringBuilder sb = new StringBuilder(); + for (String match : noMatch) { + sb.append(match) + .append(": rawType(") + .append(dataType.get(match)) + .append("), targetType(") + .append(dataTypes.get(match)) + .append(");"); + } + ioTPrinter.println( + "These columns do not match the column types in the target table：" + sb.toString()); + System.exit(1); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java new file mode 100644 index 000000000000..f01e9b700c35 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportDataTree.java @@ -0,0 +1,500 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.data; + +import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.isession.pool.SessionDataSetWrapper; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.tsfile.ImportTsFileScanTool; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.iotdb.tool.common.Constants.*; + +public class ImportDataTree extends AbstractImportData { + + private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); + private static SessionPool sessionPool; + + public void init() + throws InterruptedException, IoTDBConnectionException, StatementExecutionException { + sessionPool = + new SessionPool.Builder() + .host(host) + .port(Integer.parseInt(port)) + .user(username) + .password(password) + .maxSize(threadNum + 1) + .enableCompression(false) + .enableRedirection(false) + .enableAutoFetch(false) + .build(); + sessionPool.setEnableQueryRedirection(false); + if (timeZoneID != null) { + sessionPool.setTimeZone(timeZoneID); + zoneId = sessionPool.getZoneId(); + } + final File file = new File(targetPath); + if (!file.isFile() && !file.isDirectory()) { + ioTPrinter.println(String.format("Source file or directory %s does not exist", targetPath)); + System.exit(CODE_ERROR); + } + if (TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { + ImportTsFileScanTool.setSourceFullPath(targetPath); + ImportTsFileScanTool.traverseAndCollectFiles(); + ImportTsFileScanTool.addNoResourceOrModsToQueue(); + } else { + ImportDataScanTool.setSourceFullPath(targetPath); + ImportDataScanTool.traverseAndCollectFiles(); + } + } + + @Override + protected Runnable getAsyncImportRunnable() { + return new ImportDataTree(); // 返回子类1的Runnable对象 + } + + @SuppressWarnings("java:S2259") + protected void importFromSqlFile(File file) { + ArrayList> failedRecords = new ArrayList<>(); + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String sql; + while ((sql = br.readLine()) != null) { + try { + sessionPool.executeNonQueryStatement(sql); + } catch (IoTDBConnectionException | StatementExecutionException e) { + failedRecords.add(Arrays.asList(sql)); + } + } + processSuccessFile(null); + } catch (IOException e) { + ioTPrinter.println("SQL file read exception because: " + e.getMessage()); + } + if (!failedRecords.isEmpty()) { + FileWriter writer = null; + try { + writer = new FileWriter(failedFilePath); + for (List failedRecord : failedRecords) { + writer.write(failedRecord.get(0).toString() + "\n"); + } + } catch (IOException e) { + ioTPrinter.println("Cannot dump fail result because: " + e.getMessage()); + } finally { + if (ObjectUtils.isNotEmpty(writer)) { + try { + writer.flush(); + writer.close(); + } catch (IOException e) { + ; + } + } + } + } + } + + protected void importFromTsFile(File file) { + final String sql = "load '" + file + "' onSuccess=none "; + try { + sessionPool.executeNonQueryStatement(sql); + processSuccessFile(file.getPath()); + } catch (final Exception e) { + processFailFile(file.getPath(), e); + } + } + + protected void importFromCsvFile(File file) { + if (file.getName().endsWith(CSV_SUFFIXS) || file.getName().endsWith(TXT_SUFFIXS)) { + try { + CSVParser csvRecords = readCsvFile(file.getAbsolutePath()); + List headerNames = csvRecords.getHeaderNames(); + Stream records = csvRecords.stream(); + if (headerNames.isEmpty()) { + ioTPrinter.println("Empty file!"); + return; + } + if (!timeColumn.equalsIgnoreCase(filterBomHeader(headerNames.get(0)))) { + ioTPrinter.println("The first field of header must be `Time`!"); + return; + } + String failedFilePath; + if (failedFileDirectory == null) { + failedFilePath = file.getAbsolutePath() + ".failed"; + } else { + failedFilePath = failedFileDirectory + file.getName() + ".failed"; + } + if (!deviceColumn.equalsIgnoreCase(headerNames.get(1))) { + writeDataAlignedByTime(headerNames, records, failedFilePath); + } else { + writeDataAlignedByDevice(headerNames, records, failedFilePath); + } + processSuccessFile(null); + } catch (IOException | IllegalPathException e) { + ioTPrinter.println("CSV file read exception because: " + e.getMessage()); + } + } else { + ioTPrinter.println("The file name must end with \"csv\" or \"txt\"!"); + } + } + + /** + * if the data is aligned by time, the data will be written by this method. + * + * @param headerNames the header names of CSV file + * @param records the records of CSV file + * @param failedFilePath the directory to save the failed files + */ + @SuppressWarnings("squid:S3776") + protected static void writeDataAlignedByTime( + List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap> deviceAndMeasurementNames = new HashMap<>(); + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, deviceAndMeasurementNames, headerTypeMap, headerNameMap); + + Set devices = deviceAndMeasurementNames.keySet(); + if (headerTypeMap.isEmpty()) { + queryType(devices, headerTypeMap, "Time"); + } + + List deviceIds = new ArrayList<>(); + List times = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + + AtomicReference hasStarted = new AtomicReference<>(false); + AtomicInteger pointSize = new AtomicInteger(0); + + ArrayList> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + if (Boolean.FALSE.equals(hasStarted.get())) { + hasStarted.set(true); + } else if (pointSize.get() >= batchPointSize) { + writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + boolean isFail = false; + for (Map.Entry> entry : deviceAndMeasurementNames.entrySet()) { + String deviceId = entry.getKey(); + List measurementNames = entry.getValue(); + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + for (String measurement : measurementNames) { + String header = deviceId + "." + measurement; + String value = recordObj.get(headerNameMap.get(header)); + if (!"".equals(value)) { + TSDataType type; + if (!headerTypeMap.containsKey(header)) { + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(header, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), header, value); + isFail = true; + } + } + type = headerTypeMap.get(header); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail = true; + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), header, value, type); + } else { + measurements.add(header.replace(deviceId + '.', "")); + types.add(type); + values.add(valueTrans); + pointSize.getAndIncrement(); + } + } + } + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + deviceIds.add(deviceId); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + } + if (isFail) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + }); + if (!deviceIds.isEmpty()) { + writeAndEmptyDataSet(deviceIds, times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + } + + @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning + protected static void writeDataAlignedByDevice( + List headerNames, Stream records, String failedFilePath) + throws IllegalPathException { + HashMap headerTypeMap = new HashMap<>(); + HashMap headerNameMap = new HashMap<>(); + parseHeaders(headerNames, null, headerTypeMap, headerNameMap); + AtomicReference deviceName = new AtomicReference<>(null); + HashSet typeQueriedDevice = new HashSet<>(); + + // the data that interface need + List times = new ArrayList<>(); + List> typesList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + List> measurementsList = new ArrayList<>(); + + AtomicInteger pointSize = new AtomicInteger(0); + + List> failedRecords = new ArrayList<>(); + + records.forEach( + recordObj -> { + // only run in first record + if (deviceName.get() == null) { + deviceName.set(recordObj.get(1)); + } else if (!Objects.equals(deviceName.get(), recordObj.get(1))) { + // if device changed + writeAndEmptyDataSet( + deviceName.get(), times, typesList, valuesList, measurementsList, 3); + deviceName.set(recordObj.get(1)); + pointSize.set(0); + } else if (pointSize.get() >= batchPointSize) { + // insert a batch + writeAndEmptyDataSet( + deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + + // the data of the record + ArrayList types = new ArrayList<>(); + ArrayList values = new ArrayList<>(); + ArrayList measurements = new ArrayList<>(); + + AtomicReference isFail = new AtomicReference<>(false); + + // read data from record + for (Map.Entry headerNameEntry : headerNameMap.entrySet()) { + // headerNameWithoutType is equal to headerName if the CSV column do not have data type. + String headerNameWithoutType = headerNameEntry.getKey(); + String headerName = headerNameEntry.getValue(); + String value = recordObj.get(headerName); + if (!"".equals(value)) { + TSDataType type; + // Get the data type directly if the CSV column have data type. + if (!headerTypeMap.containsKey(headerNameWithoutType)) { + boolean hasResult = false; + // query the data type in iotdb + if (!typeQueriedDevice.contains(deviceName.get())) { + if (headerTypeMap.isEmpty()) { + Set devices = new HashSet<>(); + devices.add(deviceName.get()); + queryType(devices, headerTypeMap, deviceColumn); + } + typeQueriedDevice.add(deviceName.get()); + } + type = typeInfer(value); + if (type != null) { + headerTypeMap.put(headerNameWithoutType, type); + } else { + ioTPrinter.printf( + "Line '%s', column '%s': '%s' unknown type%n", + recordObj.getRecordNumber(), headerNameWithoutType, value); + isFail.set(true); + } + } + type = headerTypeMap.get(headerNameWithoutType); + if (type != null) { + Object valueTrans = typeTrans(value, type); + if (valueTrans == null) { + isFail.set(true); + ioTPrinter.printf( + "Line '%s', column '%s': '%s' can't convert to '%s'%n", + recordObj.getRecordNumber(), headerNameWithoutType, value, type); + } else { + values.add(valueTrans); + measurements.add(headerNameWithoutType); + types.add(type); + pointSize.getAndIncrement(); + } + } + } + } + if (Boolean.TRUE.equals(isFail.get())) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + } + if (!measurements.isEmpty()) { + times.add(parseTimestamp(recordObj.get(timeColumn))); + typesList.add(types); + valuesList.add(values); + measurementsList.add(measurements); + } + }); + if (!times.isEmpty()) { + writeAndEmptyDataSet(deviceName.get(), times, typesList, valuesList, measurementsList, 3); + pointSize.set(0); + } + if (!failedRecords.isEmpty()) { + writeFailedLinesFile(headerNames, failedFilePath, failedRecords); + } + // ioTPrinter.println("Import completely!"); + } + + private static void writeAndEmptyDataSet( + String device, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + sessionPool.insertRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } else { + sessionPool.insertAlignedRecordsOfOneDevice( + device, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet(device, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + } finally { + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + private static void writeAndEmptyDataSet( + List deviceIds, + List times, + List> typesList, + List> valuesList, + List> measurementsList, + int retryTime) { + try { + if (Boolean.FALSE.equals(aligned)) { + sessionPool.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + } else { + sessionPool.insertAlignedRecords(deviceIds, times, measurementsList, typesList, valuesList); + } + } catch (IoTDBConnectionException e) { + if (retryTime > 0) { + writeAndEmptyDataSet( + deviceIds, times, typesList, valuesList, measurementsList, --retryTime); + } + } catch (StatementExecutionException e) { + ioTPrinter.println(INSERT_CSV_MEET_ERROR_MSG + e.getMessage()); + System.exit(1); + } finally { + deviceIds.clear(); + times.clear(); + typesList.clear(); + valuesList.clear(); + measurementsList.clear(); + } + } + + /** + * query data type of timeseries from IoTDB + * + * @param deviceNames + * @param headerTypeMap + * @param alignedType + * @throws IoTDBConnectionException + * @throws StatementExecutionException + */ + private static void queryType( + Set deviceNames, HashMap headerTypeMap, String alignedType) { + for (String deviceName : deviceNames) { + String sql = "show timeseries " + deviceName + ".*"; + try (SessionDataSetWrapper sessionDataSetWrapper = sessionPool.executeQueryStatement(sql)) { + int tsIndex = + sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.TIMESERIES); + int dtIndex = sessionDataSetWrapper.getColumnNames().indexOf(ColumnHeaderConstant.DATATYPE); + while (sessionDataSetWrapper.hasNext()) { + RowRecord rowRecord = sessionDataSetWrapper.next(); + List fields = rowRecord.getFields(); + String timeseries = fields.get(tsIndex).getStringValue(); + String dataType = fields.get(dtIndex).getStringValue(); + if (Objects.equals(alignedType, "Time")) { + headerTypeMap.put(timeseries, getType(dataType)); + } else if (Objects.equals(alignedType, deviceColumn)) { + String[] split = PathUtils.splitPathToDetachedNodes(timeseries); + String measurement = split[split.length - 1]; + headerTypeMap.put(measurement, getType(dataType)); + } + } + } catch (StatementExecutionException | IllegalPathException | IoTDBConnectionException e) { + ioTPrinter.println( + "Meet error when query the type of timeseries because " + e.getMessage()); + System.exit(1); + } + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java index a2e6333fa7b8..18e8b00ea85b 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java @@ -488,7 +488,7 @@ public static Boolean writeWithTablets(SessionDataSet sessionDataSet, String fil if (!tabletList.isEmpty()) { writeWithTablets( sessionDataSet, tabletList, alignedDevices, tsFileWriter, deviceColumnIndices); - tsFileWriter.flush(); + tsFileWriter.flush(); } else { isEmpty = true; }