diff --git a/pom.xml b/pom.xml index 04f37d7e4..d0c7e0860 100644 --- a/pom.xml +++ b/pom.xml @@ -20,6 +20,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + cn.sliew + carp-spring-boot-parent + 0.0.19 + cn.sliew scaleph 2.0.6-SNAPSHOT @@ -120,43 +125,27 @@ true true - 3.3.0 - 2023.0.1 - 3.0.3 - 3.5.6 0.1.1 1.4.2.Final 0.2.0 1.18.24 2.2 - 4.4.0 1.0.1 - 5.8.27 - 3.12.0 - 1.9.4 - 1.9 - 1.19 - 2.11.0 - 1.5.0 0.9.0 4.0.0 21.1.0.0 - 8.0.28 0.3.2 5.1.26 - 42.3.3 0.22.1 1.29.0 1.4.3 4.10.0 - 32.1.3-jre 8.3.8 - 0.0.14 + 0.0.20 1.0.13 1.0.2 3.3.4 3.1.3 - 1.1.0-M1 3.21.5 4.1.82.Final 1.18.1 @@ -166,7 +155,6 @@ 3.1.7 0.4.14 2.44.0.Alpha - 3.27.2 4.3.2 0.4.1 1.5.2 @@ -175,40 +163,6 @@ - - org.springframework.cloud - spring-cloud-dependencies - ${spring.cloud.version} - pom - import - - - org.springframework.boot - spring-boot-dependencies - ${spring.boot.version} - pom - import - - - - - - - - - - io.micrometer - micrometer-bom - ${micrometer.version} - - - com.github.xiaoymin - knife4j-dependencies - ${knife4j.version} - pom - import - - ${project.groupId} scaleph-common @@ -394,39 +348,6 @@ ${project.version} - - - org.mybatis - mybatis-spring - ${mybatis.version} - - - com.baomidou - mybatis-plus-boot-starter - ${mybatis.plus.version} - - - org.mybatis - mybatis-spring - - - - - com.baomidou - mybatis-plus-core - ${mybatis.plus.version} - - - com.baomidou - mybatis-plus-generator - ${mybatis.plus.version} - - - org.apache.velocity - velocity-engine-core - ${velocity.engine.version} - - org.apache.kafka kafka-clients @@ -450,42 +371,6 @@ ${org.mapstruct.version} - - cn.hutool - hutool-all - ${hutool.version} - - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - - - commons-beanutils - commons-beanutils - ${commons.beanutils.version} - - - org.apache.commons - commons-text - ${commons.text.version} - - - org.apache.commons - commons-compress - ${commons.compress.version} - - - commons-io - commons-io - ${commons.io.version} - - - commons-cli - commons-cli - ${commons.cli.version} - - com.github.zafarkhaja java-semver @@ -498,16 +383,6 @@ ${disruptor.version} - - cn.sliew - carp-framework-common - ${carp.version} - - - cn.sliew - carp-framework-dag - ${carp.version} - cn.sliew carp-module-security-spring @@ -528,6 +403,7 @@ carp-module-datasource ${carp.version} + cn.sliew milky-common @@ -544,73 +420,6 @@ ${milky.version} - - com.google.guava - guava - ${guava.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-reload4j - - - org.slf4j - slf4j-log4j12 - - - commons-logging - commons-logging - - - ch.qos.reload4j - reload4j - - - javax.servlet - javax.servlet-api - - - javax.servlet.jsp - jsp-api - - - - - org.apache.hadoop - hadoop-hdfs-client - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - org.apache.hadoop - hadoop-aliyun - ${hadoop.version} - - - org.apache.hadoop - hadoop-aws - ${hadoop.version} - - - org.apache.hive - hive-metastore - ${hive.version} - - org.apache.flink flink-clients @@ -666,28 +475,6 @@ ${flink.version} - - org.apache.pekko - pekko-actor-typed_${scala.binary.version} - ${pekko.version} - - - org.apache.pekko - pekko-cluster-typed_${scala.binary.version} - ${pekko.version} - - - org.apache.pekko - pekko-stream_${scala.binary.version} - ${pekko.version} - - - org.apache.pekko - pekko-serialization-jackson_${scala.binary.version} - ${pekko.version} - - - com.typesafe config @@ -705,53 +492,6 @@ ${okhttp.version} - - io.minio - minio - ${minio.version} - - - - ru.yandex.clickhouse - clickhouse-jdbc - ${clickhouse-jdbc.version} - - - - mysql - mysql-connector-java - ${mysql.version} - runtime - - - org.postgresql - postgresql - ${postgresql.version} - test - com.oracle.database.jdbc ojdbc8 @@ -787,18 +527,6 @@ ${kogito.version} - - org.redisson - redisson-spring-boot-starter - ${redisson.version} - - - - com.alibaba.cola - cola-component-statemachine - ${cola.version} - - org.furyio fury-core diff --git a/scaleph-api/src/main/java/cn/sliew/scaleph/api/config/CarpDataSourceConfig.java b/scaleph-api/src/main/java/cn/sliew/scaleph/api/config/CarpDataSourceConfig.java new file mode 100644 index 000000000..c526568d0 --- /dev/null +++ b/scaleph-api/src/main/java/cn/sliew/scaleph/api/config/CarpDataSourceConfig.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cn.sliew.scaleph.api.config; + +import cn.sliew.carp.framework.mybatis.DataSourceConstants; +import cn.sliew.carp.framework.mybatis.config.CarpMybatisConfig; +import com.baomidou.mybatisplus.autoconfigure.MybatisPlusProperties; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.config.GlobalConfig; +import com.baomidou.mybatisplus.core.handlers.MybatisEnumTypeHandler; +import com.baomidou.mybatisplus.core.toolkit.GlobalConfigUtils; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; +import com.zaxxer.hikari.HikariDataSource; +import org.apache.ibatis.logging.slf4j.Slf4jImpl; +import org.apache.ibatis.session.SqlSessionFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; + +import javax.sql.DataSource; + +@Configuration +public class CarpDataSourceConfig { + + @Autowired + private MybatisPlusInterceptor mybatisPlusInterceptor; + + @Primary + @Bean(DataSourceConstants.DATA_SOURCE_FACTORY) + @ConfigurationProperties(prefix = "spring.datasource.carp") + public DataSource carpDataSource() { + return DataSourceBuilder.create().type(HikariDataSource.class) + .build(); + } + + @Primary + @Bean(DataSourceConstants.TRANSACTION_MANAGER_FACTORY) + public DataSourceTransactionManager carpTransactionManager() { + return new DataSourceTransactionManager(carpDataSource()); + } + + @Primary + @Bean(DataSourceConstants.SQL_SESSION_FACTORY) + public SqlSessionFactory carpSqlSessionFactory() throws Exception { + MybatisSqlSessionFactoryBean factoryBean = new MybatisSqlSessionFactoryBean(); + GlobalConfig globalConfig = GlobalConfigUtils.defaults(); + globalConfig.setMetaObjectHandler(new CarpMybatisConfig.CarpMetaHandler()); + + MybatisPlusProperties props = new MybatisPlusProperties(); + props.setMapperLocations(new String[]{DataSourceConstants.MAPPER_XML_PATH}); + factoryBean.setMapperLocations(props.resolveMapperLocations()); + + MybatisConfiguration configuration = new MybatisConfiguration(); + configuration.setDefaultEnumTypeHandler(MybatisEnumTypeHandler.class); + configuration.setMapUnderscoreToCamelCase(true); + configuration.setLogImpl(Slf4jImpl.class); + factoryBean.setConfiguration(configuration); + factoryBean.setGlobalConfig(globalConfig); + factoryBean.setDataSource(carpDataSource()); + factoryBean.setPlugins(mybatisPlusInterceptor); + return factoryBean.getObject(); + } + +} \ No newline at end of file diff --git a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java index 8c1e1dcd7..4548f99e6 100644 --- a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java +++ b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java @@ -125,6 +125,7 @@ public enum SeaTunnelPluginMapping { SOURCE_STARROCKS(SEATUNNEL, SOURCE, STARROCKS, "connector-starrocks", UNKNOWN, BATCH, SCHEMA_PROJECTION, PARALLELISM, SUPPORT_USER_DEFINED_SPLIT), SINK_STARROCKS(SEATUNNEL, SINK, STARROCKS, "connector-starrocks", ALPHA), SOURCE_HUDI(SEATUNNEL, SOURCE, HUDI, "connector-hudi", BETA, BATCH, EXACTLY_ONCE, PARALLELISM), + SINK_HUDI(SEATUNNEL, SINK, HUDI, "connector-hudi", BETA), SOURCE_ICEBERG(SEATUNNEL, SOURCE, ICEBERG, "connector-iceberg", BETA, BATCH, STREAM, EXACTLY_ONCE, SCHEMA_PROJECTION, PARALLELISM), SINK_ICEBERG(SEATUNNEL, SINK, ICEBERG, "connector-iceberg", UNKNOWN, CDC), SOURCE_PAIMON(SEATUNNEL, SOURCE, PAIMON, "connector-paimon", UNKNOWN, BATCH), diff --git a/scaleph-dao/pom.xml b/scaleph-dao/pom.xml index 84dce38bd..62d3c458d 100644 --- a/scaleph-dao/pom.xml +++ b/scaleph-dao/pom.xml @@ -48,14 +48,8 @@ - mysql - mysql-connector-java - - - com.google.protobuf - protobuf-java - - + com.mysql + mysql-connector-j diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java index 621123541..d8754c445 100644 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java +++ b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java @@ -19,16 +19,15 @@ package cn.sliew.scaleph.ds.gravitino; import cn.sliew.carp.framework.common.dict.datasource.DataSourceType; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.jdbc.MySQLDataSourceProperties; import cn.sliew.carp.module.datasource.service.CarpDsInfoService; import cn.sliew.carp.module.datasource.service.dto.DsInfoDTO; import cn.sliew.milky.common.util.JacksonUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource; import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.client.GravitinoAdminClient; import com.datastrato.gravitino.client.GravitinoMetalake; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -107,12 +106,13 @@ private void doInitDataSource(String metalakeName, DataSourceType type, DsInfoDT } private void initMySQL(GravitinoMetalake metalake, NameIdentifier catalogName, DsInfoDTO dsInfoDTO) { - JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) JacksonUtil.toJsonNode(dsInfoDTO)); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(JacksonUtil.toJsonNode(dsInfoDTO), DataSourceInfo.class); + MySQLDataSourceProperties props = (MySQLDataSourceProperties) dataSourceInfo.getProps(); Map properties = new HashMap<>(); - properties.put("jdbc-driver", dataSource.getDriverClassName()); - properties.put("jdbc-url", dataSource.getUrl()); - properties.put("jdbc-user", dataSource.getUser()); - properties.put("jdbc-password", dataSource.getPassword()); - metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSource.getRemark(), properties); + properties.put("jdbc-driver", props.getDriverClassName()); + properties.put("jdbc-url", props.getUrl()); + properties.put("jdbc-user", props.getUser()); + properties.put("jdbc-password", props.getPassword()); + metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSourceInfo.getRemark(), properties); } } diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/AbstractDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/AbstractDataSource.java deleted file mode 100644 index 906657eeb..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/AbstractDataSource.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.milky.common.util.JacksonUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.jackson.polymorphic.Polymorphic; -import cn.sliew.scaleph.common.jackson.polymorphic.PolymorphicResolver; -import cn.sliew.scaleph.ds.modal.file.*; -import cn.sliew.scaleph.ds.modal.jdbc.*; -import cn.sliew.scaleph.ds.modal.mq.DataHubDataSource; -import cn.sliew.scaleph.ds.modal.mq.KafkaDataSource; -import cn.sliew.scaleph.ds.modal.mq.PulsarDataSource; -import cn.sliew.scaleph.ds.modal.nosql.CassandraDataSource; -import cn.sliew.scaleph.ds.modal.nosql.ElasticsearchDataSource; -import cn.sliew.scaleph.ds.modal.nosql.MongoDBDataSource; -import cn.sliew.scaleph.ds.modal.nosql.RedisDataSource; -import cn.sliew.scaleph.ds.modal.olap.*; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; - -import java.util.Iterator; -import java.util.List; - -@Data -@JsonTypeIdResolver(AbstractDataSource.DataSourceResolver.class) -@JsonIgnoreProperties(ignoreUnknown = true) -public abstract class AbstractDataSource implements Polymorphic { - - @NotNull - @Schema(description = "data source type id") - private Long dsTypeId; - - @Schema(description = "version") - private String version; - - @NotBlank - @Schema(description = "name") - private String name; - - @Schema(description = "remark") - private String remark; - - @Schema(description = "additional props") - private List additionalProps; - - public abstract DataSourceType getType(); - - public abstract DsInfoDTO toDsInfo(); - - public static AbstractDataSource fromDsInfo(ObjectNode jsonNode) { - jsonNode.putPOJO("type", jsonNode.path("dsType").path("type").path("value")); - if (jsonNode.has("props")) { - ObjectNode props = (ObjectNode) jsonNode.get("props"); - Iterator fieldNames = props.fieldNames(); - while (fieldNames.hasNext()) { - String name = fieldNames.next(); - jsonNode.putPOJO(name, props.get(name)); - } - } - return JacksonUtil.toObject(jsonNode, AbstractDataSource.class); - } - - public static final class DataSourceResolver extends PolymorphicResolver { - public DataSourceResolver() { - bindDefault(MySQLDataSource.class); - - bind(DataSourceType.MYSQL, MySQLDataSource.class); - bind(DataSourceType.ORACLE, OracleDataSource.class); - bind(DataSourceType.POSTGRESQL, PostgreSQLDataSource.class); - bind(DataSourceType.SQLSERVER, SQLServerDataSource.class); - bind(DataSourceType.DMDB, DmDBDataSource.class); - bind(DataSourceType.GBASE8A, GBase8aDataSource.class); - bind(DataSourceType.GREENPLUM, GreenplumDataSource.class); - bind(DataSourceType.PHOENIX, PhoenixDataSource.class); - - bind(DataSourceType.REDIS, RedisDataSource.class); - bind(DataSourceType.ELASTICSEARCH, ElasticsearchDataSource.class); - bind(DataSourceType.MONGODB, MongoDBDataSource.class); - bind(DataSourceType.CASSANDRA, CassandraDataSource.class); - - bind(DataSourceType.KAFKA, KafkaDataSource.class); - bind(DataSourceType.PULSAR, PulsarDataSource.class); - bind(DataSourceType.DATAHUB, DataHubDataSource.class); - - bind(DataSourceType.FTP, FtpDataSource.class); - bind(DataSourceType.SFTP, SftpDataSource.class); - bind(DataSourceType.OSS, OSSDataSource.class); - bind(DataSourceType.OSSJINDO, OSSJindoDataSource.class); - bind(DataSourceType.S3, S3DataSource.class); - bind(DataSourceType.HDFS, HDFSDataSource.class); - - bind(DataSourceType.HIVE, HiveDataSource.class); - - bind(DataSourceType.CLICKHOUSE, ClickHouseDataSource.class); - bind(DataSourceType.KUDU, KuduDataSource.class); - bind(DataSourceType.DORIS, DorisDataSource.class); - bind(DataSourceType.STARROCKS, StarRocksDataSource.class); - bind(DataSourceType.MAXCOMPUTE, MaxComputeDataSource.class); - - bind(DataSourceType.IOTDB, IoTDBDataSource.class); - bind(DataSourceType.NEO4J, Neo4jDataSource.class); - - bind(DataSourceType.SOCKET, SocketDataSource.class); - bind(DataSourceType.HTTP, HttpDataSource.class); - bind(DataSourceType.INFLUXDB, InfluxDBDataSource.class); - } - - @Override - protected String typeFromSubtype(Object obj) { - return subTypes.inverse().get(obj.getClass()).getValue(); - } - - @Override - protected Class subTypeFromType(String id) { - Class subType = subTypes.get(DataSourceType.of(id)); - return subType != null ? subType : defaultClass; - } - } - -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/HttpDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/HttpDataSource.java deleted file mode 100644 index bd793dffc..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/HttpDataSource.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class HttpDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "http method") - private String method; - - @NotBlank - @Schema(description = "http url") - private String url; - - @Override - public DataSourceType getType() { - return DataSourceType.HTTP; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("method", method, "url", url); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/InfluxDBDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/InfluxDBDataSource.java deleted file mode 100644 index 92077cb82..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/InfluxDBDataSource.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class InfluxDBDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "InfluxDB url") - private String url; - - @Schema(description = "InfluxDB username") - private String username; - - @Schema(description = "InfluxDB password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.INFLUXDB; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("url", url); - if (StringUtils.hasText(username)) { - props.put("username", username); - } - if (StringUtils.hasText(password)) { - props.put("password", CodecUtil.encrypt(password)); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/IoTDBDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/IoTDBDataSource.java deleted file mode 100644 index 165925b84..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/IoTDBDataSource.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class IoTDBDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "node urls") - private String nodeUrls; - - @NotBlank - @Schema(description = "username") - private String username; - - @NotBlank - @Schema(description = "password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.IOTDB; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("nodeUrls", nodeUrls, "username", username, "password", CodecUtil.encrypt(password)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/Neo4jDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/Neo4jDataSource.java deleted file mode 100644 index d6de3b990..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/Neo4jDataSource.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import static cn.sliew.milky.common.check.Ensures.checkState; - -@Data -@EqualsAndHashCode(callSuper = true) -public class Neo4jDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "uri") - private String uri; - - @Schema(description = "username") - private String username; - - @Schema(description = "password") - private String password; - - @Schema(description = "Bearer Token") - private String bearerToken; - - @Schema(description = "Kerberos Ticket") - private String kerberosTicket; - - @Override - public DataSourceType getType() { - return DataSourceType.NEO4J; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("uri", uri); - if (StringUtils.hasText(username)) { - checkState(StringUtils.hasText(password), () -> "password must provide where username specified"); - props.put("username", username); - props.put("password", CodecUtil.encrypt(password)); - } - if (StringUtils.hasText(bearerToken)) { - props.put("bearerToken", bearerToken); - } - if (StringUtils.hasText(kerberosTicket)) { - props.put("kerberosTicket", kerberosTicket); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/SocketDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/SocketDataSource.java deleted file mode 100644 index 9e6bd8ca7..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/SocketDataSource.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SocketDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "host") - private String host; - - @NotNull - @Schema(description = "port") - private Integer port; - - @Override - public DataSourceType getType() { - return DataSourceType.SOCKET; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("host", host, "port", port); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/FtpDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/FtpDataSource.java deleted file mode 100644 index 59f357b21..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/FtpDataSource.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class FtpDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "host") - private String host; - - @NotNull - @Schema(description = "port") - private Integer port; - - @NotBlank - @Schema(description = "username") - private String username; - - @NotBlank - @Schema(description = "password") - private String password; - - @Schema(description = "connection mode。support active_local or passive_local") - private String connectionMode; - - @Override - public DataSourceType getType() { - return DataSourceType.FTP; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - String encryptedPass = CodecUtil.encrypt(password); - Map props = Map.of("host", host, "port", port, "username", username, "password", encryptedPass, "connectionMode", connectionMode); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/HDFSDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/HDFSDataSource.java deleted file mode 100644 index 3d3055714..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/HDFSDataSource.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class HDFSDataSource extends AbstractDataSource { - - @Schema(description = "hdfs hdfs-site.xml path") - private String hdfsSitePath; - - @NotBlank - @Schema(description = "hdfs fs.defaultFS property") - private String fsDefaultFS; - - @Override - public DataSourceType getType() { - return DataSourceType.HDFS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("fsDefaultFS", fsDefaultFS); - if (StringUtils.hasText(hdfsSitePath)) { - props.put("hdfsSitePath", hdfsSitePath); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSDataSource.java deleted file mode 100644 index f53d953b2..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSDataSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class OSSDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "endpoint") - private String endpoint; - - @NotBlank - @Schema(description = "bucket") - private String bucket; - - @NotBlank - @Schema(description = "access key") - private String accessKey; - - @NotBlank - @Schema(description = "access secret") - private String accessSecret; - - @Override - public DataSourceType getType() { - return DataSourceType.OSS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("endpoint", endpoint, "bucket", bucket, "accessKey", CodecUtil.encrypt(accessKey), "accessSecret", CodecUtil.encrypt(accessSecret)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSJindoDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSJindoDataSource.java deleted file mode 100644 index 40bf6004b..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/OSSJindoDataSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class OSSJindoDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "endpoint") - private String endpoint; - - @NotBlank - @Schema(description = "bucket") - private String bucket; - - @NotBlank - @Schema(description = "access key") - private String accessKey; - - @NotBlank - @Schema(description = "access secret") - private String accessSecret; - - @Override - public DataSourceType getType() { - return DataSourceType.OSS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("endpoint", endpoint, "bucket", bucket, "accessKey", CodecUtil.encrypt(accessKey), "accessSecret", CodecUtil.encrypt(accessSecret)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/S3DataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/S3DataSource.java deleted file mode 100644 index ca380138c..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/S3DataSource.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class S3DataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "bucket") - private String bucket; - - @NotBlank - @Schema(description = "access key") - private String accessKey; - - @NotBlank - @Schema(description = "access secret") - private String accessSecret; - - @Override - public DataSourceType getType() { - return DataSourceType.S3; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("bucket", bucket, "accessKey", CodecUtil.encrypt(accessKey), "accessSecret", CodecUtil.encrypt(accessSecret)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/SftpDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/SftpDataSource.java deleted file mode 100644 index a3dd973a5..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/file/SftpDataSource.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.file; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SftpDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "host") - private String host; - - @NotNull - @Schema(description = "port") - private Integer port; - - @NotBlank - @Schema(description = "username") - private String username; - - @NotBlank - @Schema(description = "password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.SFTP; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - String encryptedPass = CodecUtil.encrypt(password); - Map props = Map.of("host", host, "port", port, "username", username, "password", encryptedPass); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/DmDBDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/DmDBDataSource.java deleted file mode 100644 index 3a70367c0..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/DmDBDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class DmDBDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.DMDB; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GBase8aDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GBase8aDataSource.java deleted file mode 100644 index 078a23076..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GBase8aDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class GBase8aDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.GBASE8A; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GreenplumDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GreenplumDataSource.java deleted file mode 100644 index e315d0d82..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/GreenplumDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class GreenplumDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.GREENPLUM; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/JdbcDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/JdbcDataSource.java deleted file mode 100644 index 0a88fa00c..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/JdbcDataSource.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.jackson.sensitive.Sensitive; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public abstract class JdbcDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "driver class name") - private String driverClassName; - - @NotBlank - @Schema(description = "jdbc url") - private String url; - - @Schema(description = "user") - private String user; - - /** - * {@link Sensitive} not affect Map props - * - * @see Sensitive - */ - @Schema(description = "password") - private String password; - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - String encryptedPass = CodecUtil.encrypt(password); - Map props = Map.of("driverClassName", driverClassName, "url", url, "user", user, "password", encryptedPass); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/MySQLDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/MySQLDataSource.java deleted file mode 100644 index aa24bf412..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/MySQLDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class MySQLDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.MYSQL; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/OracleDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/OracleDataSource.java deleted file mode 100644 index 60c3cb747..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/OracleDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class OracleDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.ORACLE; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PhoenixDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PhoenixDataSource.java deleted file mode 100644 index 6f4de78a7..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PhoenixDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class PhoenixDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.PHOENIX; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PostgreSQLDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PostgreSQLDataSource.java deleted file mode 100644 index fea3ef3bd..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/PostgreSQLDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class PostgreSQLDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.POSTGRESQL; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/SQLServerDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/SQLServerDataSource.java deleted file mode 100644 index 7c6f8f14a..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/jdbc/SQLServerDataSource.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.jdbc; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SQLServerDataSource extends JdbcDataSource { - - @Override - public DataSourceType getType() { - return DataSourceType.SQLSERVER; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/DataHubDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/DataHubDataSource.java deleted file mode 100644 index b9ebaa8e4..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/DataHubDataSource.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.mq; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class DataHubDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "endpoint") - private String endpoint; - - @NotBlank - @Schema(description = "access id") - private String accessId; - - @NotBlank - @Schema(description = "access key") - private String accessKey; - - @Override - public DataSourceType getType() { - return DataSourceType.DATAHUB; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("endpoint", endpoint, "accessId", CodecUtil.encrypt(accessId), "accessKey", CodecUtil.encrypt(accessKey)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/KafkaDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/KafkaDataSource.java deleted file mode 100644 index bce172db5..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/KafkaDataSource.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.mq; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class KafkaDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "kafka bootstrap servers") - private String bootstrapServers; - - @Override - public DataSourceType getType() { - return DataSourceType.KAFKA; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("bootstrapServers", bootstrapServers); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/PulsarDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/PulsarDataSource.java deleted file mode 100644 index f8e6a2ec6..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/mq/PulsarDataSource.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.mq; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class PulsarDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "admin web service url") - private String webServiceUrl; - - @NotBlank - @Schema(description = "client service url") - private String clientServiceUrl; - - @Schema(description = "authentication plugin") - private String authPlugin; - - @Schema(description = "authentication plugin parameters") - private String authParams; - - @Override - public DataSourceType getType() { - return DataSourceType.PULSAR; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("webServiceUrl", webServiceUrl); - props.put("clientServiceUrl", clientServiceUrl); - if (StringUtils.hasText(authPlugin)) { - props.put("authPlugin", authPlugin); - } - if (StringUtils.hasText(authParams)) { - props.put("authParams", CodecUtil.encrypt(authParams)); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/CassandraDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/CassandraDataSource.java deleted file mode 100644 index 235b5b53b..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/CassandraDataSource.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.nosql; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class CassandraDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "host") - private String host; - - @NotBlank - @Schema(description = "keyspace") - private String keyspace; - - @Schema(description = "username") - private String username; - - @Schema(description = "password") - private String password; - - @Schema(description = "datacenter") - private String datacenter; - - @Override - public DataSourceType getType() { - return DataSourceType.CASSANDRA; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("host", host); - props.put("keyspace", keyspace); - if (StringUtils.hasText(username)) { - props.put("username", username); - } - if (StringUtils.hasText(password)) { - props.put("password", CodecUtil.encrypt(password)); - } - if (StringUtils.hasText(datacenter)) { - props.put("datacenter", datacenter); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/ElasticsearchDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/ElasticsearchDataSource.java deleted file mode 100644 index 19d12c066..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/ElasticsearchDataSource.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.nosql; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import static cn.sliew.milky.common.check.Ensures.checkState; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ElasticsearchDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "hosts") - private String hosts; - - @Schema(description = "username") - private String username; - - @Schema(description = "password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.ELASTICSEARCH; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("hosts", hosts); - if (StringUtils.hasText(username)) { - checkState(StringUtils.hasText(password), () -> "password must provide where username specified"); - props.put("username", username); - props.put("password", CodecUtil.encrypt(password)); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/MongoDBDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/MongoDBDataSource.java deleted file mode 100644 index 5a50d4877..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/MongoDBDataSource.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.nosql; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class MongoDBDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "uri") - private String uri; - - @Override - public DataSourceType getType() { - return DataSourceType.MONGODB; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("uri", CodecUtil.encrypt(uri)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/RedisDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/RedisDataSource.java deleted file mode 100644 index 9756f2987..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/nosql/RedisDataSource.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.nosql; - -import cn.sliew.milky.common.util.JacksonUtil; -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.ds.RedisMode; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class RedisDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "host") - private String host; - - @NotNull - @Schema(description = "port") - private Integer port; - - @Schema(description = "user") - private String user; - - @Schema(description = "password") - private String password; - - @Schema(description = "redis mode, single or cluster") - private RedisMode mode; - - @Schema(description = "redis nodes information, used in cluster mode") - private List nodes; - - @Override - public DataSourceType getType() { - return DataSourceType.REDIS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("host", host); - props.put("port", port); - if (StringUtils.hasText(user)) { - props.put("user", user); - } - if (StringUtils.hasText(password)) { - props.put("password", CodecUtil.encrypt(password)); - } - if (mode != null) { - props.put("mode", mode.getValue()); - } - if (CollectionUtils.isEmpty(nodes) == false) { - props.put("nodes", JacksonUtil.toJsonString(nodes)); - } - dto.setProps(props); - return dto; - } - - @Data - public static class Node { - - private String node; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/ClickHouseDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/ClickHouseDataSource.java deleted file mode 100644 index 721720c44..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/ClickHouseDataSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class ClickHouseDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "clickhouse host") - private String host; - - @NotBlank - @Schema(description = "clickhouse database") - private String database; - - @NotBlank - @Schema(description = "clickhouse username") - private String username; - - @NotBlank - @Schema(description = "clickhouse password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.CLICKHOUSE; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("host", host, "database", database, "username", username, "password", CodecUtil.encrypt(password)); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/DorisDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/DorisDataSource.java deleted file mode 100644 index ca0f49bd7..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/DorisDataSource.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class DorisDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "Node Urls") - private String nodeUrls; - - @NotBlank - @Schema(description = "username") - private String username; - - @Schema(description = "password") - private String password; - - @NotNull - @Schema(description = "fenodes query port") - private Integer queryPort; - - @Override - public DataSourceType getType() { - return DataSourceType.DORIS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("nodeUrls", nodeUrls); - if (StringUtils.hasText(username)) { - props.put("username", username); - } - if (StringUtils.hasText(password)) { - props.put("password", CodecUtil.encrypt(password)); - } - if (queryPort != null) { - props.put("queryPort", queryPort); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/HiveDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/HiveDataSource.java deleted file mode 100644 index ab0e1f5dc..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/HiveDataSource.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class HiveDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "hive metastore uri") - private String metastoreUri; - - @Schema(description = "path of hdfs-site.xml") - private String hdfsSitePath; - - @Schema(description = "kerberos keytab path") - private String kerberosKeytabPath; - - @Schema(description = "kerberos principal") - private String kerberosPrincipal; - - @Override - public DataSourceType getType() { - return DataSourceType.HIVE; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("metastoreUri", metastoreUri); - if (StringUtils.hasText(hdfsSitePath)) { - props.put("hdfsSitePath", hdfsSitePath); - } - if (StringUtils.hasText(kerberosKeytabPath)) { - props.put("kerberosKeytabPath", kerberosKeytabPath); - } - if (StringUtils.hasText(kerberosPrincipal)) { - props.put("kerberosPrincipal", kerberosPrincipal); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/KuduDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/KuduDataSource.java deleted file mode 100644 index 2729afbbd..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/KuduDataSource.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class KuduDataSource extends AbstractDataSource { - - @Schema(description = "kudu masters") - private String masters; - - @Override - public DataSourceType getType() { - return DataSourceType.KUDU; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("masters", masters); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/MaxComputeDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/MaxComputeDataSource.java deleted file mode 100644 index 1f0c31ca7..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/MaxComputeDataSource.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class MaxComputeDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "endpoint") - private String endpoint; - - @NotBlank - @Schema(description = "accessId") - private String accessId; - - @NotBlank - @Schema(description = "accesskey") - private String accesskey; - - @Override - public DataSourceType getType() { - return DataSourceType.MAXCOMPUTE; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = Map.of("endpoint", endpoint, "accessId", accessId, "accesskey", accesskey); - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/StarRocksDataSource.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/StarRocksDataSource.java deleted file mode 100644 index 0d0bd8885..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/StarRocksDataSource.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.modal.olap; - -import cn.sliew.scaleph.common.codec.CodecUtil; -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.common.util.BeanUtil; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.service.dto.DsInfoDTO; -import cn.sliew.scaleph.ds.service.dto.DsTypeDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.springframework.util.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Data -@EqualsAndHashCode(callSuper = true) -public class StarRocksDataSource extends AbstractDataSource { - - @NotBlank - @Schema(description = "Node Urls") - private String nodeUrls; - - @NotBlank - @Schema(description = "Base url") - private String baseUrl; - - @NotBlank - @Schema(description = "username") - private String username; - - @NotBlank - @Schema(description = "password") - private String password; - - @Override - public DataSourceType getType() { - return DataSourceType.STARROCKS; - } - - @Override - public DsInfoDTO toDsInfo() { - DsInfoDTO dto = BeanUtil.copy(this, new DsInfoDTO()); - DsTypeDTO dsType = new DsTypeDTO(); - dsType.setId(getDsTypeId()); - dsType.setType(getType()); - dto.setDsType(dsType); - Map props = new HashMap<>(); - props.put("nodeUrls", nodeUrls); - if (StringUtils.hasText(username)) { - props.put("username", username); - } - if (StringUtils.hasText(password)) { - props.put("password", CodecUtil.encrypt(password)); - } - dto.setProps(props); - return dto; - } -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsCategoryDTO.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsCategoryDTO.java deleted file mode 100644 index d42dffe55..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsCategoryDTO.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.service.dto; - -import cn.sliew.scaleph.dao.entity.BaseDO; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode -@Schema(name = "DsCategory对象", description = "data source category") -public class DsCategoryDTO extends BaseDO { - - private static final long serialVersionUID = 1L; - - @Schema(description = "name") - private String name; - - @Schema(description = "order") - private Integer order; - - @Schema(description = "remark") - private String remark; - -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsInfoDTO.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsInfoDTO.java deleted file mode 100644 index 932fda76e..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsInfoDTO.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.service.dto; - -import cn.sliew.scaleph.ds.modal.PropValuePair; -import cn.sliew.scaleph.system.model.BaseDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.List; -import java.util.Map; - -/** - *

- * data source info - *

- */ -@Data -@EqualsAndHashCode -@Schema(name = "DsInfo对象", description = "data source info") -public class DsInfoDTO extends BaseDTO { - - private static final long serialVersionUID = 1L; - - @Schema(description = "data source type id") - private DsTypeDTO dsType; - - @Schema(description = "version") - private String version; - - @Schema(description = "name") - private String name; - - @Schema(description = "props") - private Map props; - - @Schema(description = "additional props") - private List additionalProps; - - @Schema(description = "remark") - private String remark; - -} diff --git a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsTypeDTO.java b/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsTypeDTO.java deleted file mode 100644 index 9ae469465..000000000 --- a/scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/service/dto/DsTypeDTO.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.sliew.scaleph.ds.service.dto; - -import cn.sliew.scaleph.common.dict.job.DataSourceType; -import cn.sliew.scaleph.system.model.BaseDTO; -import io.swagger.v3.oas.annotations.media.Schema; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode -@Schema(name = "DsType对象", description = "data source type") -public class DsTypeDTO extends BaseDTO { - - private static final long serialVersionUID = 1L; - - @Schema(description = "name") - private DataSourceType type; - - @Schema(description = "logo") - private String logo; - - @Schema(description = "order") - private Integer order; - - @Schema(description = "remark") - private String remark; - -} diff --git a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-clickhouse/pom.xml b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-clickhouse/pom.xml index 9556353ea..1e0702bab 100644 --- a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-clickhouse/pom.xml +++ b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-clickhouse/pom.xml @@ -33,7 +33,6 @@ ru.yandex.clickhouse clickhouse-jdbc - ${clickhouse-jdbc.version} diff --git a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-doris/pom.xml b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-doris/pom.xml index 8962f373a..5fcdeac92 100644 --- a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-doris/pom.xml +++ b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-doris/pom.xml @@ -9,7 +9,6 @@ 2.0.6-SNAPSHOT ../pom.xml - scaleph-plugin-datasource-doris @@ -31,10 +30,8 @@ - mysql - mysql-connector-java - ${doris.version} - true + com.mysql + mysql-connector-j diff --git a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-greenplum/pom.xml b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-greenplum/pom.xml index b564eb4da..55270fce3 100644 --- a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-greenplum/pom.xml +++ b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-greenplum/pom.xml @@ -9,7 +9,6 @@ 2.0.6-SNAPSHOT ../pom.xml - scaleph-plugin-datasource-greenplum @@ -44,7 +43,6 @@ org.postgresql postgresql - ${postgresql.version} diff --git a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-mysql/pom.xml b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-mysql/pom.xml index 3d1fd7d94..8d33bfa0a 100644 --- a/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-mysql/pom.xml +++ b/scaleph-plugins/scaleph-plugin-datasource/scaleph-plugin-datasource-mysql/pom.xml @@ -9,7 +9,6 @@ 2.0.6-SNAPSHOT ../pom.xml - scaleph-plugin-datasource-mysql @@ -31,8 +30,8 @@ - mysql - mysql-connector-java + com.mysql + mysql-connector-j diff --git a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/doris/sink/DorisSinkPlugin.java b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/doris/sink/DorisSinkPlugin.java index 763e2e470..b7373ba2a 100644 --- a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/doris/sink/DorisSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/doris/sink/DorisSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.flink.cdc.connectors.doris.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.DorisDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.flink.cdc.FlinkCDCPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.DorisDataSource; import cn.sliew.scaleph.plugin.flink.cdc.FlinkCDCPipilineConnectorPlugin; import cn.sliew.scaleph.plugin.flink.cdc.connectors.CommonProperties; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; @@ -69,10 +70,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - DorisDataSource dataSource = (DorisDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(FENODES.getName(), dataSource.getNodeUrls()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + DorisDataSourceProperties props = (DorisDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(FENODES.getName(), props.getNodeUrls()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/mysql/source/MySQLSourcePlugin.java b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/mysql/source/MySQLSourcePlugin.java index 6b24cf025..51f32a051 100644 --- a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/mysql/source/MySQLSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/mysql/source/MySQLSourcePlugin.java @@ -18,10 +18,11 @@ package cn.sliew.scaleph.plugin.flink.cdc.connectors.mysql.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.jdbc.MySQLDataSourceProperties; import cn.sliew.milky.common.exception.Rethrower; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.flink.cdc.FlinkCDCPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.jdbc.MySQLDataSource; import cn.sliew.scaleph.plugin.flink.cdc.FlinkCDCPipilineConnectorPlugin; import cn.sliew.scaleph.plugin.flink.cdc.connectors.CommonProperties; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; @@ -81,12 +82,14 @@ public ObjectNode createConf() { try { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - MySQLDataSource dataSource = (MySQLDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - URI url = new URI(dataSource.getUrl().replace("jdbc:", "")); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + MySQLDataSourceProperties props = (MySQLDataSourceProperties) dataSourceInfo.getProps(); + + URI url = new URI(props.getUrl().replace("jdbc:", "")); conf.putPOJO(HOSTNAME.getName(), url.getHost()); conf.putPOJO(PORT.getName(), url.getPort()); - conf.putPOJO(USERNAME.getName(), dataSource.getUser()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + conf.putPOJO(USERNAME.getName(), props.getUser()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } catch (URISyntaxException e) { Rethrower.throwAs(e); diff --git a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/starrocks/sink/StarRocksSinkPlugin.java b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/starrocks/sink/StarRocksSinkPlugin.java index 86a5d7982..1cc8cdd6b 100644 --- a/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/starrocks/sink/StarRocksSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-flinkcdc/src/main/java/cn/sliew/scaleph/plugin/flink/cdc/connectors/starrocks/sink/StarRocksSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.flink.cdc.connectors.starrocks.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.StarRocksDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.flink.cdc.FlinkCDCPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.StarRocksDataSource; import cn.sliew.scaleph.plugin.flink.cdc.FlinkCDCPipilineConnectorPlugin; import cn.sliew.scaleph.plugin.flink.cdc.connectors.CommonProperties; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; @@ -73,11 +74,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - StarRocksDataSource dataSource = (StarRocksDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(JDBC_URL.getName(), dataSource.getNodeUrls()); - conf.putPOJO(LOAD_URL.getName(), dataSource.getNodeUrls()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + StarRocksDataSourceProperties props = (StarRocksDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(JDBC_URL.getName(), props.getNodeUrls()); + conf.putPOJO(LOAD_URL.getName(), props.getNodeUrls()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-framework/src/main/java/cn/sliew/scaleph/plugin/framework/property/Parsers.java b/scaleph-plugins/scaleph-plugin-framework/src/main/java/cn/sliew/scaleph/plugin/framework/property/Parsers.java index cb7f3ac4c..aea6f771d 100644 --- a/scaleph-plugins/scaleph-plugin-framework/src/main/java/cn/sliew/scaleph/plugin/framework/property/Parsers.java +++ b/scaleph-plugins/scaleph-plugin-framework/src/main/java/cn/sliew/scaleph/plugin/framework/property/Parsers.java @@ -45,5 +45,9 @@ public enum Parsers { public static final Parser> STRING_ARRAY_PARSER = value -> Arrays.stream(StringUtils.commaDelimitedListToStringArray(value)) .map(String::trim).collect(Collectors.toList()); - public static final Parser JSON_PARSER = value -> JacksonUtil.toJsonNode(value); + public static final Parser JSON_PARSER = value -> { + + System.out.println("value: " + value); + return JacksonUtil.toJsonNode(value); + }; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/SaveModeProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/SaveModeProperties.java index 5bef292a5..ef7ec5b30 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/SaveModeProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/SaveModeProperties.java @@ -32,7 +32,7 @@ public enum SaveModeProperties { .type(PropertyType.STRING) .parser(Parsers.STRING_PARSER) .defaultValue("CREATE_SCHEMA_WHEN_NOT_EXIST") - .allowableValues("CREATE_SCHEMA_WHEN_NOT_EXIST", "RECREATE_SCHEMA", "ERROR_WHEN_SCHEMA_NOT_EXIST") + .allowableValues("CREATE_SCHEMA_WHEN_NOT_EXIST", "RECREATE_SCHEMA", "ERROR_WHEN_SCHEMA_NOT_EXIST", "IGNORE") .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/cassandra/sink/CassandraSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/cassandra/sink/CassandraSinkPlugin.java index afa08a720..fe5afcd54 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/cassandra/sink/CassandraSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/cassandra/sink/CassandraSinkPlugin.java @@ -17,9 +17,10 @@ */ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.cassandra.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.CassandraDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.CassandraDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -67,17 +68,19 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - CassandraDataSource dataSource = (CassandraDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOST.getName(), dataSource.getHost()); - conf.putPOJO(KEYSPACE.getName(), dataSource.getKeyspace()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + CassandraDataSourceProperties props = (CassandraDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOST.getName(), props.getHost()); + conf.putPOJO(KEYSPACE.getName(), props.getKeyspace()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } - if (StringUtils.hasText(dataSource.getDatacenter())) { - conf.putPOJO(DATACENTER.getName(), dataSource.getDatacenter()); + if (StringUtils.hasText(props.getDatacenter())) { + conf.putPOJO(DATACENTER.getName(), props.getDatacenter()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkPlugin.java index 5330947ec..857f4fa06 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.clickhosue.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.ClickHouseDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.ClickHouseDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -69,11 +70,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - ClickHouseDataSource dataSource = (ClickHouseDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOST.getName(), dataSource.getHost()); - conf.putPOJO(DATABASE.getName(), dataSource.getDatabase()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + ClickHouseDataSourceProperties props = (ClickHouseDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOST.getName(), props.getHost()); + conf.putPOJO(DATABASE.getName(), props.getDatabase()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java index 6fa9562d5..030892c59 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.clickhosue.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.ClickHouseDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.ClickHouseDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -65,11 +66,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - ClickHouseDataSource dataSource = (ClickHouseDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOST.getName(), dataSource.getHost()); - conf.putPOJO(DATABASE.getName(), dataSource.getDatabase()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + ClickHouseDataSourceProperties props = (ClickHouseDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOST.getName(), props.getHost()); + conf.putPOJO(DATABASE.getName(), props.getDatabase()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/datahub/sink/DataHubSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/datahub/sink/DataHubSinkPlugin.java index dffb80b6a..fd488be41 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/datahub/sink/DataHubSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/datahub/sink/DataHubSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.datahub.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.mq.DataHubDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.mq.DataHubDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -63,10 +64,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - DataHubDataSource dataSource = (DataHubDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(ACCESS_ID.getName(), dataSource.getAccessId()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + DataHubDataSourceProperties props = (DataHubDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(ACCESS_ID.getName(), props.getAccessId()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/sink/DorisSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/sink/DorisSinkPlugin.java index 10c1df4e7..175ddbc78 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/sink/DorisSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/sink/DorisSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.DorisDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.DorisDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,18 +79,20 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - DorisDataSource dataSource = (DorisDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(FENODES.getName(), dataSource.getNodeUrls()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + DorisDataSourceProperties props = (DorisDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(FENODES.getName(), props.getNodeUrls()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } else { conf.putPOJO(PASSWORD.getName(), ""); } - if (dataSource.getQueryPort() != null) { - conf.putPOJO(QUERY_PORT.getName(), dataSource.getQueryPort()); + if (props.getQueryPort() != null) { + conf.putPOJO(QUERY_PORT.getName(), props.getQueryPort()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java index 6e4cc3bcf..238d0c3a3 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.DorisDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.DorisDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -37,7 +38,6 @@ import java.util.List; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.DorisProperties.*; -import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.DorisProperties.DORIS_BATCH_SIZE; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.source.DorisSourceProperties.*; @AutoService(SeaTunnelConnectorPlugin.class) @@ -72,16 +72,18 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - DorisDataSource dataSource = (DorisDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(FENODES.getName(), dataSource.getNodeUrls()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + DorisDataSourceProperties props = (DorisDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(FENODES.getName(), props.getNodeUrls()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } - if (dataSource.getQueryPort() != null) { - conf.putPOJO(QUERY_PORT.getName(), dataSource.getQueryPort()); + if (props.getQueryPort() != null) { + conf.putPOJO(QUERY_PORT.getName(), props.getQueryPort()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java index 320043e41..9fe39050d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.elasticsearch.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.ElasticsearchDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.ElasticsearchDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -77,12 +78,14 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - ElasticsearchDataSource dataSource = (ElasticsearchDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - List hosts = Arrays.stream(StringUtils.commaDelimitedListToStringArray(dataSource.getHosts())).map(String::trim).collect(Collectors.toList()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + ElasticsearchDataSourceProperties props = (ElasticsearchDataSourceProperties) dataSourceInfo.getProps(); + + List hosts = Arrays.stream(StringUtils.commaDelimitedListToStringArray(props.getHosts())).map(String::trim).collect(Collectors.toList()); conf.putPOJO(HOSTS.getName(), hosts); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java index 060f7e55d..285693ea7 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.elasticsearch.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.ElasticsearchDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.ElasticsearchDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -75,11 +76,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - ElasticsearchDataSource dataSource = (ElasticsearchDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOSTS.getName(), StringUtils.commaDelimitedListToStringArray(dataSource.getHosts())); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + ElasticsearchDataSourceProperties props = (ElasticsearchDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOSTS.getName(), StringUtils.commaDelimitedListToStringArray(props.getHosts())); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/sink/FtpFileSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/sink/FtpFileSinkPlugin.java index f44693aa8..0ba51f84f 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/sink/FtpFileSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/sink/FtpFileSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.ftp.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.FtpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.FtpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -38,7 +39,6 @@ import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileProperties.*; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileSinkProperties.*; -import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileSinkProperties.TMP_PATH; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.ftp.FtpFileProperties.*; @AutoService(SeaTunnelConnectorPlugin.class) @@ -87,13 +87,15 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - FtpDataSource dataSource = (FtpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - conf.putPOJO(USER.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); - if (StringUtils.hasText(dataSource.getConnectionMode())) { - conf.putPOJO(CONNECTION_MODE.getName(), dataSource.getConnectionMode()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + FtpDataSourceProperties props = (FtpDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + conf.putPOJO(USER.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); + if (StringUtils.hasText(props.getConnectionMode())) { + conf.putPOJO(CONNECTION_MODE.getName(), props.getConnectionMode()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java index 962ed5193..f3062d557 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.ftp.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.FtpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.FtpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -79,13 +80,15 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - FtpDataSource dataSource = (FtpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - conf.putPOJO(USER.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); - if (StringUtils.hasText(dataSource.getConnectionMode())) { - conf.putPOJO(CONNECTION_MODE.getName(), dataSource.getConnectionMode()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + FtpDataSourceProperties props = (FtpDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + conf.putPOJO(USER.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); + if (StringUtils.hasText(props.getConnectionMode())) { + conf.putPOJO(CONNECTION_MODE.getName(), props.getConnectionMode()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/sink/HDFSFileSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/sink/HDFSFileSinkPlugin.java index 0e80761be..9c36dc535 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/sink/HDFSFileSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/sink/HDFSFileSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.hdfs.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.HDFSDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.HDFSDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -87,10 +88,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HDFSDataSource dataSource = (HDFSDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(FS_DEFAULT_FS.getName(), dataSource.getFsDefaultFS()); - if (StringUtils.hasText(dataSource.getHdfsSitePath())) { - conf.put(HDFS_SITE_PATH.getName(), dataSource.getHdfsSitePath()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HDFSDataSourceProperties props = (HDFSDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(FS_DEFAULT_FS.getName(), props.getFsDefaultFS()); + if (StringUtils.hasText(props.getHdfsSitePath())) { + conf.put(HDFS_SITE_PATH.getName(), props.getHdfsSitePath()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java index 63b954886..48a45c5f8 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.hdfs.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.HDFSDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.HDFSDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -80,10 +81,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HDFSDataSource dataSource = (HDFSDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(FS_DEFAULT_FS.getName(), dataSource.getFsDefaultFS()); - if (StringUtils.hasText(dataSource.getHdfsSitePath())) { - conf.put(HDFS_SITE_PATH.getName(), dataSource.getHdfsSitePath()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HDFSDataSourceProperties props = (HDFSDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(FS_DEFAULT_FS.getName(), props.getFsDefaultFS()); + if (StringUtils.hasText(props.getHdfsSitePath())) { + conf.put(HDFS_SITE_PATH.getName(), props.getHdfsSitePath()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/sink/OSSSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/sink/OSSSinkPlugin.java index aeb45c764..7e0f3b0ee 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/sink/OSSSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/sink/OSSSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.oss.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.OSSDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.OSSDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -85,11 +86,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - OSSDataSource dataSource = (OSSDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + OSSDataSourceProperties props = (OSSDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/source/OSSSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/source/OSSSourcePlugin.java index 5eaef40f5..2a272a9c9 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/source/OSSSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/source/OSSSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.oss.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.OSSDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.OSSDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,11 +79,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - OSSDataSource dataSource = (OSSDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + OSSDataSourceProperties props = (OSSDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/sink/OSSJindoSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/sink/OSSJindoSinkPlugin.java index 398633c93..33506520c 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/sink/OSSJindoSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/sink/OSSJindoSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.ossjindo.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.OSSJindoDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.OSSJindoDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -85,11 +86,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - OSSJindoDataSource dataSource = (OSSJindoDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + OSSJindoDataSourceProperties props = (OSSJindoDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/source/OSSJindoSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/source/OSSJindoSourcePlugin.java index de1995a25..2d008aba3 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/source/OSSJindoSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/source/OSSJindoSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.ossjindo.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.OSSJindoDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.OSSJindoDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,11 +79,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - OSSJindoDataSource dataSource = (OSSJindoDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + OSSJindoDataSourceProperties props = (OSSJindoDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/sink/S3SinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/sink/S3SinkPlugin.java index f8092324b..8988fb514 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/sink/S3SinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/sink/S3SinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.s3.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.S3DataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.S3DataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -86,10 +87,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - S3DataSource dataSource = (S3DataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + S3DataSourceProperties props = (S3DataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/source/S3SourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/source/S3SourcePlugin.java index 9b36950da..703c27613 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/source/S3SourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/source/S3SourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.s3.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.S3DataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.S3DataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -79,10 +80,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - S3DataSource dataSource = (S3DataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + S3DataSourceProperties props = (S3DataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/sink/SftpFileSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/sink/SftpFileSinkPlugin.java index 725cf05b6..744a94ac6 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/sink/SftpFileSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/sink/SftpFileSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.sftp.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.SftpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.SftpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -85,11 +86,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - SftpDataSource dataSource = (SftpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - conf.putPOJO(USER.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + SftpDataSourceProperties props = (SftpDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + conf.putPOJO(USER.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/source/SftpFileSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/source/SftpFileSourcePlugin.java index 5e76d36c7..3d0cee41b 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/source/SftpFileSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/source/SftpFileSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.sftp.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.SftpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.SftpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,11 +79,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - SftpDataSource dataSource = (SftpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - conf.putPOJO(USER.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + SftpDataSourceProperties props = (SftpDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + conf.putPOJO(USER.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkPlugin.java index 674978e2b..311bdd7ff 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkPlugin.java @@ -53,6 +53,7 @@ public HbaseSinkPlugin() { props.add(WRITE_BUFFER_SIZE); props.add(ENCODING); props.add(HBASE_EXTRA_CONFIG); + props.add(TTL); props.add(CommonProperties.PARALLELISM); props.add(CommonProperties.SOURCE_TABLE_NAME); supportedProperties = Collections.unmodifiableList(props); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkProperties.java index 096a172df..df4ad09f7 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkProperties.java @@ -99,4 +99,12 @@ public enum HbaseSinkProperties { .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); + public static final PropertyDescriptor TTL = new PropertyDescriptor.Builder() + .name("ttl") + .description("Hbase writes data TTL time") + .type(PropertyType.LONG) + .parser(Parsers.LONG_PARSER) + .addValidator(Validators.LONG_VALIDATOR) + .validateAndBuild(); + } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/sink/HiveSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/sink/HiveSinkPlugin.java index c3d29ffc0..2c515bb96 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/sink/HiveSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/sink/HiveSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hive.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.HiveDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.HiveDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -65,16 +66,18 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HiveDataSource dataSource = (HiveDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(METASTORE_URI.getName(), dataSource.getMetastoreUri()); - if (StringUtils.hasText(dataSource.getHdfsSitePath())) { - conf.putPOJO(HDFS_SITE_PATH.getName(), dataSource.getHdfsSitePath()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HiveDataSourceProperties props = (HiveDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(METASTORE_URI.getName(), props.getMetastoreUri()); + if (StringUtils.hasText(props.getHdfsSitePath())) { + conf.putPOJO(HDFS_SITE_PATH.getName(), props.getHdfsSitePath()); } - if (StringUtils.hasText(dataSource.getKerberosKeytabPath())) { - conf.putPOJO(KERBEROS_KEYTAB_PATH.getName(), dataSource.getKerberosKeytabPath()); + if (StringUtils.hasText(props.getKerberosKeytabPath())) { + conf.putPOJO(KERBEROS_KEYTAB_PATH.getName(), props.getKerberosKeytabPath()); } - if (StringUtils.hasText(dataSource.getKerberosPrincipal())) { - conf.putPOJO(KERBEROS_PRINCIPAL.getName(), dataSource.getKerberosPrincipal()); + if (StringUtils.hasText(props.getKerberosPrincipal())) { + conf.putPOJO(KERBEROS_PRINCIPAL.getName(), props.getKerberosPrincipal()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/source/HiveSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/source/HiveSourcePlugin.java index 857b8fe2a..4804abc53 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/source/HiveSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/source/HiveSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hive.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.HiveDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.HiveDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -67,16 +68,18 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HiveDataSource dataSource = (HiveDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(METASTORE_URI.getName(), dataSource.getMetastoreUri()); - if (StringUtils.hasText(dataSource.getHdfsSitePath())) { - conf.putPOJO(HDFS_SITE_PATH.getName(), dataSource.getHdfsSitePath()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HiveDataSourceProperties props = (HiveDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(METASTORE_URI.getName(), props.getMetastoreUri()); + if (StringUtils.hasText(props.getHdfsSitePath())) { + conf.putPOJO(HDFS_SITE_PATH.getName(), props.getHdfsSitePath()); } - if (StringUtils.hasText(dataSource.getKerberosKeytabPath())) { - conf.putPOJO(KERBEROS_KEYTAB_PATH.getName(), dataSource.getKerberosKeytabPath()); + if (StringUtils.hasText(props.getKerberosKeytabPath())) { + conf.putPOJO(KERBEROS_KEYTAB_PATH.getName(), props.getKerberosKeytabPath()); } - if (StringUtils.hasText(dataSource.getKerberosPrincipal())) { - conf.putPOJO(KERBEROS_PRINCIPAL.getName(), dataSource.getKerberosPrincipal()); + if (StringUtils.hasText(props.getKerberosPrincipal())) { + conf.putPOJO(KERBEROS_PRINCIPAL.getName(), props.getKerberosPrincipal()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/sink/HttpSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/sink/HttpSinkPlugin.java index a6994b0a4..c7018869d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/sink/HttpSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/sink/HttpSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.http.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.HttpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.HttpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -68,9 +69,11 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HttpDataSource dataSource = (HttpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - checkState(dataSource.getMethod().equalsIgnoreCase("POST"), () -> "Http sink plugin only support POST method"); - conf.putPOJO(URL.getName(), dataSource.getUrl()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HttpDataSourceProperties props = (HttpDataSourceProperties) dataSourceInfo.getProps(); + + checkState(props.getMethod().equalsIgnoreCase("POST"), () -> "Http sink plugin only support POST method"); + conf.putPOJO(URL.getName(), props.getUrl()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/source/HttpSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/source/HttpSourcePlugin.java index 9ea7c1db2..c0525ab4b 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/source/HttpSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/http/source/HttpSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.http.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.HttpDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.HttpDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -76,9 +77,11 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - HttpDataSource dataSource = (HttpDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(METHOD.getName(), dataSource.getMethod()); - conf.putPOJO(URL.getName(), dataSource.getUrl()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + HttpDataSourceProperties props = (HttpDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(METHOD.getName(), props.getMethod()); + conf.putPOJO(URL.getName(), props.getUrl()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/HudiProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/HudiProperties.java new file mode 100644 index 000000000..501b5344c --- /dev/null +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/HudiProperties.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi; + +import cn.sliew.scaleph.plugin.framework.property.*; + +public enum HudiProperties { + ; + + public static final PropertyDescriptor TABLE_DFS_PATH = new PropertyDescriptor.Builder() + .name("table_dfs_path") + .description("The dfs root path of hudi table") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .properties(Property.Required) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor CONF_FILES_PATH = new PropertyDescriptor.Builder() + .name("conf_files_path") + .description("The environment conf file path list(local path), which used to init hdfs client to read hudi table file.") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder() + .name("table_name") + .description("The name of hudi table.") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .properties(Property.Required) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor DATABASE = new PropertyDescriptor.Builder() + .name("database") + .description("The database of hudi table.") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .properties(Property.Required) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor TABLE_TYPE = new PropertyDescriptor.Builder() + .name("table.type") + .description("The type of hudi table. Now we only support 'cow', 'mor' is not support yet.") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .properties(Property.Required) + .allowableValues("cow") + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); +} diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkPlugin.java new file mode 100644 index 000000000..1ccbfd73a --- /dev/null +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkPlugin.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.sink; + +import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; +import cn.sliew.scaleph.plugin.framework.core.PluginInfo; +import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; +import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; +import cn.sliew.scaleph.plugin.seatunnel.flink.env.CommonProperties; +import com.google.auto.service.AutoService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.SaveModeProperties.SCHEMA_SAVE_MODE; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.HudiProperties.CONF_FILES_PATH; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.HudiProperties.TABLE_DFS_PATH; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.sink.HudiSinkProperties.AUTO_COMMIT; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.sink.HudiSinkProperties.TABLE_LIST; + +@AutoService(SeaTunnelConnectorPlugin.class) +public class HudiSinkPlugin extends SeaTunnelConnectorPlugin { + + public HudiSinkPlugin() { + this.pluginInfo = new PluginInfo(getIdentity(), + "Hudi sink connector", + HudiSinkPlugin.class.getName()); + + final List props = new ArrayList<>(); + props.add(TABLE_DFS_PATH); + props.add(CONF_FILES_PATH); + props.add(AUTO_COMMIT); + props.add(SCHEMA_SAVE_MODE); + props.add(TABLE_LIST); + props.add(CommonProperties.PARALLELISM); + props.add(CommonProperties.RESULT_TABLE_NAME); + supportedProperties = Collections.unmodifiableList(props); + } + + @Override + protected SeaTunnelPluginMapping getPluginMapping() { + return SeaTunnelPluginMapping.SINK_HUDI; + } +} diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkProperties.java new file mode 100644 index 000000000..121b9f3a7 --- /dev/null +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/sink/HudiSinkProperties.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.sink; + +import cn.sliew.scaleph.plugin.framework.property.Parsers; +import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; +import cn.sliew.scaleph.plugin.framework.property.PropertyType; +import cn.sliew.scaleph.plugin.framework.property.Validators; +import com.fasterxml.jackson.databind.JsonNode; + +public enum HudiSinkProperties { + ; + + public static final PropertyDescriptor TABLE_LIST = new PropertyDescriptor.Builder() + .name("table_list") + .description("Table list.") + .type(PropertyType.OBJECT) + .parser(Parsers.JSON_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor AUTO_COMMIT = new PropertyDescriptor.Builder() + .name("auto_commit") + .description("Automatic transaction commit is enabled by default.") + .type(PropertyType.BOOLEAN) + .parser(Parsers.BOOLEAN_PARSER) + .addValidator(Validators.BOOLEAN_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor RECORD_KEY_FIELDS = new PropertyDescriptor.Builder() + .name("record_key_fields") + .description("The record key fields of hudi table, its are used to generate record key") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor PARTITION_FIELDS = new PropertyDescriptor.Builder() + .name("partition_fields") + .description("The partition key fields of hudi table, its are used to generate partition") + .type(PropertyType.BOOLEAN) + .parser(Parsers.BOOLEAN_PARSER) + .defaultValue("false") + .addValidator(Validators.BOOLEAN_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor INDEX_TYPE = new PropertyDescriptor.Builder() + .name("index_type") + .description("The index type of hudi table.") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .allowableValues("BLOOM", "SIMPLE", "GLOBAL SIMPLE") + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor INDEX_CLASS_NAME = new PropertyDescriptor.Builder() + .name("index_class_name") + .description("The customized index classpath of hudi table") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor RECORD_BYTE_SIZE = new PropertyDescriptor.Builder() + .name("record_byte_size") + .description("The byte size of each record") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor OP_TYPE = new PropertyDescriptor.Builder() + .name("op_type") + .description("The operation type of hudi table") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .allowableValues("insert", "upsert", "bulk_insert") + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor BATCH_INTERVAL_MS = new PropertyDescriptor.Builder() + .name("batch_interval_ms") + .description("The interval time of batch write to hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() + .name("batch_size") + .description("The size of batch write to hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor INSERT_SHUFFLE_PARALLELISM = new PropertyDescriptor.Builder() + .name("insert_shuffle_parallelism") + .description("The parallelism of insert data to hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor UPSERT_SHUFFLE_PARALLELISM = new PropertyDescriptor.Builder() + .name("upsert_shuffle_parallelism") + .description("The parallelism of upsert data to hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor MIN_COMMITS_TO_KEEP = new PropertyDescriptor.Builder() + .name("min_commits_to_keep") + .description("The min commits to keep of hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor MAX_COMMITS_TO_KEEP = new PropertyDescriptor.Builder() + .name("max_commits_to_keep") + .description("The max commits to keep of hudi table.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR) + .validateAndBuild(); + + +} diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/source/HudiSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/source/HudiSourcePlugin.java index 1496be804..5ffc1444b 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/source/HudiSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hudi/source/HudiSourcePlugin.java @@ -31,7 +31,9 @@ import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hudi.source.HudiSourceProperties.*; -@AutoService(SeaTunnelConnectorPlugin.class) +/** + * fixme 移除，2.3.8 文档找不到了 + */ public class HudiSourcePlugin extends SeaTunnelConnectorPlugin { public HudiSourcePlugin() { diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/sink/InfluxDBSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/sink/InfluxDBSinkPlugin.java index 9849851a7..2d8a1189f 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/sink/InfluxDBSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/sink/InfluxDBSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.influxdb.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.InfluxDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.InfluxDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -71,13 +72,15 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - InfluxDBDataSource dataSource = (InfluxDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URL.getName(), dataSource.getUrl()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + InfluxDBDataSourceProperties props = (InfluxDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URL.getName(), props.getUrl()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/source/InfluxDBSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/source/InfluxDBSourcePlugin.java index 250447d34..7749220be 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/source/InfluxDBSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/influxdb/source/InfluxDBSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.influxdb.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.InfluxDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.InfluxDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -72,13 +73,15 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - InfluxDBDataSource dataSource = (InfluxDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URL.getName(), dataSource.getUrl()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + InfluxDBDataSourceProperties props = (InfluxDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URL.getName(), props.getUrl()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/sink/IoTDBSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/sink/IoTDBSinkPlugin.java index 75c8c0865..c396e2d52 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/sink/IoTDBSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/sink/IoTDBSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.iotdb.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.IoTDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.IoTDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -74,10 +75,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - IoTDBDataSource dataSource = (IoTDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(NODE_URLS.getName(), dataSource.getNodeUrls()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + IoTDBDataSourceProperties props = (IoTDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(NODE_URLS.getName(), props.getNodeUrls()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/source/IoTDBSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/source/IoTDBSourcePlugin.java index e5497c458..ccfe8a75d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/source/IoTDBSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/source/IoTDBSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.iotdb.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.IoTDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.IoTDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -71,10 +72,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - IoTDBDataSource dataSource = (IoTDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(NODE_URLS.getName(), dataSource.getNodeUrls()); - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + IoTDBDataSourceProperties props = (IoTDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(NODE_URLS.getName(), props.getNodeUrls()); + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/sink/JdbcSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/sink/JdbcSinkPlugin.java index 2c8ce21b1..5cdd79ab2 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/sink/JdbcSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/sink/JdbcSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.jdbc.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.jdbc.JdbcDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -83,11 +84,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(URL.getName(), dataSource.getUrl()); - conf.putPOJO(DRIVER.getName(), dataSource.getDriverClassName()); - conf.putPOJO(USER.getName(), dataSource.getUser()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + JdbcDataSourceProperties props = (JdbcDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(URL.getName(), props.getUrl()); + conf.putPOJO(DRIVER.getName(), props.getDriverClassName()); + conf.putPOJO(USER.getName(), props.getUser()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/source/JdbcSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/source/JdbcSourcePlugin.java index cce7c0577..ec6087e54 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/source/JdbcSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/source/JdbcSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.jdbc.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.jdbc.JdbcDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -77,11 +78,13 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.put(URL.getName(), dataSource.getUrl()); - conf.putPOJO(DRIVER.getName(), dataSource.getDriverClassName()); - conf.putPOJO(USER.getName(), dataSource.getUser()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + JdbcDataSourceProperties props = (JdbcDataSourceProperties) dataSourceInfo.getProps(); + + conf.put(URL.getName(), props.getUrl()); + conf.putPOJO(DRIVER.getName(), props.getDriverClassName()); + conf.putPOJO(USER.getName(), props.getUser()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/sink/KafkaSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/sink/KafkaSinkPlugin.java index db6f104c3..259dc01b9 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/sink/KafkaSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/sink/KafkaSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.kafka.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.mq.KafkaDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.mq.KafkaDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -69,8 +70,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - KafkaDataSource dataSource = (KafkaDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(BOOTSTRAP_SERVERS.getName(), dataSource.getBootstrapServers()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + KafkaDataSourceProperties props = (KafkaDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(BOOTSTRAP_SERVERS.getName(), props.getBootstrapServers()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/source/KafkaSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/source/KafkaSourcePlugin.java index c80374b58..3bf1e0c50 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/source/KafkaSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kafka/source/KafkaSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.kafka.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.mq.KafkaDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.mq.KafkaDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,8 +79,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - KafkaDataSource dataSource = (KafkaDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(BOOTSTRAP_SERVERS.getName(), dataSource.getBootstrapServers()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + KafkaDataSourceProperties props = (KafkaDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(BOOTSTRAP_SERVERS.getName(), props.getBootstrapServers()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkPlugin.java index eb3d74fe0..7ccce1431 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.kudu.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.KuduDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.KuduDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -74,8 +75,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - KuduDataSource dataSource = (KuduDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(KUDU_MASTER.getName(), dataSource.getMasters()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + KuduDataSourceProperties props = (KuduDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(KUDU_MASTER.getName(), props.getMasters()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourcePlugin.java index a3301a6dd..68cece65a 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.kudu.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.KuduDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.KuduDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -72,8 +73,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - KuduDataSource dataSource = (KuduDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(KUDU_MASTER.getName(), dataSource.getMasters()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + KuduDataSourceProperties props = (KuduDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(KUDU_MASTER.getName(), props.getMasters()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/sink/MaxComputeSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/sink/MaxComputeSinkPlugin.java index e987cc08a..46a7f03cb 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/sink/MaxComputeSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/sink/MaxComputeSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.maxcompute.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.MaxComputeDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.MaxComputeDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -65,10 +66,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - MaxComputeDataSource dataSource = (MaxComputeDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(ACCESS_ID.getName(), dataSource.getAccessId()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccesskey()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + MaxComputeDataSourceProperties props = (MaxComputeDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(ACCESS_ID.getName(), props.getAccessId()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccesskey()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/source/MaxComputeSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/source/MaxComputeSourcePlugin.java index 5cf39aa1f..9ecbb209b 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/source/MaxComputeSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/maxcompute/source/MaxComputeSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.maxcompute.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.MaxComputeDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.MaxComputeDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -67,10 +68,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - MaxComputeDataSource dataSource = (MaxComputeDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(ENDPOINT.getName(), dataSource.getEndpoint()); - conf.putPOJO(ACCESS_ID.getName(), dataSource.getAccessId()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccesskey()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + MaxComputeDataSourceProperties props = (MaxComputeDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(ENDPOINT.getName(), props.getEndpoint()); + conf.putPOJO(ACCESS_ID.getName(), props.getAccessId()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccesskey()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/sink/MongoDBSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/sink/MongoDBSinkPlugin.java index 004e22adb..02387b12c 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/sink/MongoDBSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/sink/MongoDBSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.mongodb.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.MongoDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.MongoDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -71,8 +72,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - MongoDBDataSource dataSource = (MongoDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URI.getName(), dataSource.getUri()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + MongoDBDataSourceProperties props = (MongoDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URI.getName(), props.getUri()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/source/MongoDBSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/source/MongoDBSourcePlugin.java index ee64897e8..9ef337894 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/source/MongoDBSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/mongodb/source/MongoDBSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.mongodb.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.MongoDBDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.MongoDBDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -72,8 +73,10 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - MongoDBDataSource dataSource = (MongoDBDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URI.getName(), dataSource.getUri()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + MongoDBDataSourceProperties props = (MongoDBDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URI.getName(), props.getUri()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/sink/Neo4jSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/sink/Neo4jSinkPlugin.java index 1086b298a..9a7153950 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/sink/Neo4jSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/sink/Neo4jSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.neo4j.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.Neo4jDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.Neo4jDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -68,17 +69,19 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - Neo4jDataSource dataSource = (Neo4jDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URI.getName(), dataSource.getUri()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + Neo4jDataSourceProperties props = (Neo4jDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URI.getName(), props.getUri()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } - if (StringUtils.hasText(dataSource.getBearerToken())) { - conf.putPOJO(BEARER_TOKEN.getName(), dataSource.getBearerToken()); + if (StringUtils.hasText(props.getBearerToken())) { + conf.putPOJO(BEARER_TOKEN.getName(), props.getBearerToken()); } - if (StringUtils.hasText(dataSource.getKerberosTicket())) { - conf.putPOJO(KERBEROS_TICKET.getName(), dataSource.getKerberosTicket()); + if (StringUtils.hasText(props.getKerberosTicket())) { + conf.putPOJO(KERBEROS_TICKET.getName(), props.getKerberosTicket()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/source/Neo4jSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/source/Neo4jSourcePlugin.java index 96b550da3..7bca8b014 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/source/Neo4jSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/neo4j/source/Neo4jSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.neo4j.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.Neo4jDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.Neo4jDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -66,17 +67,19 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - Neo4jDataSource dataSource = (Neo4jDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(URI.getName(), dataSource.getUri()); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + Neo4jDataSourceProperties props = (Neo4jDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(URI.getName(), props.getUri()); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } - if (StringUtils.hasText(dataSource.getBearerToken())) { - conf.putPOJO(BEARER_TOKEN.getName(), dataSource.getBearerToken()); + if (StringUtils.hasText(props.getBearerToken())) { + conf.putPOJO(BEARER_TOKEN.getName(), props.getBearerToken()); } - if (StringUtils.hasText(dataSource.getKerberosTicket())) { - conf.putPOJO(KERBEROS_TICKET.getName(), dataSource.getKerberosTicket()); + if (StringUtils.hasText(props.getKerberosTicket())) { + conf.putPOJO(KERBEROS_TICKET.getName(), props.getKerberosTicket()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/sink/PulsarSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/sink/PulsarSinkPlugin.java index ac4e1f90c..5a4e185d9 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/sink/PulsarSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/sink/PulsarSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.pulsar.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.mq.PulsarDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.mq.PulsarDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -70,14 +71,16 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - PulsarDataSource dataSource = (PulsarDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(ADMIN_SERVICE_URL.getName(), dataSource.getWebServiceUrl()); - conf.putPOJO(CLIENT_SERVICE_URL.getName(), dataSource.getClientServiceUrl()); - if (StringUtils.hasText(dataSource.getAuthPlugin())) { - conf.putPOJO(AUTH_PLUGIN_CLASS.getName(), dataSource.getAuthPlugin()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + PulsarDataSourceProperties props = (PulsarDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(ADMIN_SERVICE_URL.getName(), props.getWebServiceUrl()); + conf.putPOJO(CLIENT_SERVICE_URL.getName(), props.getClientServiceUrl()); + if (StringUtils.hasText(props.getAuthPlugin())) { + conf.putPOJO(AUTH_PLUGIN_CLASS.getName(), props.getAuthPlugin()); } - if (StringUtils.hasText(dataSource.getAuthParams())) { - conf.putPOJO(AUTH_PARAMS.getName(), dataSource.getAuthParams()); + if (StringUtils.hasText(props.getAuthParams())) { + conf.putPOJO(AUTH_PARAMS.getName(), props.getAuthParams()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/source/PulsarSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/source/PulsarSourcePlugin.java index 497bfdc10..93f7668e8 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/source/PulsarSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/source/PulsarSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.pulsar.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.mq.PulsarDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.mq.PulsarDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -77,14 +78,16 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - PulsarDataSource dataSource = (PulsarDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(ADMIN_SERVICE_URL.getName(), dataSource.getWebServiceUrl()); - conf.putPOJO(CLIENT_SERVICE_URL.getName(), dataSource.getClientServiceUrl()); - if (StringUtils.hasText(dataSource.getAuthPlugin())) { - conf.putPOJO(AUTH_PLUGIN_CLASS.getName(), dataSource.getAuthPlugin()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + PulsarDataSourceProperties props = (PulsarDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(ADMIN_SERVICE_URL.getName(), props.getWebServiceUrl()); + conf.putPOJO(CLIENT_SERVICE_URL.getName(), props.getClientServiceUrl()); + if (StringUtils.hasText(props.getAuthPlugin())) { + conf.putPOJO(AUTH_PLUGIN_CLASS.getName(), props.getAuthPlugin()); } - if (StringUtils.hasText(dataSource.getAuthParams())) { - conf.putPOJO(AUTH_PARAMS.getName(), dataSource.getAuthParams()); + if (StringUtils.hasText(props.getAuthParams())) { + conf.putPOJO(AUTH_PARAMS.getName(), props.getAuthParams()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/sink/RedisSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/sink/RedisSinkPlugin.java index 434f77a78..1c927c15d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/sink/RedisSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/sink/RedisSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.redis.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.RedisDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.RedisDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -69,20 +70,22 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - RedisDataSource dataSource = (RedisDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - if (StringUtils.hasText(dataSource.getUser())) { - conf.putPOJO(USER.getName(), dataSource.getUser()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + RedisDataSourceProperties props = (RedisDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + if (StringUtils.hasText(props.getUser())) { + conf.putPOJO(USER.getName(), props.getUser()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(AUTH.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(AUTH.getName(), props.getPassword()); } - if (dataSource.getMode() != null) { - conf.putPOJO(MODE.getName(), dataSource.getMode().getValue()); + if (props.getMode() != null) { + conf.putPOJO(MODE.getName(), props.getMode().getValue()); } - if (CollectionUtils.isEmpty(dataSource.getNodes()) == false) { - conf.putPOJO(NODES.getName(), dataSource.getNodes().stream().map(RedisDataSource.Node::getNode).collect(Collectors.toList())); + if (CollectionUtils.isEmpty(props.getNodes()) == false) { + conf.putPOJO(NODES.getName(), props.getNodes().stream().map(RedisDataSourceProperties.Node::getNode).collect(Collectors.toList())); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/source/RedisSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/source/RedisSourcePlugin.java index a1563336c..1f21b80ef 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/source/RedisSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/redis/source/RedisSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.redis.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.nosql.RedisDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.nosql.RedisDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -70,20 +71,22 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - RedisDataSource dataSource = (RedisDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(HOST.getName(), dataSource.getHost()); - conf.putPOJO(PORT.getName(), dataSource.getPort()); - if (StringUtils.hasText(dataSource.getUser())) { - conf.putPOJO(USER.getName(), dataSource.getUser()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + RedisDataSourceProperties props = (RedisDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(HOST.getName(), props.getHost()); + conf.putPOJO(PORT.getName(), props.getPort()); + if (StringUtils.hasText(props.getUser())) { + conf.putPOJO(USER.getName(), props.getUser()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(AUTH.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(AUTH.getName(), props.getPassword()); } - if (dataSource.getMode() != null) { - conf.putPOJO(MODE.getName(), dataSource.getMode().getValue()); + if (props.getMode() != null) { + conf.putPOJO(MODE.getName(), props.getMode().getValue()); } - if (CollectionUtils.isEmpty(dataSource.getNodes()) == false) { - conf.putPOJO(NODES.getName(), dataSource.getNodes().stream().map(RedisDataSource.Node::getNode).collect(Collectors.toList())); + if (CollectionUtils.isEmpty(props.getNodes()) == false) { + conf.putPOJO(NODES.getName(), props.getNodes().stream().map(RedisDataSourceProperties.Node::getNode).collect(Collectors.toList())); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/s3redshift/sink/S3RedshiftSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/s3redshift/sink/S3RedshiftSinkPlugin.java index 3fcde1a9d..358314d6d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/s3redshift/sink/S3RedshiftSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/s3redshift/sink/S3RedshiftSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.s3redshift.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.file.S3DataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.file.S3DataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -35,8 +36,8 @@ import java.util.Collections; import java.util.List; -import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileProperties.PATH; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileProperties.FIELD_DELIMITER; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileProperties.PATH; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileSinkProperties.*; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.s3.S3Properties.*; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.s3redshift.S3RedshiftProperties.BUCKET; @@ -84,10 +85,12 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - S3DataSource dataSource = (S3DataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(BUCKET.getName(), dataSource.getBucket()); - conf.putPOJO(ACCESS_KEY.getName(), dataSource.getAccessKey()); - conf.putPOJO(ACCESS_SECRET.getName(), dataSource.getAccessSecret()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + S3DataSourceProperties props = (S3DataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(BUCKET.getName(), props.getBucket()); + conf.putPOJO(ACCESS_KEY.getName(), props.getAccessKey()); + conf.putPOJO(ACCESS_SECRET.getName(), props.getAccessSecret()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/sink/SocketSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/sink/SocketSinkPlugin.java index c2f046dfc..c260b4959 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/sink/SocketSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/sink/SocketSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.socket.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.SocketDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.SocketDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -59,9 +60,11 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - SocketDataSource dataSource = (SocketDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(SocketProperties.HOST.getName(), dataSource.getHost()); - conf.putPOJO(SocketProperties.PORT.getName(), dataSource.getPort()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + SocketDataSourceProperties props = (SocketDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(SocketProperties.HOST.getName(), props.getHost()); + conf.putPOJO(SocketProperties.PORT.getName(), props.getPort()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/source/SocketSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/source/SocketSourcePlugin.java index b9e3333d2..432135c15 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/source/SocketSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/socket/source/SocketSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.socket.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.SocketDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.SocketDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -58,9 +59,11 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - SocketDataSource dataSource = (SocketDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(SocketProperties.HOST.getName(), dataSource.getHost()); - conf.putPOJO(SocketProperties.PORT.getName(), dataSource.getPort()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + SocketDataSourceProperties props = (SocketDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(SocketProperties.HOST.getName(), props.getHost()); + conf.putPOJO(SocketProperties.PORT.getName(), props.getPort()); return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/sink/StarRocksSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/sink/StarRocksSinkPlugin.java index 6d6c4d820..b05d15b2a 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/sink/StarRocksSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/sink/StarRocksSinkPlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.starrocks.sink; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.StarRocksDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.StarRocksDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -78,16 +79,18 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - StarRocksDataSource dataSource = (StarRocksDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(NODE_URLS.getName(), StringUtils.commaDelimitedListToStringArray(dataSource.getNodeUrls())); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + StarRocksDataSourceProperties props = (StarRocksDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(NODE_URLS.getName(), StringUtils.commaDelimitedListToStringArray(props.getNodeUrls())); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } - if (StringUtils.hasText(dataSource.getBaseUrl())) { - conf.putPOJO(BASE_URL.getName(), dataSource.getBaseUrl()); + if (StringUtils.hasText(props.getBaseUrl())) { + conf.putPOJO(BASE_URL.getName(), props.getBaseUrl()); } return conf; } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/source/StarRocksSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/source/StarRocksSourcePlugin.java index 95373cf8b..b6fc86730 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/source/StarRocksSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/starrocks/source/StarRocksSourcePlugin.java @@ -18,9 +18,10 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.starrocks.source; +import cn.sliew.carp.module.datasource.modal.DataSourceInfo; +import cn.sliew.carp.module.datasource.modal.olap.StarRocksDataSourceProperties; +import cn.sliew.milky.common.util.JacksonUtil; import cn.sliew.scaleph.common.dict.seatunnel.SeaTunnelPluginMapping; -import cn.sliew.scaleph.ds.modal.AbstractDataSource; -import cn.sliew.scaleph.ds.modal.olap.StarRocksDataSource; import cn.sliew.scaleph.plugin.framework.core.PluginInfo; import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor; import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin; @@ -72,13 +73,15 @@ public List getRequiredResources() { public ObjectNode createConf() { ObjectNode conf = super.createConf(); JsonNode jsonNode = properties.get(ResourceProperties.DATASOURCE); - StarRocksDataSource dataSource = (StarRocksDataSource) AbstractDataSource.fromDsInfo((ObjectNode) jsonNode); - conf.putPOJO(NODE_URLS.getName(), StringUtils.commaDelimitedListToStringArray(dataSource.getNodeUrls())); - if (StringUtils.hasText(dataSource.getUsername())) { - conf.putPOJO(USERNAME.getName(), dataSource.getUsername()); + DataSourceInfo dataSourceInfo = JacksonUtil.toObject(jsonNode, DataSourceInfo.class); + StarRocksDataSourceProperties props = (StarRocksDataSourceProperties) dataSourceInfo.getProps(); + + conf.putPOJO(NODE_URLS.getName(), StringUtils.commaDelimitedListToStringArray(props.getNodeUrls())); + if (StringUtils.hasText(props.getUsername())) { + conf.putPOJO(USERNAME.getName(), props.getUsername()); } - if (StringUtils.hasText(dataSource.getPassword())) { - conf.putPOJO(PASSWORD.getName(), dataSource.getPassword()); + if (StringUtils.hasText(props.getPassword())) { + conf.putPOJO(PASSWORD.getName(), props.getPassword()); } return conf; } diff --git a/scaleph-security/src/main/java/cn/sliew/scaleph/security/config/AuthenticateConfigurer.java b/scaleph-security/src/main/java/cn/sliew/scaleph/security/config/AuthenticateConfigurer.java deleted file mode 100644 index 8737442bf..000000000 --- a/scaleph-security/src/main/java/cn/sliew/scaleph/security/config/AuthenticateConfigurer.java +++ /dev/null @@ -1,30 +0,0 @@ -//package cn.sliew.scaleph.security.config; -// -//import org.springframework.http.HttpMethod; -//import org.springframework.security.config.annotation.web.HttpSecurityBuilder; -//import org.springframework.security.config.annotation.web.configurers.AbstractAuthenticationFilterConfigurer; -//import org.springframework.security.web.util.matcher.AntPathRequestMatcher; -//import org.springframework.security.web.util.matcher.RequestMatcher; -// -//import javax.servlet.Filter; -// -///** -// * 注册Filter： -// * 通过 {@link org.springframework.security.config.annotation.web.builders.HttpSecurity#addFilterAt(Filter, Class)} 注册 -// * 通过 继承 {@link org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer} -// *

-// * 这里通过继承 {@link org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter} 的方式实现登陆逻辑， -// * 因此通过 AbstractHttpConfigurer 注册。 -// */ -//public class AuthenticateConfigurer> extends -// AbstractAuthenticationFilterConfigurer, CustomUsernamePasswordAuthenticateFilter> { -// -// public AuthenticateConfigurer() { -// super(new CustomUsernamePasswordAuthenticateFilter(), null); -// } -// -// @Override -// protected RequestMatcher createLoginProcessingUrlMatcher(String loginProcessingUrl) { -// return new AntPathRequestMatcher(loginProcessingUrl, HttpMethod.POST.name()); -// } -//} diff --git a/scaleph-ui-react/public/images/bg.png b/scaleph-ui-react/public/images/bg.png new file mode 100644 index 000000000..edc40be8d Binary files /dev/null and b/scaleph-ui-react/public/images/bg.png differ diff --git a/scaleph-ui-react/src/locales/zh-CN/pages/project.ts b/scaleph-ui-react/src/locales/zh-CN/pages/project.ts index 9477c72e5..8ed365b36 100644 --- a/scaleph-ui-react/src/locales/zh-CN/pages/project.ts +++ b/scaleph-ui-react/src/locales/zh-CN/pages/project.ts @@ -348,6 +348,31 @@ export default { 'pages.project.di.step.hudi.kerberosPrincipal': 'Kerberos Principal', 'pages.project.di.step.hudi.kerberosPrincipalFile': 'Kerberos Principal File', + 'pages.project.di.step.hudi.tableDfsPath': 'Hudi Table Dfs Root Path', + 'pages.project.di.step.hudi.tableDfsPath.placeholder': 'hdfs://nameserivce/data/hudi/', + 'pages.project.di.step.hudi.confFilesPath': 'Conf Files Path', + 'pages.project.di.step.hudi.confFilesPath.placeholder': + '/home/test/hdfs-site.xml;/home/test/core-site.xml;/home/test/yarn-site.xml', + 'pages.project.di.step.hudi.autoCommit': 'Auto Commit', + 'pages.project.di.step.hudi.tableList': 'Tables', + 'pages.project.di.step.hudi.tableList.tooltip': '表同步配置，支持多表', + 'pages.project.di.step.hudi.tableList.table': '配置', + 'pages.project.di.step.hudi.tableList.database': '数据库', + 'pages.project.di.step.hudi.tableList.tableName': '表名', + 'pages.project.di.step.hudi.tableList.tableType': '表类型', + 'pages.project.di.step.hudi.tableList.opType': '操作类型', + 'pages.project.di.step.hudi.tableList.indexType': '索引', + 'pages.project.di.step.hudi.tableList.indexClassName': '索引类', + 'pages.project.di.step.hudi.tableList.batchSize': '批次大小', + 'pages.project.di.step.hudi.tableList.batchIntervalMs': '批次间隔（毫秒）', + 'pages.project.di.step.hudi.tableList.minCommitsToKeep': '最小 Commit', + 'pages.project.di.step.hudi.tableList.maxCommitsToKeep': '最大 Commit', + 'pages.project.di.step.hudi.tableList.insertShuffleParallelism': 'Insert 并行度', + 'pages.project.di.step.hudi.tableList.upsertShuffleParallelism': 'Upsert 并行度', + 'pages.project.di.step.hudi.tableList.recordKeyFields': 'Record Key 字段', + 'pages.project.di.step.hudi.tableList.partitionFields': 'Partition 字段', + 'pages.project.di.step.hudi.tableList.recordByteSize': 'Record 大小（Byte）', + // iceberg 'pages.project.di.step.iceberg.catalogName': 'Catalog名称', 'pages.project.di.step.iceberg.namespace': '命名空间', @@ -868,6 +893,7 @@ export default { 'pages.project.di.step.hbase.walWrite': '是否启用WAL', 'pages.project.di.step.hbase.writeBufferSize': '写缓存大小', 'pages.project.di.step.hbase.encoding': '编码方式', + 'pages.project.di.step.hbase.ttl': 'TTL', 'pages.project.di.step.hbase.hbaseExtraConfig': 'HBase额外配置', 'pages.project.di.step.hbase.hbaseExtraConfig.tooltip': 'HBase的额外配置信息', @@ -1209,6 +1235,7 @@ export default { 'pages.project.schedule.group.name': '名称', 'pages.project.schedule.job': '任务', 'pages.project.schedule.job.name': '名称', + 'pages.project.schedule.job.jobGroupId': '分组', 'Run': '运行', 'Save': '保存', diff --git a/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/FlinkCDC/index.tsx b/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/FlinkCDC/index.tsx index aab632fdf..2d7cb424d 100644 --- a/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/FlinkCDC/index.tsx +++ b/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/FlinkCDC/index.tsx @@ -108,6 +108,7 @@ const DataIntegrationFlinkCDCWeb: React.FC = () => {