[Feature][scaleph-workspace-seatunnel] upgrade seatunnel connectors to 2.3.5 version

pom.xml

@@ -791,6 +791,10 @@
                 <artifactId>client-java</artifactId>
                 <version>${gravitino.version}</version>
                 <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-api</artifactId>
+                    </exclusion>
                     <exclusion>
                         <groupId>org.apache.logging.log4j</groupId>
                         <artifactId>log4j-slf4j2-impl</artifactId>

scaleph-datasource/pom.xml

@@ -37,14 +37,16 @@
             <artifactId>scaleph-resource</artifactId>
         </dependency>
 
-        <!--<dependency>
+        <dependency>
             <groupId>com.datastrato.gravitino</groupId>
             <artifactId>client-java-runtime</artifactId>
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>com.datastrato.gravitino</groupId>
             <artifactId>client-java</artifactId>
-        </dependency>-->
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
 </project>

scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java

@@ -6,7 +6,7 @@
 // * to you under the Apache License, Version 2.0 (the
 // * "License"); you may not use this file except in compliance
 // * with the License.  You may obtain a copy of the License at
-// *
+// *nch
 // *     http://www.apache.org/licenses/LICENSE-2.0
 // *
 // * Unless required by applicable law or agreed to in writing, software

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/KuduProperties.java

@@ -23,7 +23,7 @@
 public enum KuduProperties {
     ;
     public static final PropertyDescriptor<String> KUDU_MASTER = new PropertyDescriptor.Builder()
-            .name("kudu_master")
+            .name("kudu_masters")
             .description("The address of kudu master")
             .type(PropertyType.STRING)
             .parser(Parsers.STRING_PARSER)
@@ -63,6 +63,15 @@ public enum KuduProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
+    public static final PropertyDescriptor<String> TABLE_NAME = new PropertyDescriptor.Builder()
+            .name("table_name")
+            .description("The name of kudu table")
+            .type(PropertyType.STRING)
+            .parser(Parsers.STRING_PARSER)
+            .properties(Property.Required)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
     public static final PropertyDescriptor<Integer> CLIENT_WORKER_COUNT = new PropertyDescriptor.Builder()
             .name("client_worker_count")
             .description("Kudu worker count. Default value is twice the current number of cpu cores.")

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkPlugin.java

@@ -46,6 +46,13 @@ public KuduSinkPlugin() {
                 "Write data to Kudu.",
                 KuduSinkPlugin.class.getName());
         final List<PropertyDescriptor> props = new ArrayList<>();
+        props.add(ENABLE_KERBEROS);
+        props.add(KERBEROS_PRINCIPAL);
+        props.add(KERBEROS_KEYTAB);
+        props.add(KERBEROS_KRB5CONF);
+        props.add(CLIENT_WORKER_COUNT);
+        props.add(CLIENT_DEFAULT_OPERATION_TIMEOUT_MS);
+        props.add(CLIENT_DEFAULT_ADMIN_OPERATION_TIMEOUT_MS);
         props.add(TABLE_NAME);
         props.add(SAVE_MODE);
         props.add(SESSION_FLUSH_MODE);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/sink/KuduSinkProperties.java

@@ -23,15 +23,6 @@
 public enum KuduSinkProperties {
     ;
 
-    public static final PropertyDescriptor<String> TABLE_NAME = new PropertyDescriptor.Builder()
-            .name("table_name")
-            .description("The name of kudu table.")
-            .type(PropertyType.STRING)
-            .parser(Parsers.STRING_PARSER)
-            .properties(Property.Required)
-            .addValidator(Validators.NON_BLANK_VALIDATOR)
-            .validateAndBuild();
-
     public static final PropertyDescriptor<String> SAVE_MODE = new PropertyDescriptor.Builder()
             .name("save_mode")
             .description("Storage mode, we need support overwrite and append")

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourcePlugin.java

@@ -53,8 +53,7 @@ public KuduSourcePlugin() {
         props.add(CLIENT_WORKER_COUNT);
         props.add(CLIENT_DEFAULT_OPERATION_TIMEOUT_MS);
         props.add(CLIENT_DEFAULT_ADMIN_OPERATION_TIMEOUT_MS);
-        props.add(KUDU_TABLE);
-        props.add(COLUMNS_LIST);
+        props.add(TABLE_NAME);
         props.add(SCAN_TOKEN_QUERY_TIMEOUT);
         props.add(SCAN_TOKEN_BATCH_SIZE_BYTES);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/kudu/source/KuduSourceProperties.java

@@ -24,24 +24,6 @@
 public enum KuduSourceProperties {
     ;
 
-    public static final PropertyDescriptor<String> KUDU_TABLE = new PropertyDescriptor.Builder()
-            .name("kudu_table")
-            .description("The name of kudu table")
-            .type(PropertyType.STRING)
-            .parser(Parsers.STRING_PARSER)
-            .properties(Property.Required)
-            .addValidator(Validators.NON_BLANK_VALIDATOR)
-            .validateAndBuild();
-
-    public static final PropertyDescriptor<String> COLUMNS_LIST = new PropertyDescriptor.Builder()
-            .name("columnsList")
-            .description("Specifies the column names of the table")
-            .type(PropertyType.STRING)
-            .parser(Parsers.STRING_PARSER)
-            .properties(Property.Required)
-            .addValidator(Validators.NON_BLANK_VALIDATOR)
-            .validateAndBuild();
-
     public static final PropertyDescriptor<Long> SCAN_TOKEN_QUERY_TIMEOUT = new PropertyDescriptor.Builder()
             .name("scan_token_query_timeout")
             .description("The timeout for connecting scan token.")

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/pulsar/PulsarProperties.java

@@ -84,6 +84,9 @@ public enum PulsarProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
+    /**
+     * todo does this in use ?
+     */
     public static final PropertyDescriptor<String> FIELD_DELIMITER = new PropertyDescriptor.Builder<String>()
             .name("field_delimiter")
             .description("The separator between columns in a row of data. Only needed by text and csv file format")

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/s3redshift/sink/S3RedshiftSinkPlugin.java

@@ -55,7 +55,6 @@ public S3RedshiftSinkPlugin() {
         props.add(JDBC_USER);
         props.add(JDBC_PASSWORD);
         props.add(EXECUTE_SQL);
-        props.add(BUCKET);
 
         props.add(HADOOP_S3_PROPERTIES);
         props.add(PATH);

scaleph-ui-react/config/routes.ts

@@ -217,7 +217,7 @@ export default [
           {
             name: 'flink-cdc',
             path: '/workspace/data-integration/flink-cdc',
-            icon: 'sun',
+            icon: 'fieldNumber',
             component: './Project/Workspace/DataIntegration/FlinkCDC',
           },
           {

scaleph-ui-react/src/locales/zh-CN/pages/project.ts

@@ -317,10 +317,8 @@ export default {
   'pages.project.di.step.wechat.url': '微信机器人URL',
   'pages.project.di.step.wechat.mentionedList': '提及的(@)用户ID列表',
   'pages.project.di.step.wechat.mentionedList.tooltip': '使用"@all"提及所有人',
-  'pages.project.di.step.wechat.userId': '用户ID',
   'pages.project.di.step.wechat.mentionedMobileList': '提及的(@)手机号',
   'pages.project.di.step.wechat.mentionedMobileList.tooltip': '使用"@all"提及所有人',
-  'pages.project.di.step.wechat.mobile': '手机号',
 
   // feishu
   'pages.project.di.step.feishu.url': '飞书机器人URL',
@@ -431,6 +429,8 @@ export default {
   'pages.project.di.step.kudu.scanTokenQueryTimeout': 'Scan查询超时时间(毫秒)',
   'pages.project.di.step.kudu.scanTokenBatchSizeBytes': 'Scan批次数据量(byte)',
   'pages.project.di.step.kudu.filter': 'Scan过滤表达式',
+  'pages.project.di.step.kudu.tableList': '多表配置',
+  'pages.project.di.step.kudu.tableList.tooltip': '如果要一次读取多表数据，可以通过此选项配置',
 
   // kafka
   'pages.project.di.step.kafka.topic': 'Topic',
@@ -483,9 +483,6 @@ export default {
   'pages.project.di.step.rocketmq.startModeTimestamp': 'Consumer 时间戳位点',
   'pages.project.di.step.rocketmq.startModeOffsets': 'Consumer 分区位点',
   'pages.project.di.step.rocketmq.startModeOffsets.tooltip': '指定 Topic 每个分区位点',
-  'pages.project.di.step.rocketmq.startModeOffsetsList': '分区',
-  'pages.project.di.step.rocketmq.specificPartition': '分区',
-  'pages.project.di.step.rocketmq.specificPartitionOffset': '位点',
   'pages.project.di.step.rocketmq.topic': 'Topic',
   'pages.project.di.step.rocketmq.producerGroup': 'Producer Group',
   'pages.project.di.step.rocketmq.exactlyOnce': '启用 Exactly Once 语义（事务保证）',
@@ -587,12 +584,8 @@ export default {
   'pages.project.di.step.pulsar.messageRoutingMode': '消息路由模式',
   'pages.project.di.step.pulsar.partitionKeyFields': '分区字段',
   'pages.project.di.step.pulsar.partitionKeyFields.tooltip': '配置 Pulsar 消息分区字段',
-  'pages.project.di.step.pulsar.partitionKeyField': '字段',
   'pages.project.di.step.pulsar.pulsarConfig': 'Pulsar 配置',
   'pages.project.di.step.pulsar.pulsarConfig.tooltip': '添加额外 Pulsar Producer 配置',
-  'pages.project.di.step.pulsar.pulsarConfig.item': '配置项',
-  'pages.project.di.step.pulsar.pulsarConfigKey': '配置',
-  'pages.project.di.step.pulsar.pulsarConfigValue': '值',
 
   // datahub
   'pages.project.di.step.datahub.endpoint': '终端节点',
@@ -742,10 +735,6 @@ export default {
   'pages.project.di.step.starrocks.starrocksConfig': 'StarRocks 配置',
   'pages.project.di.step.starrocks.starrocksConfig.tooltip': '流式加载参数',
   'pages.project.di.step.starrocks.starrocksConfig.list': '配置项',
-  'pages.project.di.step.starrocks.starrocksConfigKey': '键',
-  'pages.project.di.step.starrocks.starrocksConfigKey.placeholder': 'format',
-  'pages.project.di.step.starrocks.starrocksConfigValue': '值',
-  'pages.project.di.step.starrocks.starrocksConfigValue.placeholder': 'JSON',
   'pages.project.di.step.starrocks.scanFilter': 'StarRocks 扫描过滤器',
   'pages.project.di.step.starrocks.scanConnectTimeoutMs': '扫描连接超时时间(毫秒)',
   'pages.project.di.step.starrocks.scanQueryTimeoutSec': '扫描查询超时时间(秒)',

scaleph-ui-react/src/pages/Metadata/DataSource/Info/index.tsx

@@ -2,7 +2,7 @@ import {history, useAccess, useIntl} from "@umijs/max";
 import {useRef, useState} from "react";
 import {Button, Image, message, Modal, Space, Tooltip} from "antd";
 import {DeleteOutlined} from "@ant-design/icons";
-import {ActionType, ProColumns, ProFormInstance, ProTable} from "@ant-design/pro-components";
+import {ActionType, PageContainer, ProColumns, ProFormInstance, ProTable} from "@ant-design/pro-components";
 import {PRIVILEGE_CODE} from "@/constants/privilegeCode";
 import {DICT_TYPE} from "@/constants/dictType";
 import {DsInfo} from "@/services/datasource/typings";
@@ -114,68 +114,70 @@ const DataSourceListWeb: React.FC = () => {
   ]
 
   return (
-    <ProTable<DsInfo>
-      search={{
-        labelWidth: 'auto',
-        span: {xs: 24, sm: 12, md: 8, lg: 6, xl: 6, xxl: 4},
-      }}
-      rowKey="id"
-      actionRef={actionRef}
-      formRef={formRef}
-      options={false}
-      columns={tableColumns}
-      request={(params, sorter, filter) => {
-        return DsInfoService.list(params);
-      }}
-      toolbar={{
-        actions: [
-          access.canAccess(PRIVILEGE_CODE.datadevClusterAdd) && (
-            <Button
-              key="new"
-              type="primary"
-              onClick={() => history.push("/metadata/data-source/info/stepForms")}
-            >
-              {intl.formatMessage({id: 'app.common.operate.new.label'})}
-            </Button>
-          ),
-          access.canAccess(PRIVILEGE_CODE.datadevClusterDelete) && (
-            <Button
-              key="del"
-              type="default"
-              disabled={selectedRows.length < 1}
-              onClick={() => {
-                Modal.confirm({
-                  title: intl.formatMessage({id: 'app.common.operate.delete.confirm.title'}),
-                  content: intl.formatMessage({id: 'app.common.operate.delete.confirm.content'}),
-                  okText: intl.formatMessage({id: 'app.common.operate.confirm.label'}),
-                  okButtonProps: {danger: true},
-                  cancelText: intl.formatMessage({id: 'app.common.operate.cancel.label'}),
-                  onOk() {
-                    DsInfoService.deleteBatch(selectedRows).then((d) => {
-                      if (d.success) {
-                        message.success(intl.formatMessage({id: 'app.common.operate.delete.success'}));
-                        actionRef.current?.reload();
-                      }
-                    });
-                  },
-                });
-              }}
-            >
-              {intl.formatMessage({id: 'app.common.operate.delete.label'})}
-            </Button>
-          ),
-        ],
-      }}
-      pagination={{showQuickJumper: true, showSizeChanger: true, defaultPageSize: 10}}
-      rowSelection={{
-        fixed: true,
-        onChange(selectedRowKeys, selectedRows, info) {
-          setSelectedRows(selectedRows);
-        },
-      }}
-      tableAlertRender={false}
-      tableAlertOptionRender={false}
-    />
+    <PageContainer title={false}>
+      <ProTable<DsInfo>
+        search={{
+          labelWidth: 'auto',
+          span: {xs: 24, sm: 12, md: 8, lg: 6, xl: 6, xxl: 4},
+        }}
+        rowKey="id"
+        actionRef={actionRef}
+        formRef={formRef}
+        options={false}
+        columns={tableColumns}
+        request={(params, sorter, filter) => {
+          return DsInfoService.list(params);
+        }}
+        toolbar={{
+          actions: [
+            access.canAccess(PRIVILEGE_CODE.datadevClusterAdd) && (
+              <Button
+                key="new"
+                type="primary"
+                onClick={() => history.push("/metadata/data-source/info/stepForms")}
+              >
+                {intl.formatMessage({id: 'app.common.operate.new.label'})}
+              </Button>
+            ),
+            access.canAccess(PRIVILEGE_CODE.datadevClusterDelete) && (
+              <Button
+                key="del"
+                type="default"
+                disabled={selectedRows.length < 1}
+                onClick={() => {
+                  Modal.confirm({
+                    title: intl.formatMessage({id: 'app.common.operate.delete.confirm.title'}),
+                    content: intl.formatMessage({id: 'app.common.operate.delete.confirm.content'}),
+                    okText: intl.formatMessage({id: 'app.common.operate.confirm.label'}),
+                    okButtonProps: {danger: true},
+                    cancelText: intl.formatMessage({id: 'app.common.operate.cancel.label'}),
+                    onOk() {
+                      DsInfoService.deleteBatch(selectedRows).then((d) => {
+                        if (d.success) {
+                          message.success(intl.formatMessage({id: 'app.common.operate.delete.success'}));
+                          actionRef.current?.reload();
+                        }
+                      });
+                    },
+                  });
+                }}
+              >
+                {intl.formatMessage({id: 'app.common.operate.delete.label'})}
+              </Button>
+            ),
+          ],
+        }}
+        pagination={{showQuickJumper: true, showSizeChanger: true, defaultPageSize: 10}}
+        rowSelection={{
+          fixed: true,
+          onChange(selectedRowKeys, selectedRows, info) {
+            setSelectedRows(selectedRows);
+          },
+        }}
+        tableAlertRender={false}
+        tableAlertOptionRender={false}
+      />
+    </PageContainer>
   );
 }
 

scaleph-ui-react/src/pages/Metadata/Gravitino/Catalog/index.tsx

@@ -1,11 +1,12 @@
 import {useAccess, useIntl} from "@umijs/max";
+import {PageContainer} from "@ant-design/pro-components";
 
 const MetadataGravitinoCatalogWeb: React.FC = () => {
   const intl = useIntl();
   const access = useAccess();
 
   return (
-    <div>待开发</div>
+    <PageContainer title={false}>待开发</PageContainer>
   );
 }