Skip to content

Commit

Permalink
[Feature][scaleph-ui-react] upgrade seatunnel hudi connectors to 2.3.8 (
Browse files Browse the repository at this point in the history
#756)

* feature: update schedule group web

* feature: upgrade carp version

* fix: dependency update

* fix: dependency update

* feature: upgrade carp to 0.0.20

* feature: upgrade seatunnel plugin to 2.3.8

* fix: dependency update

* feature: replace datasource class

* feature: replace datasource class

* feature: upgrade seatunnel hudi connectors to 2.3.8

* fix: license-header

* fix: license-header

---------

Co-authored-by: wangqi <[email protected]>
  • Loading branch information
kalencaya and wangqi authored Dec 9, 2024
1 parent a93627f commit 862a392
Show file tree
Hide file tree
Showing 121 changed files with 1,339 additions and 3,147 deletions.
286 changes: 7 additions & 279 deletions pom.xml

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package cn.sliew.scaleph.api.config;

import cn.sliew.carp.framework.mybatis.DataSourceConstants;
import cn.sliew.carp.framework.mybatis.config.CarpMybatisConfig;
import com.baomidou.mybatisplus.autoconfigure.MybatisPlusProperties;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.config.GlobalConfig;
import com.baomidou.mybatisplus.core.handlers.MybatisEnumTypeHandler;
import com.baomidou.mybatisplus.core.toolkit.GlobalConfigUtils;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.ibatis.logging.slf4j.Slf4jImpl;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;

import javax.sql.DataSource;

@Configuration
public class CarpDataSourceConfig {

@Autowired
private MybatisPlusInterceptor mybatisPlusInterceptor;

@Primary
@Bean(DataSourceConstants.DATA_SOURCE_FACTORY)
@ConfigurationProperties(prefix = "spring.datasource.carp")
public DataSource carpDataSource() {
return DataSourceBuilder.create().type(HikariDataSource.class)
.build();
}

@Primary
@Bean(DataSourceConstants.TRANSACTION_MANAGER_FACTORY)
public DataSourceTransactionManager carpTransactionManager() {
return new DataSourceTransactionManager(carpDataSource());
}

@Primary
@Bean(DataSourceConstants.SQL_SESSION_FACTORY)
public SqlSessionFactory carpSqlSessionFactory() throws Exception {
MybatisSqlSessionFactoryBean factoryBean = new MybatisSqlSessionFactoryBean();
GlobalConfig globalConfig = GlobalConfigUtils.defaults();
globalConfig.setMetaObjectHandler(new CarpMybatisConfig.CarpMetaHandler());

MybatisPlusProperties props = new MybatisPlusProperties();
props.setMapperLocations(new String[]{DataSourceConstants.MAPPER_XML_PATH});
factoryBean.setMapperLocations(props.resolveMapperLocations());

MybatisConfiguration configuration = new MybatisConfiguration();
configuration.setDefaultEnumTypeHandler(MybatisEnumTypeHandler.class);
configuration.setMapUnderscoreToCamelCase(true);
configuration.setLogImpl(Slf4jImpl.class);
factoryBean.setConfiguration(configuration);
factoryBean.setGlobalConfig(globalConfig);
factoryBean.setDataSource(carpDataSource());
factoryBean.setPlugins(mybatisPlusInterceptor);
return factoryBean.getObject();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ public enum SeaTunnelPluginMapping {
SOURCE_STARROCKS(SEATUNNEL, SOURCE, STARROCKS, "connector-starrocks", UNKNOWN, BATCH, SCHEMA_PROJECTION, PARALLELISM, SUPPORT_USER_DEFINED_SPLIT),
SINK_STARROCKS(SEATUNNEL, SINK, STARROCKS, "connector-starrocks", ALPHA),
SOURCE_HUDI(SEATUNNEL, SOURCE, HUDI, "connector-hudi", BETA, BATCH, EXACTLY_ONCE, PARALLELISM),
SINK_HUDI(SEATUNNEL, SINK, HUDI, "connector-hudi", BETA),
SOURCE_ICEBERG(SEATUNNEL, SOURCE, ICEBERG, "connector-iceberg", BETA, BATCH, STREAM, EXACTLY_ONCE, SCHEMA_PROJECTION, PARALLELISM),
SINK_ICEBERG(SEATUNNEL, SINK, ICEBERG, "connector-iceberg", UNKNOWN, CDC),
SOURCE_PAIMON(SEATUNNEL, SOURCE, PAIMON, "connector-paimon", UNKNOWN, BATCH),
Expand Down
10 changes: 2 additions & 8 deletions scaleph-dao/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,8 @@
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
<groupId>com.mysql</groupId>
<artifactId>mysql-connector-j</artifactId>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,15 @@
package cn.sliew.scaleph.ds.gravitino;

import cn.sliew.carp.framework.common.dict.datasource.DataSourceType;
import cn.sliew.carp.module.datasource.modal.DataSourceInfo;
import cn.sliew.carp.module.datasource.modal.jdbc.MySQLDataSourceProperties;
import cn.sliew.carp.module.datasource.service.CarpDsInfoService;
import cn.sliew.carp.module.datasource.service.dto.DsInfoDTO;
import cn.sliew.milky.common.util.JacksonUtil;
import cn.sliew.scaleph.ds.modal.AbstractDataSource;
import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource;
import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.client.GravitinoAdminClient;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -107,12 +106,13 @@ private void doInitDataSource(String metalakeName, DataSourceType type, DsInfoDT
}

private void initMySQL(GravitinoMetalake metalake, NameIdentifier catalogName, DsInfoDTO dsInfoDTO) {
JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) JacksonUtil.toJsonNode(dsInfoDTO));
DataSourceInfo dataSourceInfo = JacksonUtil.toObject(JacksonUtil.toJsonNode(dsInfoDTO), DataSourceInfo.class);
MySQLDataSourceProperties props = (MySQLDataSourceProperties) dataSourceInfo.getProps();
Map<String, String> properties = new HashMap<>();
properties.put("jdbc-driver", dataSource.getDriverClassName());
properties.put("jdbc-url", dataSource.getUrl());
properties.put("jdbc-user", dataSource.getUser());
properties.put("jdbc-password", dataSource.getPassword());
metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSource.getRemark(), properties);
properties.put("jdbc-driver", props.getDriverClassName());
properties.put("jdbc-url", props.getUrl());
properties.put("jdbc-user", props.getUser());
properties.put("jdbc-password", props.getPassword());
metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSourceInfo.getRemark(), properties);
}
}

This file was deleted.

This file was deleted.

Loading

0 comments on commit 862a392

Please sign in to comment.