From 9b0aeb0be809c6934232b00439d1e2e28b59e394 Mon Sep 17 00:00:00 2001 From: liujh Date: Tue, 7 Jan 2025 14:37:21 +0800 Subject: [PATCH 1/2] support cosn --- paimon-filesystems/paimon-cosn-impl/pom.xml | 122 ++++++++ .../org/apache/paimon/cosn/COSNFileIO.java | 151 ++++++++++ .../paimon/cosn/HadoopCompliantFileIO.java | 281 ++++++++++++++++++ paimon-filesystems/paimon-cosn/pom.xml | 113 +++++++ .../org/apache/paimon/cosn/COSNLoader.java | 73 +++++ .../org.apache.paimon.fs.FileIOLoader | 15 + paimon-filesystems/pom.xml | 2 + .../org/apache/paimon/flink/FlinkFileIO.java | 3 +- paimon-flink/pom.xml | 4 + paimon-format/pom.xml | 4 + 10 files changed, 767 insertions(+), 1 deletion(-) create mode 100644 paimon-filesystems/paimon-cosn-impl/pom.xml create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java create mode 100644 paimon-filesystems/paimon-cosn/pom.xml create mode 100644 paimon-filesystems/paimon-cosn/src/main/java/org/apache/paimon/cosn/COSNLoader.java create mode 100644 paimon-filesystems/paimon-cosn/src/main/resources/META-INF.services/org.apache.paimon.fs.FileIOLoader diff --git a/paimon-filesystems/paimon-cosn-impl/pom.xml b/paimon-filesystems/paimon-cosn-impl/pom.xml new file mode 100644 index 000000000000..1a025cd2bbba --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/pom.xml @@ -0,0 +1,122 @@ + + + + 4.0.0 + + paimon-filesystems + org.apache.paimon + 1.0-SNAPSHOT + + paimon-cosn-impl + Paimon : FileSystems : COSN : Impl + jar + + 3.3.5 + 5.6.139 + + + + org.apache.paimon + paimon-common + ${project.version} + provided + + + org.apache.paimon + paimon-hadoop-shaded + ${project.version} + + + org.apache.hadoop + hadoop-cos + ${fs.cosn.sdk.version} + + + + org.apache.hadoop + hadoop-common + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + com.qcloud + cos_api + ${fs.cosn.api.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + *:* + + + + + org.apache.hadoop + org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop + + + + + * + + .gitkeep + mime.types + mozilla/** + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + \ No newline at end of file diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java new file mode 100644 index 000000000000..f09535edf1f7 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/COSNFileIO.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cosn; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.options.Options; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.cosn.CosNFileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** COSN {@link FileIO}. */ +public class COSNFileIO extends HadoopCompliantFileIO { + private static final long serialVersionUID = 1L; + private static final Logger LOG = LoggerFactory.getLogger(COSNFileIO.class); + /** + * In order to simplify, we make paimon cosn configuration keys same with hadoop cosn module. + * So, we add all configuration key with prefix `fs.cosn` in paimon conf to hadoop conf. + */ + private static final String[] CONFIG_PREFIXES = {"fs.cosn."}; + + private static final String COSN_USER_INFO_KEY_ID = "fs.cosn.userinfo.secretId"; + private static final String COSN_USER_INFO_KEY_SECRET = "fs.cosn.userinfo.secretKey"; + private static final Map CASE_SENSITIVE_KEYS = + new HashMap() { + { + put(COSN_USER_INFO_KEY_ID.toLowerCase(), COSN_USER_INFO_KEY_ID); + put(COSN_USER_INFO_KEY_SECRET.toLowerCase(), COSN_USER_INFO_KEY_SECRET); + } + }; + /** + * Cache CosNFileSystem, at present, there is no good mechanism to ensure that the file system + * will be shut down, so here the fs cache is used to avoid resource leakage. + */ + private static final Map CACHE = new ConcurrentHashMap<>(); + + private Options hadoopOptions; + + @Override + public boolean isObjectStore() { + return true; + } + + @Override + public void configure(CatalogContext context) { + hadoopOptions = new Options(); + // read all configuration with prefix 'CONFIG_PREFIXES' + for (String key : context.options().keySet()) { + for (String prefix : CONFIG_PREFIXES) { + if (key.startsWith(prefix)) { + String value = context.options().get(key); + if (CASE_SENSITIVE_KEYS.containsKey(key.toLowerCase())) { + key = CASE_SENSITIVE_KEYS.get(key.toLowerCase()); + } + hadoopOptions.set(key, value); + LOG.warn( + "Adding config entry for {} as {} to Hadoop config", + key, + hadoopOptions.get(key)); + } + } + } + } + + @Override + protected FileSystem createFileSystem(org.apache.hadoop.fs.Path path) { + final String scheme = path.toUri().getScheme(); + final String authority = path.toUri().getAuthority(); + return CACHE.computeIfAbsent( + new CacheKey(hadoopOptions, scheme, authority), + key -> { + Configuration hadoopConf = new Configuration(); + key.options.toMap().forEach(hadoopConf::set); + URI fsUri = path.toUri(); + if (scheme == null && authority == null) { + fsUri = FileSystem.getDefaultUri(hadoopConf); + } else if (scheme != null && authority == null) { + URI defaultUri = FileSystem.getDefaultUri(hadoopConf); + if (scheme.equals(defaultUri.getScheme()) + && defaultUri.getAuthority() != null) { + fsUri = defaultUri; + } + } + CosNFileSystem fs = new CosNFileSystem(); + try { + fs.initialize(fsUri, hadoopConf); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return fs; + }); + } + + private static class CacheKey { + private final Options options; + private final String scheme; + private final String authority; + + private CacheKey(Options options, String scheme, String authority) { + this.options = options; + this.scheme = scheme; + this.authority = authority; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CacheKey cacheKey = (CacheKey) o; + return Objects.equals(options, cacheKey.options) + && Objects.equals(scheme, cacheKey.scheme) + && Objects.equals(authority, cacheKey.authority); + } + + @Override + public int hashCode() { + return Objects.hash(options, scheme, authority); + } + } +} diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java new file mode 100644 index 000000000000..6af5fef4f429 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cosn; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** + * Hadoop {@link FileIO}. + * + *

Important: copy this class from HadoopFileIO here to avoid class loader conflicts. + */ +public abstract class HadoopCompliantFileIO implements FileIO { + private static final long serialVersionUID = 1L; + protected transient volatile FileSystem fs; + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return new HadoopSeekableInputStream(getFileSystem(hadoopPath).open(hadoopPath)); + } + + @Override + public PositionOutputStream newOutputStream(Path path, boolean overwrite) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return new HadoopPositionOutputStream( + getFileSystem(hadoopPath).create(hadoopPath, overwrite)); + } + + @Override + public FileStatus getFileStatus(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return new HadoopFileStatus(getFileSystem(hadoopPath).getFileStatus(hadoopPath)); + } + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + FileStatus[] statuses = new FileStatus[0]; + org.apache.hadoop.fs.FileStatus[] hadoopStatuses = + getFileSystem(hadoopPath).listStatus(hadoopPath); + if (hadoopStatuses != null) { + statuses = new FileStatus[hadoopStatuses.length]; + for (int i = 0; i < hadoopStatuses.length; i++) { + statuses[i] = new HadoopFileStatus(hadoopStatuses[i]); + } + } + return statuses; + } + + @Override + public boolean exists(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return getFileSystem(hadoopPath).exists(hadoopPath); + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return getFileSystem(hadoopPath).delete(hadoopPath, recursive); + } + + @Override + public boolean mkdirs(Path path) throws IOException { + org.apache.hadoop.fs.Path hadoopPath = path(path); + return getFileSystem(hadoopPath).mkdirs(hadoopPath); + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + org.apache.hadoop.fs.Path hadoopSrc = path(src); + org.apache.hadoop.fs.Path hadoopDst = path(dst); + return getFileSystem(hadoopSrc).rename(hadoopSrc, hadoopDst); + } + + private org.apache.hadoop.fs.Path path(Path path) { + return new org.apache.hadoop.fs.Path(path.toUri()); + } + + private FileSystem getFileSystem(org.apache.hadoop.fs.Path path) throws IOException { + if (fs == null) { + synchronized (this) { + if (fs == null) { + fs = createFileSystem(path); + } + } + } + return fs; + } + + protected abstract FileSystem createFileSystem(org.apache.hadoop.fs.Path path) + throws IOException; + + private static class HadoopSeekableInputStream extends SeekableInputStream { + /** + * Minimum amount of bytes to skip forward before we issue a seek instead of discarding + * read. + * + *

The current value is just a magic number. In the long run, this value could become + * configurable, but for now it is a conservative, relatively small value that should bring + * safe improvements for small skips (e.g. in reading meta data), that would hurt the most + * with frequent seeks. + * + *

The optimal value depends on the DFS implementation and configuration plus the + * underlying filesystem. For now, this number is chosen "big enough" to provide + * improvements for smaller seeks, and "small enough" to avoid disadvantages over real + * seeks. While the minimum should be the page size, a true optimum per system would be the + * amounts of bytes the can be consumed sequentially within the seektime. Unfortunately, + * seektime is not constant and devices, OS, and DFS potentially also use read buffers and + * read-ahead. + */ + private static final int MIN_SKIP_BYTES = 1024 * 1024; + + private final FSDataInputStream in; + + private HadoopSeekableInputStream(FSDataInputStream in) { + this.in = in; + } + + @Override + public void seek(long seekPos) throws IOException { + // We do some optimizations to avoid that some implementations of distributed FS perform + // expensive seeks when they are actually not needed. + long delta = seekPos - getPos(); + if (delta > 0L && delta <= MIN_SKIP_BYTES) { + // Instead of a small forward seek, we skip over the gap + skipFully(delta); + } else if (delta != 0L) { + // For larger gaps and backward seeks, we do a real seek + forceSeek(seekPos); + } // Do nothing if delta is zero. + } + + @Override + public long getPos() throws IOException { + return in.getPos(); + } + + @Override + public int read() throws IOException { + return in.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return in.read(b, off, len); + } + + @Override + public void close() throws IOException { + in.close(); + } + /** + * Positions the stream to the given location. In contrast to {@link #seek(long)}, this + * method will always issue a "seek" command to the dfs and may not replace it by {@link + * #skip(long)} for small seeks. + * + *

Notice that the underlying DFS implementation can still decide to do skip instead of + * seek. + * + * @param seekPos the position to seek to. + */ + public void forceSeek(long seekPos) throws IOException { + in.seek(seekPos); + } + /** + * Skips over a given amount of bytes in the stream. + * + * @param bytes the number of bytes to skip. + */ + public void skipFully(long bytes) throws IOException { + while (bytes > 0) { + bytes -= in.skip(bytes); + } + } + } + + private static class HadoopPositionOutputStream extends PositionOutputStream { + private final FSDataOutputStream out; + + private HadoopPositionOutputStream(FSDataOutputStream out) { + this.out = out; + } + + @Override + public long getPos() { + return out.getPos(); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void flush() throws IOException { + out.hflush(); + } + + @Override + public void close() throws IOException { + out.close(); + } + } + + private static class HadoopFileStatus implements FileStatus { + private final org.apache.hadoop.fs.FileStatus status; + + private HadoopFileStatus(org.apache.hadoop.fs.FileStatus status) { + this.status = status; + } + + @Override + public long getLen() { + return status.getLen(); + } + + @Override + public boolean isDir() { + return status.isDirectory(); + } + + @Override + public Path getPath() { + return new Path(status.getPath().toUri()); + } + + @Override + public long getModificationTime() { + return status.getModificationTime(); + } + + @Override + public long getAccessTime() { + return status.getAccessTime(); + } + + @Nullable + @Override + public String getOwner() { + return status.getOwner(); + } + } +} diff --git a/paimon-filesystems/paimon-cosn/pom.xml b/paimon-filesystems/paimon-cosn/pom.xml new file mode 100644 index 000000000000..84d32ad4650e --- /dev/null +++ b/paimon-filesystems/paimon-cosn/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + + paimon-filesystems + org.apache.paimon + 1.0-SNAPSHOT + + paimon-cosn + Paimon : FileSystems : COSN + jar + + + org.apache.paimon + paimon-common + ${project.version} + provided + + + org.apache.paimon + paimon-cosn-impl + ${project.version} + runtime + true + + + * + * + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + true + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-cosn-classes + prepare-package + + unpack + + + + + org.apache.paimon + paimon-cosn-impl + ${project.version} + jar + true + ${project.build.directory}/classes/paimon-plugin-cosn + META-INF/services/** + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + + + + org.apache.paimon:paimon-cosn-impl + + + + + org.apache.paimon:paimon-cosn-impl + + META-INF/** + + + META-INF/services/** + + + + + + + + + + \ No newline at end of file diff --git a/paimon-filesystems/paimon-cosn/src/main/java/org/apache/paimon/cosn/COSNLoader.java b/paimon-filesystems/paimon-cosn/src/main/java/org/apache/paimon/cosn/COSNLoader.java new file mode 100644 index 000000000000..6d86fdc1791e --- /dev/null +++ b/paimon-filesystems/paimon-cosn/src/main/java/org/apache/paimon/cosn/COSNLoader.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cosn; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileIOLoader; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PluginFileIO; +import org.apache.paimon.plugin.PluginLoader; + +/** A {@link PluginLoader} to load cosn. */ +public class COSNLoader implements FileIOLoader { + private static final String COSN_JAR = "paimon-plugin-cosn"; + private static final String COSN_CLASS = "org.apache.paimon.cosn.COSNFileIO"; + // Singleton lazy initialization + private static PluginLoader loader; + + private static synchronized PluginLoader getLoader() { + if (loader == null) { + // Avoid NoClassDefFoundError without cause by exception + loader = new PluginLoader(COSN_JAR); + } + return loader; + } + + @Override + public String getScheme() { + return "cosn"; + } + + @Override + public FileIO load(Path path) { + return new COSNPluginFileIO(); + } + + private static class COSNPluginFileIO extends PluginFileIO { + private static final long serialVersionUID = 1L; + + @Override + public boolean isObjectStore() { + return true; + } + + @Override + protected FileIO createFileIO(Path path) { + FileIO fileIO = getLoader().newInstance(COSN_CLASS); + fileIO.configure(CatalogContext.create(options)); + return fileIO; + } + + @Override + protected ClassLoader pluginClassLoader() { + return getLoader().submoduleClassLoader(); + } + } +} diff --git a/paimon-filesystems/paimon-cosn/src/main/resources/META-INF.services/org.apache.paimon.fs.FileIOLoader b/paimon-filesystems/paimon-cosn/src/main/resources/META-INF.services/org.apache.paimon.fs.FileIOLoader new file mode 100644 index 000000000000..b17805b5bc6d --- /dev/null +++ b/paimon-filesystems/paimon-cosn/src/main/resources/META-INF.services/org.apache.paimon.fs.FileIOLoader @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.paimon.cosn.COSNLoader \ No newline at end of file diff --git a/paimon-filesystems/pom.xml b/paimon-filesystems/pom.xml index 405350a3312b..1eede962e891 100644 --- a/paimon-filesystems/pom.xml +++ b/paimon-filesystems/pom.xml @@ -37,6 +37,8 @@ paimon-oss-impl paimon-s3 paimon-s3-impl + paimon-cosn + paimon-cosn-impl diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java index 617d25125f37..75906f2356b0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java @@ -55,7 +55,8 @@ public boolean isObjectStore() { || scheme.startsWith("emr") || scheme.startsWith("oss") || scheme.startsWith("wasb") - || scheme.startsWith("gs")) { + || scheme.startsWith("gs") + || scheme.startsWith("cosn")) { // the Amazon S3 storage or Aliyun OSS storage or Azure Blob Storage // or Google Cloud Storage return true; diff --git a/paimon-flink/pom.xml b/paimon-flink/pom.xml index 031570c0e684..cf0563bc9640 100644 --- a/paimon-flink/pom.xml +++ b/paimon-flink/pom.xml @@ -132,6 +132,10 @@ under the License. org.slf4j slf4j-log4j12 + + log4j + log4j + jdk.tools jdk.tools diff --git a/paimon-format/pom.xml b/paimon-format/pom.xml index 5804a704b0cf..a99a9a95dea2 100644 --- a/paimon-format/pom.xml +++ b/paimon-format/pom.xml @@ -286,6 +286,10 @@ under the License. jdk.tools jdk.tools + + log4j + log4j + From ca9c1d8ed9701fffe6a9c53203dff3ae957ca9d7 Mon Sep 17 00:00:00 2001 From: liujh Date: Tue, 7 Jan 2025 16:12:20 +0800 Subject: [PATCH 2/2] support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn --- docs/content/maintenance/filesystems.md | 13 +- paimon-filesystems/paimon-cosn-impl/pom.xml | 119 ++++++++++++--- .../paimon/cosn/HadoopCompliantFileIO.java | 2 + .../src/main/resources/META-INF/NOTICE | 97 +++++++++++++ .../META-INF/licenses/LICENSE.animal-sniffer | 21 +++ .../LICENSE.checker-framework-qualifiers | 22 +++ .../META-INF/licenses/LICENSE.dnsjava | 30 ++++ .../META-INF/licenses/LICENSE.jakarta | 28 ++++ .../resources/META-INF/licenses/LICENSE.jaxb | 135 ++++++++++++++++++ .../resources/META-INF/licenses/LICENSE.re2j | 32 +++++ .../META-INF/licenses/LICENSE.stax2api | 22 +++ paimon-filesystems/paimon-cosn/pom.xml | 35 ++++- .../tools/ci/licensecheck/LicenseChecker.java | 2 + 13 files changed, 530 insertions(+), 28 deletions(-) create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jakarta create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jaxb create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.re2j create mode 100644 paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.stax2api diff --git a/docs/content/maintenance/filesystems.md b/docs/content/maintenance/filesystems.md index dc030a9ec2bd..8a85eeaf39aa 100644 --- a/docs/content/maintenance/filesystems.md +++ b/docs/content/maintenance/filesystems.md @@ -35,12 +35,13 @@ FileSystem pluggable jars for user to query tables from Spark/Hive side. ## Supported FileSystems -| FileSystem | URI Scheme | Pluggable | Description | -|:------------------|:-----------|-----------|:-----------------------------------------------------------------------| -| Local File System | file:// | N | Built-in Support | -| HDFS | hdfs:// | N | Built-in Support, ensure that the cluster is in the hadoop environment | -| Aliyun OSS | oss:// | Y | | -| S3 | s3:// | Y | | +| FileSystem | URI Scheme | Pluggable | Description | +|:-----------------------------|:-----------|-----------|:-----------------------------------------------------------------------| +| Local File System | file:// | N | Built-in Support | +| HDFS | hdfs:// | N | Built-in Support, ensure that the cluster is in the hadoop environment | +| Aliyun OSS | oss:// | Y | | +| S3 | s3:// | Y | | +| Tencent Cloud Object Storage | cosn:// | Y | | ## Dependency diff --git a/paimon-filesystems/paimon-cosn-impl/pom.xml b/paimon-filesystems/paimon-cosn-impl/pom.xml index 1a025cd2bbba..9fecf7fa78fd 100644 --- a/paimon-filesystems/paimon-cosn-impl/pom.xml +++ b/paimon-filesystems/paimon-cosn-impl/pom.xml @@ -20,37 +20,46 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + paimon-filesystems org.apache.paimon - 1.0-SNAPSHOT + 1.1-SNAPSHOT + paimon-cosn-impl Paimon : FileSystems : COSN : Impl jar + 3.3.5 5.6.139 + org.apache.paimon - paimon-common + paimon-hadoop-shaded ${project.version} - provided + org.apache.paimon - paimon-hadoop-shaded + paimon-common ${project.version} + provided + org.apache.hadoop hadoop-cos ${fs.cosn.sdk.version} - + com.qcloud + cos_api + + org.apache.hadoop hadoop-common @@ -64,15 +73,96 @@ + com.qcloud cos_api ${fs.cosn.api.version} + + + javax.xml.bind + jaxb-api + + + + + + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + + provided + - + + + org.apache.maven.plugins + maven-jar-plugin + + + + + true + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-javax-jars + process-resources + + copy + + + + + + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + jar + true + + + ${project.build.directory}/temporary + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + unpack-javax-libraries + process-resources + + run + + + + + + + + + + + + + + + + org.apache.maven.plugins maven-shade-plugin @@ -89,12 +179,6 @@ *:* - - - org.apache.hadoop - org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop - - * @@ -102,14 +186,9 @@ .gitkeep mime.types mozilla/** - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA + META-INF/maven/** + META-INF/versions/11/META-INF/maven/** + META-INF/LICENSE.txt diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java index 6af5fef4f429..4cfa26fd0f5a 100644 --- a/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java @@ -177,6 +177,7 @@ public int read(byte[] b, int off, int len) throws IOException { public void close() throws IOException { in.close(); } + /** * Positions the stream to the given location. In contrast to {@link #seek(long)}, this * method will always issue a "seek" command to the dfs and may not replace it by {@link @@ -190,6 +191,7 @@ public void close() throws IOException { public void forceSeek(long seekPos) throws IOException { in.seek(seekPos); } + /** * Skips over a given amount of bytes in the stream. * diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..25a08bcb2b39 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE @@ -0,0 +1,97 @@ +paimon-cosn-impl +Copyright 2023-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.fasterxml.jackson.core:jackson-core:2.14.2 +- com.fasterxml.jackson.core:jackson-annotations:2.14.2 +- com.google.guava:failureaccess:1.0.1 +- com.google.errorprone:error_prone_annotations:2.5.1 +- com.google.j2objc:j2objc-annotations:1.3 +- org.apache.hadoop:hadoop-common:3.3.4 +- org.apache.hadoop:hadoop-annotations:3.3.4 +- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1 +- org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:1.1.1 +- com.google.guava:guava:27.0-jre +- com.google.guava:failureaccess:1.0 +- com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava +- com.google.j2objc:j2objc-annotations:1.1 +- org.codehaus.mojo:animal-sniffer-annotations:1.17 +- commons-io:commons-io:2.8.0 +- commons-collections:commons-collections:3.2.2 +- commons-logging:commons-logging:1.1.3 +- commons-beanutils:commons-beanutils:1.9.4 +- org.apache.commons:commons-configuration2:2.1.1 +- org.apache.commons:commons-lang3:3.12.0 +- org.apache.commons:commons-text:1.4 +- org.apache.hadoop:hadoop-auth:3.3.4 +- org.apache.commons:commons-compress:1.21 +- org.apache.kerby:kerb-core:1.0.1 +- org.apache.kerby:kerby-pkix:1.0.1 +- org.apache.kerby:kerby-asn1:1.0.1 +- org.apache.kerby:kerby-util:1.0.1 +- com.fasterxml.woodstox:woodstox-core:5.3.0 +- org.xerial.snappy:snappy-java:1.1.8.2 +- org.apache.hadoop:hadoop-cos:3.3.5 +- com.qcloud:cos_api-bundle:5.6.69 +- com.qcloud:cos_api:5.6.139 +- org.apache.httpcomponents:httpclient:4.5.13 +- org.apache.httpcomponents:httpcore:4.4.13 +- commons-codec:commons-codec:1.13 +- joda-time:joda-time:2.9.9 +- com.fasterxml.jackson.core:jackson-databind:2.14.1 +- com.squareup.okhttp:logging-interceptor:2.7.5 +- com.squareup.okhttp:okhttp:2.7.5 +- com.squareup.okio:okio:1.12.0 +- com.google.code.gson:gson:2.2.4 + +This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause). +You find it under licenses/LICENSE.dnsjava. + +- dnsjava:dnsjava:2.1.7 + +This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT) +You find them under licenses/LICENSE.checker-framework-qualifiers and licenses/LICENSE.animal-sniffer. + +- org.checkerframework:checker-qual:3.8.0 +- org.checkerframework:checker-qual:2.5.2 +- org.slf4j:slf4j-api:1.7.32 +- org.bouncycastle:bcprov-jdk15on:1.67 +- com.tencentcloudapi:tencentcloud-sdk-java-kms:3.1.213 +- com.tencentcloudapi:tencentcloud-sdk-java-common:3.1.213 + +This project bundles the following dependencies under the CDDL 1.1 license. +You find it under licenses/LICENSE.jaxb. + +- javax.xml.bind:jaxb-api:2.3.1 + +This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). +You find it under licenses/LICENSE.re2j. + +- com.google.re2j:re2j:1.1 + +This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). +You find it under licenses/LICENSE.stax2api. + +- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1) +- com.google.code.findbugs:jsr305:1.3.9 + +The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs +org.apache.hadoop.thirdparty:hadoop-shaded-guava dependency bundles the following dependencies under +the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:30.1.1-jre + +The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7 dependency bundles the following dependencies under +the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.protobuf:protobuf-java:3.7.1 + +This project bundles the following dependencies under the Eclipse Distribution License - v 1.0 +You find it under licenses/LICENSE.jakarta + +- jakarta.activation:jakarta.activation-api:1.2.1 diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer new file mode 100644 index 000000000000..2062eb88b46e --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2009 codehaus.org. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers new file mode 100644 index 000000000000..7b59b5c9820a --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers @@ -0,0 +1,22 @@ +Checker Framework qualifiers +Copyright 2004-present by the Checker Framework developers + +MIT License: + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava new file mode 100644 index 000000000000..8daf3fc25498 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava @@ -0,0 +1,30 @@ +Copyright (c) 1998-2019, Brian Wellington +Copyright (c) 2005 VeriSign. All rights reserved. +Copyright (c) 2019-2021, dnsjava authors + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jakarta b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jakarta new file mode 100644 index 000000000000..0dea72127c36 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jakarta @@ -0,0 +1,28 @@ +Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jaxb b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jaxb new file mode 100644 index 000000000000..fd16ea9546a4 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.jaxb @@ -0,0 +1,135 @@ +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 + +1. Definitions. + + 1.1. "Contributor" means each individual or entity that creates or contributes to the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + + 1.3. "Covered Software" means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + + 1.4. "Executable" means the Covered Software in any form other than Source Code. + + 1.5. "Initial Developer" means the individual or entity that first makes Original Software available under this License. + + 1.6. "Larger Work" means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + + 1.7. "License" means this document. + + 1.8. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means the Source Code and Executable form of any of the following: + + A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + + B. Any new file that contains any part of the Original Software or previous Modification; or + + C. Any new file that is contributed or otherwise made available under the terms of this License. + + 1.10. "Original Software" means the Source Code and Executable form of computer software code that is originally released under this License. + + 1.11. "Patent Claims" means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + + 1.12. "Source Code" means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + + 1.13. "You" (or "Your") means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as "Participant") alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + + The Covered Software is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" (as that term is defined at 48 C.F.R. ? 252.227-7014(a)(1)) and "commercial computer software documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + + This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +---------- +NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) +The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.re2j b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.re2j new file mode 100644 index 000000000000..b620ae68fe33 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.re2j @@ -0,0 +1,32 @@ +This is a work derived from Russ Cox's RE2 in Go, whose license +http://golang.org/LICENSE is as follows: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.stax2api b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.stax2api new file mode 100644 index 000000000000..0ed636169965 --- /dev/null +++ b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.stax2api @@ -0,0 +1,22 @@ +Copyright woodstox stax2api contributors. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/paimon-filesystems/paimon-cosn/pom.xml b/paimon-filesystems/paimon-cosn/pom.xml index 84d32ad4650e..7ef2f8d0dcc2 100644 --- a/paimon-filesystems/paimon-cosn/pom.xml +++ b/paimon-filesystems/paimon-cosn/pom.xml @@ -1,16 +1,36 @@ + 4.0.0 + paimon-filesystems org.apache.paimon - 1.0-SNAPSHOT + 1.1-SNAPSHOT + paimon-cosn Paimon : FileSystems : COSN jar + org.apache.paimon @@ -18,6 +38,7 @@ ${project.version} provided + org.apache.paimon paimon-cosn-impl @@ -32,6 +53,7 @@ + @@ -73,7 +95,16 @@ jar true ${project.build.directory}/classes/paimon-plugin-cosn - META-INF/services/** + META-INF/** + + + org.apache.paimon + paimon-cosn-impl + ${project.version} + jar + true + ${project.build.directory}/classes/paimon-plugin-cosn + META-INF/services/**,META-INF/versions/** diff --git a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java index 9142f104a943..0b04377c63e5 100644 --- a/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java +++ b/tools/ci/paimon-ci-tools/src/main/java/org/apache/paimon/tools/ci/licensecheck/LicenseChecker.java @@ -41,6 +41,8 @@ public static void main(String[] args) throws Exception { int severeIssueCount = NoticeFileChecker.run(new File(args[0]), Paths.get(args[1])); + LOG.warn("Found a total of {} severe issues", severeIssueCount); + severeIssueCount += JarFileChecker.checkPath(Paths.get(args[2])); if (severeIssueCount > 0) {