support cosn

support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn support cosn
apache · Jan 8, 2025 · d5664af · d5664af
1 parent 6cda28e
commit d5664af
Show file tree

Hide file tree

Showing 13 changed files with 523 additions and 28 deletions.
diff --git a/docs/content/maintenance/filesystems.md b/docs/content/maintenance/filesystems.md
@@ -35,12 +35,13 @@ FileSystem pluggable jars for user to query tables from Spark/Hive side.
 
 ## Supported FileSystems
 
-| FileSystem        | URI Scheme | Pluggable | Description                                                            |
-|:------------------|:-----------|-----------|:-----------------------------------------------------------------------|
-| Local File System | file://    | N         | Built-in Support                                                       |
-| HDFS              | hdfs://    | N         | Built-in Support, ensure that the cluster is in the hadoop environment |
-| Aliyun OSS        | oss://     | Y         |                                                                        |
-| S3                | s3://      | Y         |                                                                        |
+| FileSystem                   | URI Scheme | Pluggable | Description                                                            |
+|:-----------------------------|:-----------|-----------|:-----------------------------------------------------------------------|
+| Local File System            | file://    | N         | Built-in Support                                                       |
+| HDFS                         | hdfs://    | N         | Built-in Support, ensure that the cluster is in the hadoop environment |
+| Aliyun OSS                   | oss://     | Y         |                                                                        |
+| S3                           | s3://      | Y         |                                                                        |
+| Tencent Cloud Object Storage | cosn://    | Y         |                                                                        |
 
 ## Dependency
 

diff --git a/paimon-filesystems/paimon-cosn-impl/pom.xml b/paimon-filesystems/paimon-cosn-impl/pom.xml
@@ -20,37 +20,46 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
+
     <parent>
         <artifactId>paimon-filesystems</artifactId>
         <groupId>org.apache.paimon</groupId>
-        <version>1.0-SNAPSHOT</version>
+        <version>1.1-SNAPSHOT</version>
     </parent>
+
     <artifactId>paimon-cosn-impl</artifactId>
     <name>Paimon : FileSystems : COSN : Impl</name>
     <packaging>jar</packaging>
+
     <properties>
         <fs.cosn.sdk.version>3.3.5</fs.cosn.sdk.version>
         <fs.cosn.api.version>5.6.139</fs.cosn.api.version>
     </properties>
+
     <dependencies>
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-common</artifactId>
+            <artifactId>paimon-hadoop-shaded</artifactId>
             <version>${project.version}</version>
-            <scope>provided</scope>
         </dependency>
+
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-hadoop-shaded</artifactId>
+            <artifactId>paimon-common</artifactId>
             <version>${project.version}</version>
+            <scope>provided</scope>
         </dependency>
+
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-cos</artifactId>
             <version>${fs.cosn.sdk.version}</version>
             <exclusions>
                 <exclusion>
-                    <!-- provided by flink-fs-hadoop-shaded -->
+                    <groupId>com.qcloud</groupId>
+                    <artifactId>cos_api</artifactId>
+                </exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
@@ -64,15 +73,96 @@
                 </exclusion>
             </exclusions>
         </dependency>
+
         <dependency>
             <groupId>com.qcloud</groupId>
             <artifactId>cos_api</artifactId>
             <version>${fs.cosn.api.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.xml.bind</groupId>
+                    <artifactId>jaxb-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <!-- Hadoop requires jaxb-api for javax.xml.bind.JAXBException -->
+            <groupId>javax.xml.bind</groupId>
+            <artifactId>jaxb-api</artifactId>
+            <version>${jaxb.api.version}</version>
+            <!-- packaged as an optional dependency that is only accessible on Java 11+ -->
+            <scope>provided</scope>
         </dependency>
     </dependencies>
+
     <build>
         <plugins>
-            <!-- Relocate all COSN related classes -->
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifestEntries>
+                            <!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
+                            <Multi-Release>true</Multi-Release>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-javax-jars</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <artifactItems>
+                        <artifactItem>
+                            <groupId>javax.xml.bind</groupId>
+                            <artifactId>jaxb-api</artifactId>
+                            <version>${jaxb.api.version}</version>
+                            <type>jar</type>
+                            <overWrite>true</overWrite>
+                        </artifactItem>
+                    </artifactItems>
+                    <outputDirectory>${project.build.directory}/temporary</outputDirectory>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>unpack-javax-libraries</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <echo message="unpacking javax jars"/>
+                                <unzip dest="${project.build.directory}/classes/META-INF/versions/11">
+                                    <fileset dir="${project.build.directory}/temporary">
+                                        <include name="*"/>
+                                    </fileset>
+                                </unzip>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <!-- Relocate all CONS related classes -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -89,27 +179,16 @@
                                     <include>*:*</include>
                                 </includes>
                             </artifactSet>
-                            <relocations>
-                                <relocation>
-                                    <pattern>org.apache.hadoop</pattern>
-                                    <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop</shadedPattern>
-                                </relocation>
-                            </relocations>
                             <filters>
                                 <filter>
                                     <artifact>*</artifact>
                                     <excludes>
                                         <exclude>.gitkeep</exclude>
                                         <exclude>mime.types</exclude>
                                         <exclude>mozilla/**</exclude>
-                                    </excludes>
-                                </filter>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>META-INF/maven/**</exclude>
+                                        <exclude>META-INF/versions/11/META-INF/maven/**</exclude>
+                                        <exclude>META-INF/LICENSE.txt</exclude>
                                     </excludes>
                                 </filter>
                             </filters>

diff --git a/...esystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java b/...esystems/paimon-cosn-impl/src/main/java/org/apache/paimon/cosn/HadoopCompliantFileIO.java
@@ -177,6 +177,7 @@ public int read(byte[] b, int off, int len) throws IOException {
         public void close() throws IOException {
             in.close();
         }
+
         /**
          * Positions the stream to the given location. In contrast to {@link #seek(long)}, this
          * method will always issue a "seek" command to the dfs and may not replace it by {@link
@@ -190,6 +191,7 @@ public void close() throws IOException {
         public void forceSeek(long seekPos) throws IOException {
             in.seek(seekPos);
         }
+
         /**
          * Skips over a given amount of bytes in the stream.
          *

diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,90 @@
+paimon-cosn-impl
+Copyright 2023-2024 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.fasterxml.jackson.core:jackson-core:2.14.2
+- com.fasterxml.jackson.core:jackson-annotations:2.14.2
+- com.google.guava:failureaccess:1.0.1
+- com.google.errorprone:error_prone_annotations:2.5.1
+- com.google.j2objc:j2objc-annotations:1.3
+- org.apache.hadoop:hadoop-common:3.3.4
+- org.apache.hadoop:hadoop-annotations:3.3.4
+- org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.1.1
+- commons-io:commons-io:2.8.0
+- commons-collections:commons-collections:3.2.2
+- commons-logging:commons-logging:1.1.3
+- commons-beanutils:commons-beanutils:1.9.4
+- org.apache.commons:commons-configuration2:2.1.1
+- org.apache.commons:commons-lang3:3.12.0
+- org.apache.commons:commons-text:1.4
+- org.apache.hadoop:hadoop-auth:3.3.4
+- org.apache.commons:commons-compress:1.21
+- org.apache.kerby:kerb-core:1.0.1
+- org.apache.kerby:kerby-pkix:1.0.1
+- org.apache.kerby:kerby-asn1:1.0.1
+- org.apache.kerby:kerby-util:1.0.1
+- com.fasterxml.woodstox:woodstox-core:5.3.0
+- org.xerial.snappy:snappy-java:1.1.8.2
+- org.apache.hadoop:hadoop-cos:3.3.5
+- com.qcloud:cos_api-bundle:5.6.69
+- com.qcloud:cos_api:5.6.139
+- org.apache.httpcomponents:httpclient:4.5.13
+- org.apache.httpcomponents:httpcore:4.4.13
+- commons-codec:commons-codec:1.13
+- joda-time:joda-time:2.9.9
+- com.fasterxml.jackson.core:jackson-databind:2.14.1
+- com.squareup.okhttp:logging-interceptor:2.7.5
+- com.squareup.okhttp:okhttp:2.7.5
+- com.squareup.okio:okio:1.12.0
+- com.google.code.gson:gson:2.2.4
+
+This project bundles the following dependencies under BSD-2 License (https://opensource.org/licenses/BSD-2-Clause).
+You find it under licenses/LICENSE.dnsjava.
+
+- dnsjava:dnsjava:2.1.7
+
+This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
+You find them under licenses/LICENSE.checker-framework-qualifiers and licenses/LICENSE.animal-sniffer.
+
+- org.checkerframework:checker-qual:3.8.0
+- org.slf4j:slf4j-api:1.7.32
+- org.bouncycastle:bcprov-jdk15on:1.67
+- com.tencentcloudapi:tencentcloud-sdk-java-kms:3.1.213
+- com.tencentcloudapi:tencentcloud-sdk-java-common:3.1.213
+
+This project bundles the following dependencies under the CDDL 1.1 license.
+You find it under licenses/LICENSE.jaxb.
+
+- javax.xml.bind:jaxb-api:2.3.1
+
+This project bundles the following dependencies under the Go License (https://golang.org/LICENSE).
+You find it under licenses/LICENSE.re2j.
+
+- com.google.re2j:re2j:1.1
+
+This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
+You find it under licenses/LICENSE.stax2api.
+
+- org.codehaus.woodstox:stax2-api:4.2.1 (https://github.com/FasterXML/stax2-api/tree/stax2-api-4.2.1)
+- com.google.code.findbugs:jsr305:1.3.9
+
+The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs
+org.apache.hadoop.thirdparty:hadoop-shaded-guava dependency bundles the following dependencies under
+the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.google.guava:guava:30.1.1-jre
+
+The bundled Apache Hadoop Relocated (Shaded) Third-party Miscellaneous Libs
+org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7 dependency bundles the following dependencies under
+the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.google.protobuf:protobuf-java:3.7.1
+
+This project bundles the following dependencies under the Eclipse Distribution License - v 1.0
+You find it under licenses/LICENSE.jakarta
+
+- jakarta.activation:jakarta.activation-api:1.2.1
diff --git a/...-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer b/...-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.animal-sniffer
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2009 codehaus.org.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/...aimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers b/...aimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.checker-framework-qualifiers
@@ -0,0 +1,22 @@
+Checker Framework qualifiers
+Copyright 2004-present by the Checker Framework developers
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava b/paimon-filesystems/paimon-cosn-impl/src/main/resources/META-INF/licenses/LICENSE.dnsjava
@@ -0,0 +1,30 @@
+Copyright (c) 1998-2019, Brian Wellington
+Copyright (c) 2005 VeriSign. All rights reserved.
+Copyright (c) 2019-2021, dnsjava authors
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.