Merge branch 'master' of github.com:tikv/client-java into remove-shad…

…owing-of-grpc-and-netty Signed-off-by: iosmanthus <[email protected]>
tikv · Mar 28, 2024 · fb372d7 · fb372d7
2 parents d063dfb + 533dbc2
commit fb372d7
Show file tree

Hide file tree

Showing 43 changed files with 1,566 additions and 143 deletions.
diff --git a/.github/config/tikv_rawkv.toml b/.github/config/tikv_rawkv.toml
@@ -2,17 +2,20 @@
 
 [raftstore]
 # set store capacity, if no set, use disk capacity.
-capacity = "8G"
+capacity = "6G"
 pd-heartbeat-tick-interval = "2s"
 pd-store-heartbeat-tick-interval = "5s"
 split-region-check-tick-interval = "1s"
 
-[storage]
-enable-ttl = true
-
 [rocksdb]
 max-open-files = 10000
 
 [raftdb]
 max-open-files = 10000
 
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"
+enable-ttl = true
diff --git a/.github/config/tikv_txnkv.toml b/.github/config/tikv_txnkv.toml
@@ -2,7 +2,7 @@
 
 [raftstore]
 # set store capacity, if no set, use disk capacity.
-capacity = "8G"
+capacity = "6G"
 pd-heartbeat-tick-interval = "2s"
 pd-store-heartbeat-tick-interval = "5s"
 split-region-check-tick-interval = "1s"
@@ -12,3 +12,9 @@ max-open-files = 10000
 
 [raftdb]
 max-open-files = 10000
+
+[storage.block-cache]
+capacity = "128MB"
+
+[storage]
+reserve-space = "0MB"
diff --git a/.github/config/tikv_v2.toml b/.github/config/tikv_v2.toml
@@ -0,0 +1,17 @@
+# TiKV Configuration.
+
+[raftstore]
+pd-heartbeat-tick-interval = "2s"
+pd-store-heartbeat-tick-interval = "5s"
+split-region-check-tick-interval = "1s"
+
+[rocksdb]
+max-open-files = 10000
+
+[raftdb]
+max-open-files = 10000
+
+[storage]
+reserve-space = "0MB"
+api-version = 2
+enable-ttl = true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,7 +31,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        tikv_version: [nightly, v5.0.4, v5.3.0, v5.4.0]
+        tikv_version: [v5.0.6, v5.3.4, v5.4.3]
+      fail-fast: false
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 8
@@ -40,29 +41,41 @@ jobs:
           java-version: '8.0'
           distribution: 'adopt'
       - name: Install TiUP
-        run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
       - name: Start TiUP Playground
         run: |
           # Start TiKV in APIV1TTL
-          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> raw.out 2>&1 &
-
-          # The first run of `tiup` has to download all components so it'll take longer.
-          sleep 1m 30s
+          touch tiup-v1ttl.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag rawkv --mode tikv-slim --kv 1 --without-monitor --kv.port 20160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_rawkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup-v1ttl.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1ttl.log)
+          cat tiup-v1ttl.log
+          echo "Wait for bootstrap"
+          sleep 10s
 
           # Start TiKV in APIV1
-          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml &> txn.out 2>&1 &
+          touch tiup-v1.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --host 127.0.0.1 --tag txnkv --mode tikv-slim --kv 1 --without-monitor --kv.port 30160 --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_txnkv.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2381 2>&1 >> tiup-v1.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup-v1.log)
+          cat tiup-v1.log
+          echo "Wait for bootstrap"
+          sleep 10s
 
-          sleep 30s
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2381" >> $GITHUB_ENV
 
-          # Parse PD address from `tiup` output
-          echo "RAWKV_PD_ADDRESSES=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV
-          echo "TXNKV_PD_ADDRESSES=$(cat txn.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV
-
-          # Log the output
-          echo "$(cat raw.out)" >&2
-          echo "$(cat txn.out)" >&2
       - name: Run Integration Test
         run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "RawKV TiKV logs"
+          cat /home/runner/.tiup/data/rawkv/tikv-0/tikv.log
+          
+          echo "TxnKV TiKV logs"
+          cat /home/runner/.tiup/data/txnkv/tikv-0/tikv.log
       - name: Upload coverage
         uses: codecov/codecov-action@v2
         with:

diff --git a/.github/workflows/ci_v2.yml b/.github/workflows/ci_v2.yml
@@ -0,0 +1,52 @@
+name: CI (APIv2)
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  integration-test:
+    name: Integration Test - ${{ matrix.tikv_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tikv_version: [v6.5.3, v7.1.1, nightly]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8.0'
+          distribution: 'adopt'
+      - name: Install TiUP
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+          /home/runner/.tiup/bin/tiup install playground pd:${{ matrix.tikv_version }} tikv:${{ matrix.tikv_version }}
+      - name: Start TiUP Playground
+        run: |
+          # Start TiKV in APIV2
+          touch tiup.log
+          /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --tag kv --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/client-java/client-java/.github/config/tikv_v2.toml --pd.config /home/runner/work/client-java/client-java/.github/config/pd.toml --pd.port 2379 2>&1 >> tiup.log &
+          timeout 300 grep -q "PD Endpoints:" <(tail -f tiup.log)
+          cat tiup.log
+
+          # Get PD address
+          echo "RAWKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+          echo "TXNKV_PD_ADDRESSES=127.0.0.1:2379" >> $GITHUB_ENV
+
+      - name: Run Integration Test
+        run: mvn clean test
+      - name: Print TiKV logs
+        if: failure()
+        run: |
+          echo "TiKV logs"
+          cat /home/runner/.tiup/data/kv/tikv-0/tikv.log
+      - name: Upload coverage
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ github.workspace }}/target/site/jacoco/jacoco.xml
+          fail_ci_if_error: true
+          verbose: true
diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Check License Header
-        uses: apache/skywalking-eyes@main
+        uses: apache/skywalking-eyes@v0.3.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

diff --git a/.github/workflows/stale-checker.yml b/.github/workflows/stale-checker.yml
diff --git a/README.md b/README.md
@@ -4,18 +4,17 @@
 
 ## TiKV JAVA Client
 
-A Java client for [TiDB](https://github.com/pingcap/tidb)/[TiKV](https://github.com/tikv/tikv).
+A Java client for [TiKV](https://github.com/tikv/tikv).
 It is supposed to:
 + Communicate via [gRPC](http://www.grpc.io/)
 + Talk to Placement Driver searching for a region
-+ Talk to TiKV for reading/writing data and the resulted data is encoded/decoded just like what we do in TiDB.
-+ Talk to Coprocessor for calculation pushdown
++ Talk to TiKV for reading/writing data
 
 ## Quick Start
 
-> TiKV Java Client is designed to communicate with [pd](https://github.com/tikv/pd) and [tikv](https://github.com/tikv/tikv), please run TiKV and PD in advance.
+> TiKV Java Client is designed to communicate with [PD](https://github.com/tikv/pd) and [TiKV](https://github.com/tikv/tikv), please run PD and TiKV in advance.
 
-Build java client from source file:
+Build Java client from source file:
 
 ```sh
 mvn clean install -Dmaven.test.skip=true
@@ -27,28 +26,43 @@ Add maven dependency to `pom.xml`:
 <dependency>
 	<groupId>org.tikv</groupId>
 	<artifactId>tikv-client-java</artifactId>
-	<version>3.1.0</version>
+	<version>3.3.0</version>
 </dependency>
 ```
 
-Create a RawKVClient and communicates with TiKV:
+Create a transactional `KVClient` and communicates with TiKV:
+
+```java
+import org.tikv.common.TiConfiguration;
+import org.tikv.common.TiSession;
+import org.tikv.txn.KVClient;
+
+public class Main {
+	public static void main(String[] args) throws Exception {
+		TiConfiguration conf = TiConfiguration.createDefault(YOUR_PD_ADDRESSES);
+		TiSession session = TiSession.create(conf);
+		KVClient client = session.createKVClient();
+	}
+}
+```
+
+Or create a `RawKVClient` if you don't need the transaction semantic:
 
 ```java
 import org.tikv.common.TiConfiguration;
 import org.tikv.common.TiSession;
 import org.tikv.raw.RawKVClient;
 
 public class Main {
-	public static void main() {
-		// You MUST create a raw configuration if you are using RawKVClient.
+	public static void main(String[] args) throws Exception {
 		TiConfiguration conf = TiConfiguration.createRawDefault(YOUR_PD_ADDRESSES);
 		TiSession session = TiSession.create(conf);
 		RawKVClient client = session.createRawClient();
 	}
 }
 ```
 
-Find more demo in [KVRawClientTest](https://github.com/birdstorm/KVRawClientTest/)
+Find more demo in [TiKV Java Client User Documents](https://tikv.github.io/client-java/examples/introduction.html)
 
 ## Documentation
 

diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -4,6 +4,8 @@
 
 - [Introduction](./introduction/introduction.md)
 
+- [Production Readiness](./production-readiness.md)
+
 - [Start With Examples](./examples/introduction.md)
     - [Quick Start](./examples/quick-start.md)
     - [Interact with TiKV RawKV API](./examples/rawkv.md)

diff --git a/docs/src/examples/quick-start.md b/docs/src/examples/quick-start.md
@@ -24,7 +24,7 @@ Add maven dependency to `pom.xml`.
 <dependency>
   <groupId>org.tikv</groupId>
   <artifactId>tikv-client-java</artifactId>
-  <version>3.1.0</version>
+  <version>3.3.0</version>
 </dependency>
 <dependency>
   <groupId>org.slf4j</groupId>

diff --git a/docs/src/examples/rawkv.md b/docs/src/examples/rawkv.md
@@ -15,7 +15,7 @@ import org.tikv.raw.RawKVClient;
 import org.tikv.shade.com.google.protobuf.ByteString;
 
 public class Main {
-  public static void main() {
+  public static void main(String[] args) throws Exception {
     // You MUST create a raw configuration if you are using RawKVClient.
     TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
     TiSession session = TiSession.create(conf);
@@ -61,7 +61,7 @@ To enable the API V2 mode, users need to specify the API version of the client.
 import org.tikv.common.TiConfiguration.ApiVersion;
 
 public class Main {
-  public static void main() {
+  public static void main(String[] args) throws Exception {
     TiConfiguration conf = TiConfiguration.createRawDefault("127.0.0.1:2379");
     conf.setApiVersion(ApiVersion.V2);
     try(TiSession session = TiSession.create(conf)) {

diff --git a/docs/src/examples/txnkv.md b/docs/src/examples/txnkv.md
@@ -23,7 +23,7 @@ import org.tikv.txn.TwoPhaseCommitter;
 
 public class App {
     public static void main(String[] args) throws Exception {
-        TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2389");
+        TiConfiguration conf = TiConfiguration.createDefault("127.0.0.1:2379");
         try (TiSession session = TiSession.create(conf)) {
             // two-phrase write
             long startTS = session.getTimestamp().getVersion();

diff --git a/docs/src/production-readiness.md b/docs/src/production-readiness.md
@@ -0,0 +1,18 @@
+# Production Readiness
+
+In general, the latest [release](https://github.com/tikv/client-java/releases) of TiKV Java Client is ready for production use. But it is not battle-tested as full featured client for TiKV in all use cases. This page will give you more details.
+
+## RawKV
+All RawKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+At this time, RawKV has been used in the production environment of some commercial customers in latency sensitive systems. But they only use part of the RawKV APIs (mainly including `raw_put`, `raw_get`, `raw_compare_and_swap`, and `raw_batch_put`).
+
+## TxnKV
+All TxnKV APIs are covered by [CI](https://github.com/tikv/client-java/actions/workflows/ci.yml).
+
+In addition, TxnKV has been used in the [TiSpark](https://docs.pingcap.com/tidb/stable/tispark-overview) and [TiBigData](https://github.com/tidb-incubator/TiBigData) project to integrate data from TiDB to Big Data ecosystem. TiSpark and TiBigData are used in the production system of some commercial customers and internet companies.
+
+Similar to RawKV, only part of APIs are used in this scenario (mainly including `prewrite/commit` and `coprocessor`). And this use case doesn't care about latency but throughput and reliability.
+
+## TiDB Cloud
+Directly using TiKV is not possible on TiDB Cloud due to the fact that client has to access the whole cluster, which has security issues. And TiKV managed service is not coming soon as it's not contained in [roadmap](https://docs.pingcap.com/tidbcloud/tidb-cloud-roadmap) yet.
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.tikv</groupId>
     <artifactId>tikv-client-java</artifactId>
-    <version>3.3.0-SNAPSHOT</version>
+    <version>3.3.4-SNAPSHOT</version>
     <packaging>jar</packaging>
     <name>TiKV Java Client</name>
     <description>A Java Client for TiKV</description>
@@ -54,15 +54,15 @@
         <maven.compiler.target>1.8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <protobuf.version>3.5.1</protobuf.version>
+        <protobuf.version>3.18.0</protobuf.version>
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.7.16</slf4j.version>
         <grpc.version>1.48.0</grpc.version>
         <netty.tcnative.version>2.0.34.Final</netty.tcnative.version>
         <gson.version>2.8.9</gson.version>
         <powermock.version>1.6.6</powermock.version>
         <jackson-annotations.version>2.13.2</jackson-annotations.version>
-        <jackson.version>2.13.2.2</jackson.version>
+        <jackson.version>2.13.4.2</jackson.version>
         <trove4j.version>3.0.1</trove4j.version>
         <jetcd.version>0.4.1</jetcd.version>
         <joda-time.version>2.9.9</joda-time.version>