Add Zipkin Dependencies support of OpenSearch storage

Signed-off-by: Andriy Redko <[email protected]>
openzipkin · May 11, 2024 · b3523e3 · b3523e3
1 parent 8e8ea60
commit b3523e3
Show file tree

Hide file tree

Showing 15 changed files with 852 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -16,6 +16,7 @@ are supported, including Cassandra, MySQL and Elasticsearch.
 * `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 4.0
 * `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 10.11
 * `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+; tested against last minor release of 7.x and 8.x
+* `STORAGE_TYPE=opensearch` : requires OpenSearch 2+; tested against last minor release of 2.x
 
 ## Quick-start
 
@@ -117,6 +118,31 @@ $ STORAGE_TYPE=elasticsearch ES_HOSTS=host1,host2 java -jar zipkin-dependencies.
 $ STORAGE_TYPE=elasticsearch ES_HOSTS=host1:9201 java -jar zipkin-dependencies.jar
 ```
 
+### OpenSearch Storage
+OpenSearch is used when `STORAGE_TYPE=opensearch`. The schema is compatible with Zipkin's [Elasticsearch storage component](https://github.com/openzipkin/zipkin/tree/master/zipkin-storage/elasticsearch).
+
+    * `OS_INDEX`: The index prefix to use when generating daily index names. Defaults to zipkin.
+    * `OS_DATE_SEPARATOR`: The separator used when generating dates in index.
+                           Defaults to '-' so the queried index look like zipkin-yyyy-DD-mm
+                           Could for example be changed to '.' to give zipkin-yyyy.MM.dd
+    * `OS_HOSTS`: A comma separated list of OpenSearch hosts advertising http. Defaults to
+                  localhost. Add port section if not listening on port 9200. Only one of these hosts
+                  needs to be available to fetch the remaining nodes in the cluster. It is
+                  recommended to set this to all the master nodes of the cluster. Use url format for
+                  SSL. For example, "https://yourhost:8888"
+    * `OS_NODES_WAN_ONLY`: Set to true to only use the values set in OS_HOSTS, for example if your
+                           OpenSearch cluster is in Docker. Defaults to false
+    * `OS_USERNAME` and `OS_PASSWORD`: OpenSearch basic authentication. Use when security plugin
+                                       is in place. By default no username or password is provided to OpenSearch.
+
+Example usage:
+
+```bash
+$ STORAGE_TYPE=opensearch OS_HOSTS=host1,host2 java -jar zipkin-dependencies.jar
+# To override the http port, add it to the host string
+$ STORAGE_TYPE=opensearch OS_HOSTS=host1:9201 java -jar zipkin-dependencies.jar
+```
+
 #### Custom certificates
 
 When using an https endpoint in `ES_HOSTS`, you can use the following standard properties to

diff --git a/docker/examples/docker-compose-opensearch.yml b/docker/examples/docker-compose-opensearch.yml
@@ -0,0 +1,40 @@
+#
+# Copyright The OpenZipkin Authors
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# This file uses the version 2 docker-compose file format, described here:
+# https://docs.docker.com/compose/compose-file/#version-2
+#
+# It extends the default configuration from docker-compose.yml to run the
+# zipkin-opensearch2 container instead of the zipkin-mysql container.
+
+version: '2.4'
+
+services:
+  storage:
+    image: ghcr.io/openzipkin/zipkin-opensearch2:${TAG:-latest}
+    container_name: opensearch
+    # Uncomment to expose the storage port for testing
+    # ports:
+    #   - 9200:9200
+
+  # Use OpenSearch instead of in-memory storage
+  zipkin:
+    extends:
+      file: docker-compose.yml
+      service: zipkin
+    environment:
+      - STORAGE_TYPE=elasticsearch
+      # Point the zipkin at the storage backend
+      - ES_HOSTS=elasticsearch:9200
+      # Uncomment to see requests to and from elasticsearch
+      # - ES_HTTP_LOGGING=BODY
+
+  dependencies:
+    extends:
+      file: docker-compose.yml
+      service: dependencies
+    environment:
+      - STORAGE_TYPE=opensearch
+      - OS_HOSTS=opensearch
diff --git a/main/pom.xml b/main/pom.xml
@@ -39,6 +39,12 @@
       <artifactId>zipkin-dependencies-elasticsearch</artifactId>
       <version>${project.version}</version>
     </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zipkin-dependencies-opensearch</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/main/src/main/java/zipkin2/dependencies/ZipkinDependenciesJob.java b/main/src/main/java/zipkin2/dependencies/ZipkinDependenciesJob.java
@@ -13,6 +13,7 @@
 import java.util.LinkedHashMap;
 import java.util.TimeZone;
 import zipkin2.dependencies.elasticsearch.ElasticsearchDependenciesJob;
+import zipkin2.dependencies.opensearch.OpensearchDependenciesJob;
 import zipkin2.dependencies.mysql.MySQLDependenciesJob;
 
 public final class ZipkinDependenciesJob {
@@ -69,9 +70,18 @@ public static void main(String[] args) throws UnsupportedEncodingException {
           .build()
           .run();
         break;
+      case "opensearch":
+          OpensearchDependenciesJob.builder()
+            .logInitializer(logInitializer)
+            .jars(jarPath)
+            .day(day)
+            .conf(sparkConf)
+            .build()
+            .run();
+          break;
       default:
         throw new UnsupportedOperationException("Unsupported STORAGE_TYPE: " + storageType + "\n"
-          + "Options are: cassandra3, mysql, elasticsearch");
+          + "Options are: cassandra3, mysql, elasticsearch, opensearch");
     }
   }
 

diff --git a/opensearch/pom.xml b/opensearch/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright The OpenZipkin Authors
+    SPDX-License-Identifier: Apache-2.0
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>io.zipkin.dependencies</groupId>
+    <artifactId>zipkin-dependencies-parent</artifactId>
+    <version>3.2.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>zipkin-dependencies-opensearch</artifactId>
+  <name>Zipkin Dependencies: OpenSearch</name>
+
+  <properties>
+    <main.basedir>${project.basedir}/..</main.basedir>
+    <okhttp.version>4.12.0</okhttp.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.opensearch.client</groupId>
+      <artifactId>opensearch-spark-30_${scala.binary.version}</artifactId>
+      <version>${opensearch-spark.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.zipkin.zipkin2</groupId>
+      <artifactId>zipkin-storage-elasticsearch</artifactId>
+      <version>${zipkin.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <version>${okhttp.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp-tls</artifactId>
+      <version>${okhttp.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Temporarily override ES verson of SLF4J -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+
+    <!-- integration tests -->
+    <dependency>
+      <groupId>com.linecorp.armeria</groupId>
+      <artifactId>armeria-junit5</artifactId>
+      <version>${armeria.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${testcontainers.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>