Supports OpenSearch V2 through ES_* environment variables

.github/workflows/test.yml

@@ -50,6 +50,9 @@ jobs:
           - name: zipkin-dependencies-elasticsearch-v8
             module: zipkin-dependencies-elasticsearch
             groups: docker,elasticsearch8
+          - name: zipkin-dependencies-opensearch-v2
+            module: zipkin-dependencies-opensearch
+            groups: docker,opensearch2
           - name: zipkin-dependencies-mysql
             module: zipkin-dependencies-mysql
             groups: docker

README.md

@@ -15,7 +15,7 @@ are supported, including Cassandra, MySQL and Elasticsearch.
 
 * `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 4.0
 * `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 10.11
-* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+; tested against last minor release of 7.x and 8.x
+* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+ or OpenSearch 2.x; tested against last minor release of Elasticsearch 7.x and 8.x, OpenSearch 2.x
 
 ## Quick-start
 
@@ -92,20 +92,20 @@ $ STORAGE_TYPE=mysql MYSQL_USER=root java -jar zipkin-dependencies.jar
 ```
 
 ### Elasticsearch Storage
-Elasticsearch is used when `STORAGE_TYPE=elasticsearch`. The schema is compatible with Zipkin's [Elasticsearch storage component](https://github.com/openzipkin/zipkin/tree/master/zipkin-storage/elasticsearch).
+Elasticsearch/OpenSearch is used when `STORAGE_TYPE=elasticsearch`. The schema is compatible with Zipkin's [Elasticsearch storage component](https://github.com/openzipkin/zipkin/tree/master/zipkin-storage/elasticsearch).
 
     * `ES_INDEX`: The index prefix to use when generating daily index names. Defaults to zipkin.
     * `ES_DATE_SEPARATOR`: The separator used when generating dates in index.
                            Defaults to '-' so the queried index look like zipkin-yyyy-DD-mm
                            Could for example be changed to '.' to give zipkin-yyyy.MM.dd
-    * `ES_HOSTS`: A comma separated list of elasticsearch hosts advertising http. Defaults to
+    * `ES_HOSTS`: A comma separated list of Elasticsearch / OpenSearch hosts advertising http. Defaults to
                   localhost. Add port section if not listening on port 9200. Only one of these hosts
                   needs to be available to fetch the remaining nodes in the cluster. It is
                   recommended to set this to all the master nodes of the cluster. Use url format for
                   SSL. For example, "https://yourhost:8888"
     * `ES_NODES_WAN_ONLY`: Set to true to only use the values set in ES_HOSTS, for example if your
-                           elasticsearch cluster is in Docker. Defaults to false
-    * `ES_USERNAME` and `ES_PASSWORD`: Elasticsearch basic authentication. Use when X-Pack security
+                           Elasticsearch / OpenSearch cluster is in Docker. Defaults to false
+    * `ES_USERNAME` and `ES_PASSWORD`: Elasticsearch / OpenSearch basic authentication. Use when X-Pack security
                                        (formerly Shield) is in place. By default no username or
                                        password is provided to elasticsearch.
 

docker/examples/docker-compose-opensearch.yml

@@ -0,0 +1,40 @@
+#
+# Copyright The OpenZipkin Authors
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# This file uses the version 2 docker-compose file format, described here:
+# https://docs.docker.com/compose/compose-file/#version-2
+#
+# It extends the default configuration from docker-compose.yml to run the
+# zipkin-opensearch2 container instead of the zipkin-mysql container.
+
+version: '2.4'
+
+services:
+  storage:
+    image: ghcr.io/openzipkin/zipkin-opensearch2:${TAG:-latest}
+    container_name: opensearch
+    # Uncomment to expose the storage port for testing
+    # ports:
+    #   - 9200:9200
+
+  # Use OpenSearch instead of in-memory storage
+  zipkin:
+    extends:
+      file: docker-compose.yml
+      service: zipkin
+    environment:
+      - STORAGE_TYPE=elasticsearch
+      # Point the zipkin at the storage backend
+      - ES_HOSTS=opensearch:9200
+      # Uncomment to see requests to and from elasticsearch
+      # - ES_HTTP_LOGGING=BODY
+
+  dependencies:
+    extends:
+      file: docker-compose.yml
+      service: dependencies
+    environment:
+      - STORAGE_TYPE=elasticsearch
+      - ES_HOSTS=opensearch

main/pom.xml

@@ -39,6 +39,12 @@
       <artifactId>zipkin-dependencies-elasticsearch</artifactId>
       <version>${project.version}</version>
     </dependency>
+
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>zipkin-dependencies-opensearch</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

main/src/main/java/zipkin2/dependencies/ZipkinDependenciesJob.java

@@ -13,6 +13,7 @@
 import java.util.LinkedHashMap;
 import java.util.TimeZone;
 import zipkin2.dependencies.elasticsearch.ElasticsearchDependenciesJob;
+import zipkin2.dependencies.opensearch.OpensearchDependenciesJob;
 import zipkin2.dependencies.mysql.MySQLDependenciesJob;
 
 public final class ZipkinDependenciesJob {
@@ -61,20 +62,30 @@ public static void main(String[] args) throws UnsupportedEncodingException {
           .run();
         break;
       case "elasticsearch":
-        ElasticsearchDependenciesJob.builder()
-          .logInitializer(logInitializer)
-          .jars(jarPath)
-          .day(day)
-          .conf(sparkConf)
-          .build()
-          .run();
+        if (ZipkinElasticsearchStorage.flavor().equalsIgnoreCase("elasticsearch")) {
+          ElasticsearchDependenciesJob.builder()
+            .logInitializer(logInitializer)
+            .jars(jarPath)
+            .day(day)
+            .conf(sparkConf)
+            .build()
+            .run();
+        } else { // "opensearch"
+          OpensearchDependenciesJob.builder()
+            .logInitializer(logInitializer)
+            .jars(jarPath)
+            .day(day)
+            .conf(sparkConf)
+            .build()
+            .run();
+        }
         break;
       default:
         throw new UnsupportedOperationException("Unsupported STORAGE_TYPE: " + storageType + "\n"
           + "Options are: cassandra3, mysql, elasticsearch");
     }
   }
-
+  
   static String[] pathToUberJar() throws UnsupportedEncodingException {
     URL jarFile = ZipkinDependenciesJob.class.getProtectionDomain().getCodeSource().getLocation();
     return new File(jarFile.getPath()).isDirectory() ? null

main/src/main/java/zipkin2/dependencies/ZipkinElasticsearchStorage.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright The OpenZipkin Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package zipkin2.dependencies;
+
+import java.io.IOException;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.net.http.HttpResponse.BodyHandlers;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+final class ZipkinElasticsearchStorage {
+  private static final Logger LOG = LoggerFactory.getLogger(ZipkinElasticsearchStorage.class);
+  private static final Pattern DISTRIBUTION = Pattern.compile("\"distribution\"\s*[:]\s*\"([^\"]+)\"");
+
+  static final String HOSTS = getEnv("ES_HOSTS", "127.0.0.1");
+  static final String USERNAME = getEnv("ES_USERNAME", null);
+  static final String PASSWORD = getEnv("ES_PASSWORD", null);
+
+  static TrustManager[] TRUST_ALL = new TrustManager [] {
+    new X509TrustManager() {
+      @Override
+      public X509Certificate[] getAcceptedIssuers() {
+        return null;
+      }
+
+      @Override
+      public void checkClientTrusted(X509Certificate[] certs, String authType) {
+      }
+
+      @Override
+      public void checkServerTrusted(X509Certificate[] certs, String authType) {
+      }
+    }
+  };
+
+  static String flavor() {
+    final HttpClient.Builder builder = HttpClient
+      .newBuilder()
+      .connectTimeout(Duration.ofSeconds(5));
+
+    if (USERNAME != null && PASSWORD != null) {
+      builder.authenticator(new Authenticator() {
+         @Override
+         protected PasswordAuthentication getPasswordAuthentication() {
+            return new PasswordAuthentication(USERNAME, PASSWORD.toCharArray());
+        }
+      });
+    }
+
+    try {
+      final SSLContext sslContext = SSLContext.getInstance("TLS");
+      sslContext.init(null, TRUST_ALL, new SecureRandom());
+
+      final HttpClient client = builder.sslContext(sslContext).build();
+      try {
+        for (String host: parseHosts(HOSTS)) {
+          final HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(host)).build();
+          try {
+            final HttpResponse<String> response = client.send(request, BodyHandlers.ofString());
+            final Matcher matcher = DISTRIBUTION.matcher(response.body());
+            if (matcher.find()) {
+              return matcher.group(1).toLowerCase();
+            }
+          } catch (InterruptedException | IOException ex) {
+            LOG.warn("Unable issue HTTP GET request to '" + host + "'", ex);
+          }
+        }
+      } finally {
+        if (client instanceof AutoCloseable) {
+          try {
+            // Since JDK-21, the HttpClient is AutoCloseable
+            ((AutoCloseable) client).close();
+          } catch (Exception ex) {
+            /* Ignore */
+          }
+        }
+      }
+    } catch (final NoSuchAlgorithmException | KeyManagementException ex) {
+      LOG.warn("Unable to configure HttpClient", ex);
+    }
+
+    return "elasticsearch";
+  }
+
+  private static String getEnv(String key, String defaultValue) {
+    String result = System.getenv(key);
+    return result != null && !result.isEmpty() ? result : defaultValue;
+  }
+
+  static String[] parseHosts(String hosts) {
+    final String[] hostParts = hosts.split(",", -1);
+
+    // Detect default scheme to use if not specified
+    String defaultScheme = "http";
+    for (int i = 0; i < hostParts.length; i++) {
+      String host = hostParts[i];
+      if (host.startsWith("https")) {
+        defaultScheme = "https";
+        break;
+      }
+    }
+
+    Collection<String> list = new ArrayList<>();
+    for (int i = 0; i < hostParts.length; i++) {
+      String host = hostParts[i];
+      URI httpUri = host.startsWith("http") ? URI.create(host) : URI.create(defaultScheme + "://" + host);
+
+      int port = httpUri.getPort();
+      if (port == -1) {
+        port = 9200; /* default Elasticsearch / OpenSearch port */
+      }
+
+      list.add(httpUri.getScheme() + "://" + httpUri.getHost() + ":" + port);
+    }
+
+    return list.toArray(new String[0]);
+  }
+
+  public static void main(String[] s) {
+      System.out.println(flavor());
+  }
+}

opensearch/pom.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright The OpenZipkin Authors
+    SPDX-License-Identifier: Apache-2.0
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>io.zipkin.dependencies</groupId>
+    <artifactId>zipkin-dependencies-parent</artifactId>
+    <version>3.2.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>zipkin-dependencies-opensearch</artifactId>
+  <name>Zipkin Dependencies: OpenSearch</name>
+
+  <properties>
+    <main.basedir>${project.basedir}/..</main.basedir>
+    <okhttp.version>4.12.0</okhttp.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.opensearch.client</groupId>
+      <artifactId>opensearch-spark-30_${scala.binary.version}</artifactId>
+      <version>${opensearch-spark.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.zipkin.zipkin2</groupId>
+      <artifactId>zipkin-storage-elasticsearch</artifactId>
+      <version>${zipkin.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>mockwebserver</artifactId>
+      <version>${okhttp.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp-tls</artifactId>
+      <version>${okhttp.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <!-- Temporarily override ES verson of SLF4J -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+
+    <!-- integration tests -->
+    <dependency>
+      <groupId>com.linecorp.armeria</groupId>
+      <artifactId>armeria-junit5</artifactId>
+      <version>${armeria.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${testcontainers.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>