Add/Update Metrics TCK

tck/logs/README.adoc

@@ -81,6 +81,14 @@ To test that the OpenTelemetry instance is set properly at runtime initializatio
 </suite>
 ----
 
+== Logging File Configuration 
+OpenTelemetry logs are sent to stdout in the tests. Ensure logs written to stdout are captured in a file and set the system property `log.file.path` to the file containing the log output when running the logs TCK. For example:
+
+[source, xml]
+----
+log.file.path=console.log
+----
+
 == Configuration in Apache Maven pom.xml
 If you use Apache Maven then the tests are run via the `maven-surefire-plugin`
 

tck/logs/pom.xml

@@ -46,36 +46,6 @@
             <groupId>io.opentelemetry</groupId>
             <artifactId>opentelemetry-sdk</artifactId>
         </dependency>
-        <dependency>
-            <groupId>io.opentelemetry</groupId>
-            <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.opentelemetry.instrumentation</groupId>
-            <artifactId>opentelemetry-instrumentation-annotations</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>io.opentelemetry.semconv</groupId>
-            <artifactId>opentelemetry-semconv</artifactId>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/io.opentelemetry.instrumentation/opentelemetry-logback-appender-1.0 -->
-        <dependency>
-            <groupId>io.opentelemetry.instrumentation</groupId>
-            <artifactId>opentelemetry-logback-appender-1.0</artifactId>
-            <version>2.2.0-alpha</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <version>1.5.6</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>jul-to-slf4j</artifactId>
-            <version>2.0.13</version>
-        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>

tck/logs/src/main/java/org/eclipse/microprofile/telemetry/logs/tck/application/LogAppenderTest.java → tck/logs/src/main/java/org/eclipse/microprofile/telemetry/logs/tck/application/JulTest.java

@@ -20,6 +20,10 @@
 
 package org.eclipse.microprofile.telemetry.logs.tck.application;
 
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.jboss.arquillian.container.test.api.Deployment;
@@ -28,16 +32,17 @@
 import org.jboss.shrinkwrap.api.asset.EmptyAsset;
 import org.jboss.shrinkwrap.api.asset.StringAsset;
 import org.jboss.shrinkwrap.api.spec.WebArchive;
+import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import io.opentelemetry.api.OpenTelemetry;
 import jakarta.inject.Inject;
 
-public class LogAppenderTest extends Arquillian {
+public class JulTest extends Arquillian {
+
     @Deployment
     public static WebArchive createDeployment() {
         return ShrinkWrap.create(WebArchive.class)
-                .addAsLibrary(TestLibraries.AWAITILITY_LIB)
                 .addAsResource(new StringAsset(
                         "otel.sdk.disabled=false\notel.metrics.exporter=none\notel.traces.exporter=none\notel.logs.exporter=logging\notel.service.name=openliberty"),
                         "META-INF/microprofile-config.properties")
@@ -49,13 +54,46 @@ public static WebArchive createDeployment() {
 
     private static final Logger julLogger = Logger.getLogger("jul-logger");
 
+    private static final String logFilePath = System.getProperty("log.file.path");
+
+    private static final String JUL_INFO_MESSAGE = "a very distinguishable info message";
+    private static final String JUL_WARN_MESSAGE = "a very distinguishable warning message";
+
     @Test
-    void julTest() throws InterruptedException {
+    void julInfoTest() throws IOException {
+        julLogger.log(Level.INFO, JUL_INFO_MESSAGE);
+        try {
+            Assert.assertTrue(checkMessage(".*INFO.*" + JUL_INFO_MESSAGE + ".*scopeInfo:.*"));
+        } catch (IOException e) {
+        }
+    }
+
+    @Test
+    void julWarnTest() throws IOException {
+        julLogger.log(Level.WARNING, JUL_WARN_MESSAGE);
+        try {
+            Assert.assertTrue(checkMessage(".*WARN.*" + JUL_WARN_MESSAGE + ".*scopeInfo:.*"));
+        } catch (IOException e) {
+        }
+    }
 
-        julLogger.info("A JUL log message");
-        julLogger.info("A JUL log message");
-        julLogger.info("A JUL log message");
-        julLogger.info("A JUL log message");
-        Thread.sleep(10000);
+    public boolean checkMessage(String logMessage) throws IOException {
+        try {
+            try {
+                Thread.sleep(5000);
+                BufferedReader reader = new BufferedReader(new FileReader(logFilePath));
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    if (line.matches(logMessage)) {
+                        return true;
+                    }
+                }
+                return false;
+            } catch (IOException e) {
+                return false;
+            }
+        } catch (InterruptedException e) {
+            return false;
+        }
     }
 }

tck/logs/src/main/java/org/eclipse/microprofile/telemetry/logs/tck/config/ServerInstanceTest.java

@@ -40,7 +40,6 @@ public class ServerInstanceTest extends Arquillian {
     @Deployment
     public static WebArchive createDeployment() {
         return ShrinkWrap.create(WebArchive.class)
-                .addAsLibrary(TestLibraries.AWAITILITY_LIB)
                 .addAsResource(new StringAsset(
                         "otel.service.name=" + serviceNameUnused),
                         "META-INF/microprofile-config.properties")

tck/metrics/README.adoc

@@ -49,7 +49,32 @@ To enable the tests in your project you need to add the following dependency to
 
 ----
 
-If you want to run the metrics tests, you can specify all tests in the `tck-suite.xml`. E.g. 
+== Running the tests
+
+The JVM metrics tests require runtime configuration to enable metric reading at a runtime level. The metrics must be sent to stdout in the tests. Ensure logs written to stdout are captured in a file and set the system property `log.file.path` to the file containing the log output when running the logs TCK. Configure the runtime with the following as system properties / environment variables:
+
+ * `otel.sdk.disabled=false`/`OTEL_SDK_DISABLED=FALSE`
+ * `otel.metrics.exporter=logging`/`OTEL_METRICS_EXPORTER=LOGGING`
+ * `otel.traces.exporter=none`/`OTEL_TRACES_EXPORTER=none`
+ * `otel.logs.exporter=none`/`OTEL_LOGS_EXPORTER=none`
+ * `otel.metric.export.interval=3000`/`OTEL_METRIC_EXPORT_INTERVAL=3000`
+
+To run the JVM metrics tests, include the following content in the `tck-suite.xml` in your project:
+
+[source, xml]
+----
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
+
+<suite name="microprofile-telemetry-metrics-TCK" verbose="2" configfailurepolicy="continue" >
+    <test name="telemetry-metrics-tests" verbose="10">
+        <packages>
+            <package name="org.eclipse.microprofile.telemetry.metrics.tck.jvm.*" />
+        </packages>
+    </test>
+</suite>
+----
+
+The remaining metrics tests must use an OpenTelemetry SDK instance that is configured by the configuration properties set in the application. Ensure that `otel.sdk.disabled` and `otel.metrics.exporter` are NOT set by the runtime. To run the application metrics tests, include the following content in the `tck-suite.xml` in your project:
 
 [source, xml]
 ----
@@ -58,7 +83,7 @@ If you want to run the metrics tests, you can specify all tests in the `tck-suit
 <suite name="microprofile-telemetry-metrics-TCK" verbose="2" configfailurepolicy="continue" >
     <test name="telemetry-metrics-tests" verbose="10">
         <packages>
-            <package name="org.eclipse.microprofile.telemetry.metrics.tck.*" />
+            <package name="org.eclipse.microprofile.telemetry.metrics.tck.application.*" />
         </packages>
     </test>
 </suite>

tck/metrics/src/main/java/org/eclipse/microprofile/telemetry/metrics/tck/application/BasicHttpClient.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2022-2023 Contributors to the Eclipse Foundation
+ *
+ *  See the NOTICE file(s) distributed with this work for additional
+ *  information regarding copyright ownership.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  You may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.eclipse.microprofile.telemetry.metrics.tck.application;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.jboss.arquillian.test.api.ArquillianResource;
+
+/**
+ * A really basic client for doing Http requests
+ * <p>
+ * For use when we don't want to use JAX-RS client or something else which has integration with telemetry
+ */
+public class BasicHttpClient {
+
+    private URI baseUri;
+
+    /**
+     * @param baseUrl
+     *            The base URL. Any path requested through this client will be appended to this URL. This should usually
+     *            be a URL injected using {@link ArquillianResource}
+     */
+    public BasicHttpClient(URL baseUrl) {
+        try {
+            baseUri = baseUrl.toURI();
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Makes a GET request to a path and returns the response code
+     *
+     * @param path
+     *            the path to request, relative to the baseUrl
+     * @return the response code
+     */
+    public int get(String path) {
+        if (path.startsWith("/")) {
+            path = path.substring(1);
+        }
+        try {
+            URL spanUrl = baseUri.resolve(path).toURL();
+            HttpURLConnection connection = (HttpURLConnection) spanUrl.openConnection();
+            try {
+                return connection.getResponseCode();
+            } catch (Exception e) {
+                throw new RuntimeException("Exception retrieving " + spanUrl, e);
+            } finally {
+                connection.disconnect();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Exception retrieving path " + path, e);
+        }
+    }
+
+    /**
+     * Makes a GET request to a path and returns the response code
+     *
+     * @param path
+     *            the path to request, relative to the baseUrl
+     * @return the response message
+     */
+    public String getResponseMessage(String path) {
+        if (path.startsWith("/")) {
+            path = path.substring(1);
+        }
+        try {
+            URL spanUrl = baseUri.resolve(path).toURL();
+            HttpURLConnection connection = (HttpURLConnection) spanUrl.openConnection();
+            try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+                String inputLine;
+                StringBuilder response = new StringBuilder();
+
+                while ((inputLine = in.readLine()) != null) {
+                    response.append(inputLine);
+                }
+                return response.toString();
+            } catch (Exception e) {
+                throw new RuntimeException("Exception retrieving " + spanUrl, e);
+            } finally {
+                connection.disconnect();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Exception retrieving path " + path, e);
+        }
+    }
+
+}

tck/metrics/src/main/java/org/eclipse/microprofile/telemetry/metrics/tck/ConfigAsset.java → tck/metrics/src/main/java/org/eclipse/microprofile/telemetry/metrics/tck/application/ConfigAsset.java

@@ -17,7 +17,7 @@
  * limitations under the License.
  *
  */
-package org.eclipse.microprofile.telemetry.metrics.tck;
+package org.eclipse.microprofile.telemetry.metrics.tck.application;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;

tck/logs/src/main/java/org/eclipse/microprofile/telemetry/logs/tck/application/TestLibraries.java → tck/metrics/src/main/java/org/eclipse/microprofile/telemetry/metrics/tck/application/TestLibraries.java

@@ -17,7 +17,7 @@
  * limitations under the License.
  *
  */
-package org.eclipse.microprofile.telemetry.logs.tck.application;
+package org.eclipse.microprofile.telemetry.metrics.tck.application;
 
 import org.jboss.shrinkwrap.api.ShrinkWrap;
 import org.jboss.shrinkwrap.api.spec.JavaArchive;