Improvements to the Grafana LGTM dashboards

extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/ContainerConstants.java

@@ -15,4 +15,12 @@ public final class ContainerConstants {
 
     public static final String OTEL_GRPC_PROTOCOL = "grpc";
     public static final String OTEL_HTTP_PROTOCOL = "http/protobuf";
+
+    // Overrides
+
+    public static final int SCRAPING_INTERVAL = 10;
+    public static final String OTEL_METRIC_EXPORT_INTERVAL = "10s";
+    public static final String OTEL_BSP_SCHEDULE_DELAY = "10s";
+    public static final String OTEL_BLRP_SCHEDULE_DELAY = "5s";
+
 }

extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfigUtil.java

@@ -2,7 +2,10 @@
 
 import java.lang.reflect.Method;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 public class ContainerConfigUtil {
     /**
@@ -16,11 +19,7 @@ public static boolean isEqual(ContainerConfig cc1, ContainerConfig cc2) {
             return false;
         }
 
-        Class<?> i = Arrays.stream(c1.getInterfaces())
-                .filter(ContainerConfig.class::isAssignableFrom)
-                .findFirst()
-                .orElseThrow(() -> new IllegalArgumentException("Missing ContainerConfig based interface"));
-        Method[] methods = i.getMethods(); // should get all config methods
+        Method[] methods = getMethods(c1);
         for (Method m : methods) {
             Object v1 = invoke(m, cc1);
             Object v2 = invoke(m, cc2);
@@ -31,6 +30,38 @@ public static boolean isEqual(ContainerConfig cc1, ContainerConfig cc2) {
         return true;
     }
 
+    /**
+     * Get all properties to override from container config instance.
+     *
+     * @param config the container config
+     * @return map of properties to override
+     */
+    public static Map<String, Object> propertiesToOverride(ContainerConfig config) {
+        Map<String, Object> map = new HashMap<>();
+        for (Method m : getMethods(config.getClass())) {
+            OverrideProperty override = m.getAnnotation(OverrideProperty.class);
+            if (override != null) {
+                String key = override.value();
+                Object value = invoke(m, config);
+                if (value instanceof Optional<?>) {
+                    Optional<?> optional = (Optional<?>) value;
+                    optional.ifPresent(o -> map.put(key, o));
+                } else if (value != null) {
+                    map.put(key, value);
+                }
+            }
+        }
+        return map;
+    }
+
+    private static Method[] getMethods(Class<?> c1) {
+        Class<?> i = Arrays.stream(c1.getInterfaces())
+                .filter(ContainerConfig.class::isAssignableFrom)
+                .findFirst()
+                .orElseThrow(() -> new IllegalArgumentException("Missing ContainerConfig based interface"));
+        return i.getMethods();
+    }
+
     private static Object invoke(Method m, Object target) {
         try {
             return m.invoke(target);

extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/LgtmConfig.java

@@ -37,4 +37,36 @@ public interface LgtmConfig extends GrafanaConfig {
      */
     @WithDefault(ContainerConstants.OTEL_HTTP_PROTOCOL)
     String otlpProtocol();
+
+    /**
+     * The (Prometheus) scraping interval, in seconds.
+     */
+    @WithDefault(ContainerConstants.SCRAPING_INTERVAL + "")
+    int scrapingInterval();
+
+    /**
+     * Do we force scraping.
+     */
+    Optional<Boolean> forceScraping();
+
+    /**
+     * A way to override `quarkus.otel.metric.export.interval` property's default value.
+     */
+    @OverrideProperty("quarkus.otel.metric.export.interval")
+    @WithDefault(ContainerConstants.OTEL_METRIC_EXPORT_INTERVAL)
+    String otelMetricExportInterval();
+
+    /**
+     * A way to override `quarkus.otel.bsp.schedule.delay` property's default value.
+     */
+    @OverrideProperty("quarkus.otel.bsp.schedule.delay")
+    @WithDefault(ContainerConstants.OTEL_BSP_SCHEDULE_DELAY)
+    String otelBspScheduleDelay();
+
+    /**
+     * A way to override `quarkus.otel.metric.export.interval` property's default value.
+     */
+    @OverrideProperty("quarkus.otel.blrp.schedule.delay")
+    @WithDefault(ContainerConstants.OTEL_BLRP_SCHEDULE_DELAY)
+    String otelBlrpScheduleDelay();
 }

extensions/observability-devservices/common/src/main/java/io/quarkus/observability/common/config/OverrideProperty.java

@@ -0,0 +1,21 @@
+package io.quarkus.observability.common.config;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Override the property in the value,
+ * with the value of the annotated method's return.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.METHOD })
+public @interface OverrideProperty {
+    /**
+     * The property key to override.
+     *
+     * @return the property key
+     */
+    String value();
+}

extensions/observability-devservices/deployment/src/main/java/io/quarkus/observability/deployment/ObservabilityDevServiceProcessor.java

@@ -28,6 +28,7 @@
 import io.quarkus.deployment.builditem.DockerStatusBuildItem;
 import io.quarkus.deployment.builditem.FeatureBuildItem;
 import io.quarkus.deployment.builditem.LaunchModeBuildItem;
+import io.quarkus.deployment.builditem.RunTimeConfigurationDefaultBuildItem;
 import io.quarkus.deployment.console.ConsoleInstalledBuildItem;
 import io.quarkus.deployment.console.StartupLogCompressor;
 import io.quarkus.deployment.dev.devservices.DevServicesConfig;
@@ -84,6 +85,7 @@ public void startContainers(LaunchModeBuildItem launchMode,
             LoggingSetupBuildItem loggingSetupBuildItem,
             DevServicesConfig devServicesConfig,
             BuildProducer<DevServicesResultBuildItem> services,
+            BuildProducer<RunTimeConfigurationDefaultBuildItem> properties,
             Capabilities capabilities,
             Optional<MetricsCapabilityBuildItem> metricsConfiguration,
             BuildProducer<ObservabilityDevServicesConfigBuildItem> configBuildProducer) {
@@ -118,6 +120,8 @@ public void startContainers(LaunchModeBuildItem launchMode,
             ContainerConfig currentDevServicesConfiguration = dev.config(
                     configuration,
                     new ExtensionsCatalog(
+                            QuarkusClassLoader::isResourcePresentAtRuntime,
+                            QuarkusClassLoader::isClassPresentAtRuntime,
                             capabilities.isPresent(Capability.OPENTELEMETRY_TRACER),
                             hasMicrometerOtlp(metricsConfiguration)));
 
@@ -140,6 +144,13 @@ public void startContainers(LaunchModeBuildItem launchMode,
             devServices.remove(devId); // clean-up
             capturedDevServicesConfigurations.put(devId, currentDevServicesConfiguration);
 
+            // override some OTel, etc defaults - rates, intervals, delays, ...
+            Map<String, Object> propertiesToOverride = ContainerConfigUtil
+                    .propertiesToOverride(currentDevServicesConfiguration);
+            propertiesToOverride
+                    .forEach((k, v) -> properties.produce(new RunTimeConfigurationDefaultBuildItem(k, v.toString())));
+            log.infof("Dev Service %s properties override: %s", devId, propertiesToOverride);
+
             StartupLogCompressor compressor = new StartupLogCompressor(
                     (launchMode.isTest() ? "(test) " : "") + devId + " Dev Services Starting:",
                     consoleInstalledBuildItem,

extensions/observability-devservices/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java

@@ -15,11 +15,12 @@
 import io.quarkus.observability.common.ContainerConstants;
 import io.quarkus.observability.common.config.AbstractGrafanaConfig;
 import io.quarkus.observability.common.config.LgtmConfig;
+import io.quarkus.runtime.LaunchMode;
 
 public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
     protected static final String LGTM_NETWORK_ALIAS = "ltgm.testcontainer.docker";
 
-    protected static final String PROMETHEUS_CONFIG = """
+    protected static final String PROMETHEUS_CONFIG_DEFAULT = """
             ---
             otlp:
               # Recommended attributes to be promoted to labels.
@@ -47,12 +48,15 @@ public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
                 # A 10min time window is enough because it can easily absorb retries and network delays.
                 out_of_order_time_window: 10m
             global:
-              scrape_interval: 5s
+              scrape_interval: %s
               evaluation_interval: 5s
+            """;
+
+    protected static final String PROMETHEUS_CONFIG_SCRAPE = """
             scrape_configs:
               - job_name: '%s'
                 metrics_path: '%s%s'
-                scrape_interval: 5s
+                scrape_interval: %s
                 static_configs:
                   - targets: ['%s:%d']
             """;
@@ -73,12 +77,16 @@ public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
                   foldersFromFilesStructure: false
             """;
 
-    public LgtmContainer() {
-        this(new LgtmConfigImpl());
+    private final boolean scrapingRequired;
+
+    public LgtmContainer(boolean scrapingRequired) {
+        this(new LgtmConfigImpl(), scrapingRequired);
     }
 
-    public LgtmContainer(LgtmConfig config) {
+    public LgtmContainer(LgtmConfig config, boolean scrapingRequired) {
         super(config);
+        // do we require scraping
+        this.scrapingRequired = scrapingRequired;
         // always expose both -- since the LGTM image already does that as well
         addExposedPorts(ContainerConstants.OTEL_GRPC_EXPORTER_PORT, ContainerConstants.OTEL_HTTP_EXPORTER_PORT);
 
@@ -140,12 +148,22 @@ public static int getPrivateOtlpPort(String otlpProtocol) {
     }
 
     private String getPrometheusConfig() {
-        Config runtimeConfig = ConfigProvider.getConfig();
-        String rootPath = runtimeConfig.getOptionalValue("quarkus.management.root-path", String.class).orElse("/q");
-        String metricsPath = runtimeConfig.getOptionalValue("quarkus.management.metrics.path", String.class).orElse("/metrics");
-        int httpPort = runtimeConfig.getOptionalValue("quarkus.http.port", Integer.class).orElse(8080); // when not set use default
-
-        return String.format(PROMETHEUS_CONFIG, config.serviceName(), rootPath, metricsPath, "host.docker.internal", httpPort);
+        String scraping = config.scrapingInterval() + "s";
+        String prometheusConfig = String.format(PROMETHEUS_CONFIG_DEFAULT, scraping);
+        if (config.forceScraping().orElse(scrapingRequired)) {
+            boolean isTest = LaunchMode.current() == LaunchMode.TEST;
+            Config runtimeConfig = ConfigProvider.getConfig();
+            String rootPath = runtimeConfig.getOptionalValue("quarkus.management.root-path", String.class).orElse("/q");
+            String metricsPath = runtimeConfig.getOptionalValue("quarkus.management.metrics.path", String.class)
+                    .orElse("/metrics");
+            String httpPortKey = isTest ? "quarkus.http.test-port" : "quarkus.http.port";
+            Optional<Integer> optionalValue = runtimeConfig.getOptionalValue(httpPortKey, Integer.class);
+            int httpPort = optionalValue.orElse(isTest ? 8081 : 8080); // when not set use default
+
+            prometheusConfig += String.format(PROMETHEUS_CONFIG_SCRAPE, config.serviceName(), rootPath, metricsPath, scraping,
+                    "host.docker.internal", httpPort);
+        }
+        return prometheusConfig;
     }
 
     protected static class LgtmConfigImpl extends AbstractGrafanaConfig implements LgtmConfig {
@@ -166,6 +184,31 @@ public Optional<Set<String>> networkAliases() {
         public String otlpProtocol() {
             return ContainerConstants.OTEL_HTTP_PROTOCOL;
         }
+
+        @Override
+        public int scrapingInterval() {
+            return ContainerConstants.SCRAPING_INTERVAL;
+        }
+
+        @Override
+        public Optional<Boolean> forceScraping() {
+            return Optional.empty();
+        }
+
+        @Override
+        public String otelMetricExportInterval() {
+            return ContainerConstants.OTEL_METRIC_EXPORT_INTERVAL;
+        }
+
+        @Override
+        public String otelBspScheduleDelay() {
+            return ContainerConstants.OTEL_BSP_SCHEDULE_DELAY;
+        }
+
+        @Override
+        public String otelBlrpScheduleDelay() {
+            return ContainerConstants.OTEL_BLRP_SCHEDULE_DELAY;
+        }
     }
 
     protected static class LgtmLoggingFilter implements Predicate<OutputFrame> {

extensions/observability-devservices/testlibs/devresource-common/src/main/java/io/quarkus/observability/devresource/ExtensionsCatalog.java

@@ -1,8 +1,13 @@
 package io.quarkus.observability.devresource;
 
+import java.util.function.Function;
+
 /**
  * Relevant Observability extensions present.
  */
-public record ExtensionsCatalog(boolean hasOpenTelemetry,
+public record ExtensionsCatalog(
+        Function<String, Boolean> resourceChecker,
+        Function<String, Boolean> classChecker,
+        boolean hasOpenTelemetry,
         boolean hasMicrometerOtlp) {
 }

extensions/observability-devservices/testlibs/devresource-lgtm/src/main/java/io/quarkus/observability/devresource/lgtm/LgtmResource.java

@@ -2,6 +2,10 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+import org.jboss.logging.Logger;
 
 import io.quarkus.observability.common.ContainerConstants;
 import io.quarkus.observability.common.config.LgtmConfig;
@@ -14,6 +18,22 @@
 
 public class LgtmResource extends ContainerResource<LgtmContainer, LgtmConfig> {
 
+    private static final Logger log = Logger.getLogger(LgtmResource.class.getName());
+
+    protected static final Set<String> SCRAPING_REGISTRIES = Set.of(
+            "io.micrometer.prometheus.PrometheusMeterRegistry");
+
+    protected static final Function<String, Boolean> TCCL_FN = s -> {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        try {
+            cl.loadClass(s);
+            return true;
+        } catch (Exception e) {
+            // any exception
+            return false;
+        }
+    };
+
     private ExtensionsCatalog catalog;
     private LgtmConfig config;
 
@@ -32,7 +52,26 @@ public LgtmConfig config(ModulesConfiguration configuration, ExtensionsCatalog c
 
     @Override
     public Container<LgtmConfig> container(LgtmConfig config, ModulesConfiguration root) {
-        return set(new LgtmContainer(config));
+        return set(new LgtmContainer(config, isScrapingRequired(catalog.classChecker())));
+    }
+
+    private boolean isScrapingRequired(Function<String, Boolean> checker) {
+        boolean result = false;
+        String foundRegistry = null;
+        for (String clazz : SCRAPING_REGISTRIES) {
+            if (checker.apply(clazz)) {
+                foundRegistry = clazz;
+                result = true;
+                break;
+            }
+        }
+
+        if (result && (catalog != null && catalog.hasMicrometerOtlp())) {
+            log.warnf("Multiple Micrometer registries found - OTLP and %s, no Prometheus scrapping required.", foundRegistry);
+            return false;
+        }
+
+        return result;
     }
 
     private int getPrivateOtlpPort() {
@@ -86,7 +125,7 @@ public Map<String, String> config(int privatePort, String host, int publicPort)
 
     @Override
     protected LgtmContainer defaultContainer() {
-        return new LgtmContainer();
+        return new LgtmContainer(isScrapingRequired(TCCL_FN)); // best we can do?
     }
 
     @Override