Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improvements to the Grafana LGTM dashboards #45747

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Improvements to the Grafana LGTM dashboards #45747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ...-devservices/common/src/main/java/io/quarkus/observability/common/ContainerConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,12 @@ public final class ContainerConstants {

public static final String OTEL_GRPC_PROTOCOL = "grpc";
public static final String OTEL_HTTP_PROTOCOL = "http/protobuf";

// Overrides

public static final int SCRAPING_INTERVAL = 10;
public static final String OTEL_METRIC_EXPORT_INTERVAL = "10s";
public static final String OTEL_BSP_SCHEDULE_DELAY = "3s";
public static final String OTEL_BLRP_SCHEDULE_DELAY = "1s";

}
41 changes: 36 additions & 5 deletions ...ices/common/src/main/java/io/quarkus/observability/common/config/ContainerConfigUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

public class ContainerConfigUtil {
/**
Expand All @@ -16,11 +19,7 @@ public static boolean isEqual(ContainerConfig cc1, ContainerConfig cc2) {
return false;
}

Class<?> i = Arrays.stream(c1.getInterfaces())
.filter(ContainerConfig.class::isAssignableFrom)
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Missing ContainerConfig based interface"));
Method[] methods = i.getMethods(); // should get all config methods
Method[] methods = getMethods(c1);
for (Method m : methods) {
Object v1 = invoke(m, cc1);
Object v2 = invoke(m, cc2);
Expand All @@ -31,6 +30,38 @@ public static boolean isEqual(ContainerConfig cc1, ContainerConfig cc2) {
return true;
}

/**
* Get all properties to override from container config instance.
*
* @param config the container config
* @return map of properties to override
*/
public static Map<String, Object> propertiesToOverride(ContainerConfig config) {
Map<String, Object> map = new HashMap<>();
for (Method m : getMethods(config.getClass())) {
OverrideProperty override = m.getAnnotation(OverrideProperty.class);
if (override != null) {
String key = override.value();
Object value = invoke(m, config);
if (value instanceof Optional<?>) {
Optional<?> optional = (Optional<?>) value;
optional.ifPresent(o -> map.put(key, o));
} else if (value != null) {
map.put(key, value);
}
}
}
return map;
}

private static Method[] getMethods(Class<?> c1) {
Class<?> i = Arrays.stream(c1.getInterfaces())
.filter(ContainerConfig.class::isAssignableFrom)
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Missing ContainerConfig based interface"));
return i.getMethods();
}

private static Object invoke(Method m, Object target) {
try {
return m.invoke(target);
Expand Down
36 changes: 36 additions & 0 deletions ...y-devservices/common/src/main/java/io/quarkus/observability/common/config/LgtmConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.Set;

import io.quarkus.observability.common.ContainerConstants;
import io.quarkus.runtime.annotations.ConfigDocIgnore;
import io.quarkus.runtime.annotations.ConfigGroup;
import io.smallrye.config.WithDefault;

Expand Down Expand Up @@ -37,4 +38,39 @@ public interface LgtmConfig extends GrafanaConfig {
*/
@WithDefault(ContainerConstants.OTEL_HTTP_PROTOCOL)
String otlpProtocol();

/**
* The (Prometheus) scraping interval, in seconds.
*/
@WithDefault(ContainerConstants.SCRAPING_INTERVAL + "")
int scrapingInterval();

/**
* Do we force scraping.
*/
Optional<Boolean> forceScraping();

/**
* A way to override `quarkus.otel.metric.export.interval` property's default value.
*/
@OverrideProperty("quarkus.otel.metric.export.interval")
@WithDefault(ContainerConstants.OTEL_METRIC_EXPORT_INTERVAL)
@ConfigDocIgnore
String otelMetricExportInterval();

/**
* A way to override `quarkus.otel.bsp.schedule.delay` property's default value.
*/
@OverrideProperty("quarkus.otel.bsp.schedule.delay")
@WithDefault(ContainerConstants.OTEL_BSP_SCHEDULE_DELAY)
@ConfigDocIgnore
String otelBspScheduleDelay();

/**
* A way to override `quarkus.otel.metric.export.interval` property's default value.
*/
@OverrideProperty("quarkus.otel.blrp.schedule.delay")
@WithDefault(ContainerConstants.OTEL_BLRP_SCHEDULE_DELAY)
@ConfigDocIgnore
String otelBlrpScheduleDelay();
}
21 changes: 21 additions & 0 deletions ...ervices/common/src/main/java/io/quarkus/observability/common/config/OverrideProperty.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package io.quarkus.observability.common.config;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* Override the property in the value,
* with the value of the annotated method's return.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.METHOD })
public @interface OverrideProperty {
/**
* The property key to override.
*
* @return the property key
*/
String value();
}
11 changes: 11 additions & 0 deletions ...t/src/main/java/io/quarkus/observability/deployment/ObservabilityDevServiceProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import io.quarkus.deployment.builditem.DockerStatusBuildItem;
import io.quarkus.deployment.builditem.FeatureBuildItem;
import io.quarkus.deployment.builditem.LaunchModeBuildItem;
import io.quarkus.deployment.builditem.RunTimeConfigurationDefaultBuildItem;
import io.quarkus.deployment.console.ConsoleInstalledBuildItem;
import io.quarkus.deployment.console.StartupLogCompressor;
import io.quarkus.deployment.dev.devservices.DevServicesConfig;
Expand Down Expand Up @@ -84,6 +85,7 @@ public void startContainers(LaunchModeBuildItem launchMode,
LoggingSetupBuildItem loggingSetupBuildItem,
DevServicesConfig devServicesConfig,
BuildProducer<DevServicesResultBuildItem> services,
BuildProducer<RunTimeConfigurationDefaultBuildItem> properties,
Capabilities capabilities,
Optional<MetricsCapabilityBuildItem> metricsConfiguration,
BuildProducer<ObservabilityDevServicesConfigBuildItem> configBuildProducer) {
Expand Down Expand Up @@ -118,6 +120,8 @@ public void startContainers(LaunchModeBuildItem launchMode,
ContainerConfig currentDevServicesConfiguration = dev.config(
configuration,
new ExtensionsCatalog(
QuarkusClassLoader::isResourcePresentAtRuntime,
QuarkusClassLoader::isClassPresentAtRuntime,
capabilities.isPresent(Capability.OPENTELEMETRY_TRACER),
hasMicrometerOtlp(metricsConfiguration)));

Expand All @@ -140,6 +144,13 @@ public void startContainers(LaunchModeBuildItem launchMode,
devServices.remove(devId); // clean-up
capturedDevServicesConfigurations.put(devId, currentDevServicesConfiguration);

// override some OTel, etc defaults - rates, intervals, delays, ...
Map<String, Object> propertiesToOverride = ContainerConfigUtil
.propertiesToOverride(currentDevServicesConfiguration);
propertiesToOverride
.forEach((k, v) -> properties.produce(new RunTimeConfigurationDefaultBuildItem(k, v.toString())));
log.infof("Dev Service %s properties override: %s", devId, propertiesToOverride);

StartupLogCompressor compressor = new StartupLogCompressor(
(launchMode.isTest() ? "(test) " : "") + devId + " Dev Services Starting:",
consoleInstalledBuildItem,
Expand Down
67 changes: 55 additions & 12 deletions ...s/testcontainers/src/main/java/io/quarkus/observability/testcontainers/LgtmContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@
import io.quarkus.observability.common.ContainerConstants;
import io.quarkus.observability.common.config.AbstractGrafanaConfig;
import io.quarkus.observability.common.config.LgtmConfig;
import io.quarkus.runtime.LaunchMode;

public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
protected static final String LGTM_NETWORK_ALIAS = "ltgm.testcontainer.docker";

protected static final String PROMETHEUS_CONFIG = """
protected static final String PROMETHEUS_CONFIG_DEFAULT = """
---
otlp:
# Recommended attributes to be promoted to labels.
Expand Down Expand Up @@ -47,12 +48,15 @@ public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
# A 10min time window is enough because it can easily absorb retries and network delays.
out_of_order_time_window: 10m
global:
scrape_interval: 5s
scrape_interval: %s
evaluation_interval: 5s
""";

protected static final String PROMETHEUS_CONFIG_SCRAPE = """
scrape_configs:
- job_name: '%s'
metrics_path: '%s%s'
scrape_interval: 5s
scrape_interval: %s
static_configs:
- targets: ['%s:%d']
""";
Expand All @@ -73,12 +77,16 @@ public class LgtmContainer extends GrafanaContainer<LgtmContainer, LgtmConfig> {
foldersFromFilesStructure: false
""";

public LgtmContainer() {
this(new LgtmConfigImpl());
private final boolean scrapingRequired;

public LgtmContainer(boolean scrapingRequired) {
this(new LgtmConfigImpl(), scrapingRequired);
}

public LgtmContainer(LgtmConfig config) {
public LgtmContainer(LgtmConfig config, boolean scrapingRequired) {
super(config);
// do we require scraping
this.scrapingRequired = scrapingRequired;
// always expose both -- since the LGTM image already does that as well
addExposedPorts(ContainerConstants.OTEL_GRPC_EXPORTER_PORT, ContainerConstants.OTEL_HTTP_EXPORTER_PORT);

Expand Down Expand Up @@ -140,12 +148,22 @@ public static int getPrivateOtlpPort(String otlpProtocol) {
}

private String getPrometheusConfig() {
Config runtimeConfig = ConfigProvider.getConfig();
String rootPath = runtimeConfig.getOptionalValue("quarkus.management.root-path", String.class).orElse("/q");
String metricsPath = runtimeConfig.getOptionalValue("quarkus.management.metrics.path", String.class).orElse("/metrics");
int httpPort = runtimeConfig.getOptionalValue("quarkus.http.port", Integer.class).orElse(8080); // when not set use default

return String.format(PROMETHEUS_CONFIG, config.serviceName(), rootPath, metricsPath, "host.docker.internal", httpPort);
String scraping = config.scrapingInterval() + "s";
String prometheusConfig = String.format(PROMETHEUS_CONFIG_DEFAULT, scraping);
if (config.forceScraping().orElse(scrapingRequired)) {
boolean isTest = LaunchMode.current() == LaunchMode.TEST;
Config runtimeConfig = ConfigProvider.getConfig();
String rootPath = runtimeConfig.getOptionalValue("quarkus.management.root-path", String.class).orElse("/q");
String metricsPath = runtimeConfig.getOptionalValue("quarkus.management.metrics.path", String.class)
.orElse("/metrics");
String httpPortKey = isTest ? "quarkus.http.test-port" : "quarkus.http.port";
Optional<Integer> optionalValue = runtimeConfig.getOptionalValue(httpPortKey, Integer.class);
int httpPort = optionalValue.orElse(isTest ? 8081 : 8080); // when not set use default
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is very confusing and brittle, but I have no better suggestions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, and the interesting thing is, this orElse actually kicks in at the end, as the prop is Optional.empty().


prometheusConfig += String.format(PROMETHEUS_CONFIG_SCRAPE, config.serviceName(), rootPath, metricsPath, scraping,
"host.docker.internal", httpPort);
}
return prometheusConfig;
}

protected static class LgtmConfigImpl extends AbstractGrafanaConfig implements LgtmConfig {
Expand All @@ -166,6 +184,31 @@ public Optional<Set<String>> networkAliases() {
public String otlpProtocol() {
return ContainerConstants.OTEL_HTTP_PROTOCOL;
}

@Override
public int scrapingInterval() {
return ContainerConstants.SCRAPING_INTERVAL;
}

@Override
public Optional<Boolean> forceScraping() {
return Optional.empty();
}

@Override
public String otelMetricExportInterval() {
return ContainerConstants.OTEL_METRIC_EXPORT_INTERVAL;
}

@Override
public String otelBspScheduleDelay() {
return ContainerConstants.OTEL_BSP_SCHEDULE_DELAY;
}

@Override
public String otelBlrpScheduleDelay() {
return ContainerConstants.OTEL_BLRP_SCHEDULE_DELAY;
}
}

protected static class LgtmLoggingFilter implements Predicate<OutputFrame> {
Expand Down
7 changes: 6 additions & 1 deletion ...resource-common/src/main/java/io/quarkus/observability/devresource/ExtensionsCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
package io.quarkus.observability.devresource;

import java.util.function.Function;

/**
* Relevant Observability extensions present.
*/
public record ExtensionsCatalog(boolean hasOpenTelemetry,
public record ExtensionsCatalog(
Function<String, Boolean> resourceChecker,
Function<String, Boolean> classChecker,
boolean hasOpenTelemetry,
boolean hasMicrometerOtlp) {
}
43 changes: 41 additions & 2 deletions ...evresource-lgtm/src/main/java/io/quarkus/observability/devresource/lgtm/LgtmResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;

import org.jboss.logging.Logger;

import io.quarkus.observability.common.ContainerConstants;
import io.quarkus.observability.common.config.LgtmConfig;
Expand All @@ -14,6 +18,22 @@

public class LgtmResource extends ContainerResource<LgtmContainer, LgtmConfig> {

private static final Logger log = Logger.getLogger(LgtmResource.class.getName());

protected static final Set<String> SCRAPING_REGISTRIES = Set.of(
"io.micrometer.prometheus.PrometheusMeterRegistry");

protected static final Function<String, Boolean> TCCL_FN = s -> {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
try {
cl.loadClass(s);
return true;
} catch (Exception e) {
// any exception
return false;
}
};

private ExtensionsCatalog catalog;
private LgtmConfig config;

Expand All @@ -32,7 +52,26 @@ public LgtmConfig config(ModulesConfiguration configuration, ExtensionsCatalog c

@Override
public Container<LgtmConfig> container(LgtmConfig config, ModulesConfiguration root) {
return set(new LgtmContainer(config));
return set(new LgtmContainer(config, isScrapingRequired(catalog.classChecker())));
}

private boolean isScrapingRequired(Function<String, Boolean> checker) {
boolean result = false;
String foundRegistry = null;
for (String clazz : SCRAPING_REGISTRIES) {
if (checker.apply(clazz)) {
foundRegistry = clazz;
result = true;
break;
}
}

if (result && (catalog != null && catalog.hasMicrometerOtlp())) {
log.warnf("Multiple Micrometer registries found - OTLP and %s, no Prometheus scrapping required.", foundRegistry);
return false;
}

return result;
}

private int getPrivateOtlpPort() {
Expand Down Expand Up @@ -86,7 +125,7 @@ public Map<String, String> config(int privatePort, String host, int publicPort)

@Override
protected LgtmContainer defaultContainer() {
return new LgtmContainer();
return new LgtmContainer(isScrapingRequired(TCCL_FN)); // best we can do?
}

@Override
Expand Down
Loading
Loading