Skip to content

Commit

Permalink
make explain return planner metrics as separate column (FoundationDB#…
Browse files Browse the repository at this point in the history
  • Loading branch information
normen662 authored Jan 28, 2025
1 parent cec2491 commit ee8f327
Show file tree
Hide file tree
Showing 15 changed files with 355 additions and 93 deletions.
2 changes: 1 addition & 1 deletion docs/ReleaseNotes.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ Our API stability annotations have been updated to reflect greater API instabili
* **Feature** Feature 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Feature** make EXPLAIN return a column for planner metrics [(Issue #3063)](https://github.com/FoundationDB/fdb-record-layer/issues/3063)
* **Feature** Feature 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Breaking change** Change 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
* **Breaking change** Change 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public Builder toBuilder() {
* @param <T> The type of the value (not used in this method)
* @return TRUE if the key exists in the table, FALSE otherwise
*/
public <T> boolean containsKey(@Nonnull QueryPlanInfoKey<T> key) {
public <T> boolean containsKey(@Nonnull final QueryPlanInfoKey<T> key) {
return info.containsKey(key);
}

Expand Down Expand Up @@ -111,22 +111,23 @@ public static class QueryPlanInfoKey<T> {
@Nonnull
private final String name;

public QueryPlanInfoKey(@Nonnull String name) {
public QueryPlanInfoKey(@Nonnull final String name) {
this.name = name;
}

@Nonnull
public String getName() {
return name;
}

// Suppress Unchecked Cast exception since all put() into the map use the right type for the value from the key.
@SuppressWarnings("unchecked")
public T narrow(@Nonnull Object o) {
public T narrow(@Nonnull final Object o) {
return (T) o;
}

@Override
public boolean equals(Object o) {
public boolean equals(final Object o) {
if (this == o) {
return true;
}
Expand Down Expand Up @@ -159,7 +160,7 @@ private Builder() {
infoMap = new HashMap<>();
}

private Builder(QueryPlanInfo source) {
private Builder(@Nonnull final QueryPlanInfo source) {
infoMap = new HashMap<>(source.info);
}

Expand All @@ -172,13 +173,13 @@ private Builder(QueryPlanInfo source) {
* @return this
*/
@Nonnull
public <T> Builder put(@Nonnull QueryPlanInfoKey<T> key, @Nonnull T value) {
public <T> Builder put(@Nonnull final QueryPlanInfoKey<T> key, @Nullable final T value) {
infoMap.put(key, value);
return this;
}

@Nullable
public <T> T get(@Nonnull QueryPlanInfoKey<T> key) {
public <T> T get(@Nonnull final QueryPlanInfoKey<T> key) {
return key.narrow(infoMap.get(key));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,20 @@

package com.apple.foundationdb.record.query.plan;

import com.apple.foundationdb.record.query.plan.cascades.debug.StatsMaps;

/**
* Container for {@link QueryPlanInfo.QueryPlanInfoKey} static instances used in the planner.
*/
public class QueryPlanInfoKeys {
public static final QueryPlanInfo.QueryPlanInfoKey<Integer> TOTAL_TASK_COUNT = new QueryPlanInfo.QueryPlanInfoKey<>("totalTaskCount");
public static final QueryPlanInfo.QueryPlanInfoKey<Integer> MAX_TASK_QUEUE_SIZE = new QueryPlanInfo.QueryPlanInfoKey<>("maxTaskQueueSize");
public static final QueryPlanInfo.QueryPlanInfoKey<QueryPlanConstraint> CONSTRAINTS = new QueryPlanInfo.QueryPlanInfoKey<>("constraints");
public static final QueryPlanInfo.QueryPlanInfoKey<Integer> TOTAL_TASK_COUNT =
new QueryPlanInfo.QueryPlanInfoKey<>("totalTaskCount");
public static final QueryPlanInfo.QueryPlanInfoKey<Integer> MAX_TASK_QUEUE_SIZE =
new QueryPlanInfo.QueryPlanInfoKey<>("maxTaskQueueSize");
public static final QueryPlanInfo.QueryPlanInfoKey<QueryPlanConstraint> CONSTRAINTS =
new QueryPlanInfo.QueryPlanInfoKey<>("constraints");
public static final QueryPlanInfo.QueryPlanInfoKey<StatsMaps> STATS_MAPS =
new QueryPlanInfo.QueryPlanInfoKey<>("statsMaps");

private QueryPlanInfoKeys() {
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,9 @@ public QueryPlanResult planQuery(@Nonnull final RecordQuery query, @Nonnull Para
.put(QueryPlanInfoKeys.TOTAL_TASK_COUNT, taskCount)
.put(QueryPlanInfoKeys.MAX_TASK_QUEUE_SIZE, maxQueueSize)
.put(QueryPlanInfoKeys.CONSTRAINTS, constraints)
.put(QueryPlanInfoKeys.STATS_MAPS,
Debugger.getDebuggerMaybe().flatMap(Debugger::getStatsMaps)
.orElse(null))
.build();
return new QueryPlanResult(plan, info);
}
Expand Down Expand Up @@ -354,7 +357,13 @@ public QueryPlanResult planGraph(@Nonnull Supplier<Reference> referenceSupplier,
evaluationContext);
final var plan = resultOrFail();
final var constraints = QueryPlanConstraint.collectConstraints(plan);
return new QueryPlanResult(plan, QueryPlanInfo.newBuilder().put(QueryPlanInfoKeys.CONSTRAINTS, constraints).build());
return new QueryPlanResult(plan,
QueryPlanInfo.newBuilder()
.put(QueryPlanInfoKeys.CONSTRAINTS, constraints)
.put(QueryPlanInfoKeys.STATS_MAPS,
Debugger.getDebuggerMaybe()
.flatMap(Debugger::getStatsMaps).orElse(null))
.build());
} finally {
Debugger.withDebugger(Debugger::onDone);
}
Expand Down Expand Up @@ -391,33 +400,40 @@ private void planPartial(@Nonnull Supplier<Reference> referenceSupplier,
maxQueueSize = 0;
while (!taskStack.isEmpty()) {
try {
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.ExecutingTaskEvent(currentRoot, taskStack, Objects.requireNonNull(taskStack.peek()))));
if (isTaskTotalCountExceeded(configuration, taskCount)) {
throw new RecordQueryPlanComplexityException("Maximum number of tasks (" + configuration.getMaxTotalTaskCount() + ") was exceeded");
}
taskCount++;

Debugger.withDebugger(debugger -> debugger.onEvent(
new Debugger.ExecutingTaskEvent(currentRoot, taskStack, Location.BEGIN,
Objects.requireNonNull(taskStack.peek()))));
Task nextTask = taskStack.pop();
if (logger.isTraceEnabled()) {
logger.trace(KeyValueLogMessage.of("executing task", "nextTask", nextTask.toString()));
}

Debugger.withDebugger(debugger -> debugger.onEvent(nextTask.toTaskEvent(Location.BEGIN)));
try {
nextTask.execute();
} finally {
Debugger.withDebugger(debugger -> debugger.onEvent(nextTask.toTaskEvent(Location.END)));
}
if (logger.isTraceEnabled()) {
logger.trace(KeyValueLogMessage.of("executing task", "nextTask", nextTask.toString()));
}

if (logger.isTraceEnabled()) {
logger.trace(KeyValueLogMessage.of("planner state",
"taskStackSize", taskStack.size(),
"memo", new ReferencePrinter(currentRoot)));
}
Debugger.withDebugger(debugger -> debugger.onEvent(nextTask.toTaskEvent(Location.BEGIN)));
try {
nextTask.execute();
} finally {
Debugger.withDebugger(debugger -> debugger.onEvent(nextTask.toTaskEvent(Location.END)));
}

maxQueueSize = Math.max(maxQueueSize, taskStack.size());
if (isTaskQueueSizeExceeded(configuration, taskStack.size())) {
throw new RecordQueryPlanComplexityException("Maximum task queue size (" + configuration.getMaxTaskQueueSize() + ") was exceeded");
if (logger.isTraceEnabled()) {
logger.trace(KeyValueLogMessage.of("planner state",
"taskStackSize", taskStack.size(),
"memo", new ReferencePrinter(currentRoot)));
}

maxQueueSize = Math.max(maxQueueSize, taskStack.size());
if (isTaskQueueSizeExceeded(configuration, taskStack.size())) {
throw new RecordQueryPlanComplexityException("Maximum task queue size (" + configuration.getMaxTaskQueueSize() + ") was exceeded");
}
} finally {
Debugger.withDebugger(debugger -> debugger.onEvent(
new Debugger.ExecutingTaskEvent(currentRoot, taskStack, Location.END, nextTask)));
}
} catch (final RestartException restartException) {
if (logger.isTraceEnabled()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ static Debugger getDebugger() {
return THREAD_LOCAL.get();
}

@Nonnull
static Optional<Debugger> getDebuggerMaybe() {
final var debugger = getDebugger();
return Optional.ofNullable(debugger);
}

/**
* Invoke the {@link Consumer} on the currently set debugger. Do not do anything if there is no debugger set.
* @param action consumer to invoke
Expand Down Expand Up @@ -207,6 +213,9 @@ static Optional<Integer> getOrRegisterSingleton(Object singleton) {
@SuppressWarnings("unused") // only used by debugger
String showStats();

@Nonnull
Optional<StatsMaps> getStatsMaps();

/**
* Shorthands to identify a kind of event.
*/
Expand Down Expand Up @@ -435,8 +444,9 @@ class ExecutingTaskEvent extends AbstractEventWithState {

public ExecutingTaskEvent(@Nonnull final Reference rootReference,
@Nonnull final Deque<Task> taskStack,
@Nonnull final Location location,
@Nonnull final Task task) {
super(rootReference, taskStack, Location.COUNT);
super(rootReference, taskStack, location);
this.task = task;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Stats.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.apple.foundationdb.record.query.plan.cascades.debug;

import com.google.common.collect.ImmutableMap;

import javax.annotation.Nonnull;
import java.util.Map;

public class Stats {
@Nonnull
protected final Map<Debugger.Location, Long> locationCountMap;

protected long totalTimeInNs;
protected long ownTimeInNs;

protected Stats(@Nonnull final Map<Debugger.Location, Long> locationCountMap,
final long totalTimeInNs,
final long ownTimeInNs) {
this.locationCountMap = locationCountMap;
this.totalTimeInNs = totalTimeInNs;
this.ownTimeInNs = ownTimeInNs;
}

@Nonnull
public Map<Debugger.Location, Long> getLocationCountMap() {
return locationCountMap;
}

public long getCount(@Nonnull final Debugger.Location location) {
return locationCountMap.getOrDefault(location, 0L);
}

public long getTotalTimeInNs() {
return totalTimeInNs;
}

public long getOwnTimeInNs() {
return ownTimeInNs;
}

@Nonnull
public Stats toImmutable() {
return new Stats(ImmutableMap.copyOf(locationCountMap), totalTimeInNs, ownTimeInNs);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* StatsMaps.java
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.apple.foundationdb.record.query.plan.cascades.debug;

import com.apple.foundationdb.record.query.plan.cascades.CascadesRule;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;

import javax.annotation.Nonnull;
import java.util.Map;
import java.util.function.Supplier;

public class StatsMaps {
@Nonnull
private final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap;
@Nonnull
private final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap;

@Nonnull
private final Supplier<Map<Class<? extends Debugger.Event>, Stats>> immutableEventClassStatsMapSupplier;
@Nonnull
private final Supplier<Map<Class<? extends CascadesRule<?>>, Stats>> immutablePlannerRuleClassStatsMapSupplier;

public StatsMaps(@Nonnull final Map<Class<? extends Debugger.Event>, ? extends Stats> eventClassStatsMap,
@Nonnull final Map<Class<? extends CascadesRule<?>>, ? extends Stats> plannerRuleClassStatsMap) {
this.eventClassStatsMap = eventClassStatsMap;
this.plannerRuleClassStatsMap = plannerRuleClassStatsMap;
this.immutableEventClassStatsMapSupplier = Suppliers.memoize(this::computeImmutableEventClassStatsMap);
this.immutablePlannerRuleClassStatsMapSupplier = Suppliers.memoize(this::computeImmutablePlannerRuleClassStatsMap);
}

@Nonnull
public Map<Class<? extends Debugger.Event>, Stats> getEventClassStatsMap() {
return immutableEventClassStatsMapSupplier.get();
}

@Nonnull
public Map<Class<? extends CascadesRule<?>>, Stats> getPlannerRuleClassStatsMap() {
return immutablePlannerRuleClassStatsMapSupplier.get();
}

@Nonnull
private Map<Class<? extends Debugger.Event>, Stats> computeImmutableEventClassStatsMap() {
final var eventClassStatsMapBuilder =
ImmutableMap.<Class<? extends Debugger.Event>, Stats>builder();
for (final var entry : eventClassStatsMap.entrySet()) {
eventClassStatsMapBuilder.put(entry.getKey(), entry.getValue().toImmutable());
}
return eventClassStatsMapBuilder.build();
}

@Nonnull
private Map<Class<? extends CascadesRule<?>>, Stats> computeImmutablePlannerRuleClassStatsMap() {
final var plannerRuleClassStatsMapBuilder =
ImmutableMap.<Class<? extends CascadesRule<?>>, Stats>builder();
for (final var entry : plannerRuleClassStatsMap.entrySet()) {
plannerRuleClassStatsMapBuilder.put(entry.getKey(), entry.getValue().toImmutable());
}
return plannerRuleClassStatsMapBuilder.build();
}
}
Loading

0 comments on commit ee8f327

Please sign in to comment.