Skip to content

Commit

Permalink
[APM] Paginate big traces (elastic#165584)
Browse files Browse the repository at this point in the history
  • Loading branch information
sorenlouv authored Sep 11, 2023
1 parent 1ada79e commit 93ba71c
Show file tree
Hide file tree
Showing 24 changed files with 807 additions and 152 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
timestamp,
children: (_) => {
_.service({
repeat: 10,
repeat: 80,
serviceInstance: synthNode,
transactionName: 'GET /nodejs/products',
latency: 100,
Expand All @@ -60,7 +60,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
transactionName: 'GET /go',
children: (_) => {
_.service({
repeat: 20,
repeat: 50,
serviceInstance: synthJava,
transactionName: 'GET /java',
children: (_) => {
Expand All @@ -83,15 +83,15 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
serviceInstance: synthNode,
transactionName: 'GET /nodejs/users',
latency: 100,
repeat: 10,
repeat: 40,
children: (_) => {
_.service({
serviceInstance: synthGo,
transactionName: 'GET /go/security',
latency: 50,
children: (_) => {
_.service({
repeat: 10,
repeat: 40,
serviceInstance: synthDotnet,
transactionName: 'GET /dotnet/cases/4',
latency: 50,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ describe('getCriticalPath', () => {
errorDocs: [],
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: events.length,
traceDocsTotal: events.length,
maxTraceItems: 5000,
},
entryTransaction,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

/* eslint-disable @typescript-eslint/no-shadow */
import { apm, timerange, DistributedTrace } from '@kbn/apm-synthtrace-client';
import { synthtrace } from '../../../../synthtrace';

const RATE_PER_MINUTE = 1;

export function generateLargeTrace({
start,
end,
rootTransactionName,
repeaterFactor,
environment,
}: {
start: number;
end: number;
rootTransactionName: string;
repeaterFactor: number;
environment: string;
}) {
const range = timerange(start, end);

const synthRum = apm
.service({ name: 'synth-rum', environment, agentName: 'rum-js' })
.instance('my-instance');

const synthNode = apm
.service({ name: 'synth-node', environment, agentName: 'nodejs' })
.instance('my-instance');

const synthGo = apm
.service({ name: 'synth-go', environment, agentName: 'go' })
.instance('my-instance');

const synthDotnet = apm
.service({ name: 'synth-dotnet', environment, agentName: 'dotnet' })
.instance('my-instance');

const synthJava = apm
.service({ name: 'synth-java', environment, agentName: 'java' })
.instance('my-instance');

const traces = range.ratePerMinute(RATE_PER_MINUTE).generator((timestamp) => {
return new DistributedTrace({
serviceInstance: synthRum,
transactionName: rootTransactionName,
timestamp,
children: (_) => {
_.service({
repeat: 5 * repeaterFactor,
serviceInstance: synthNode,
transactionName: 'GET /nodejs/products',
latency: 100,

children: (_) => {
_.service({
serviceInstance: synthGo,
transactionName: 'GET /go',
children: (_) => {
_.service({
repeat: 5 * repeaterFactor,
serviceInstance: synthJava,
transactionName: 'GET /java',
children: (_) => {
_.external({
name: 'GET telemetry.elastic.co',
url: 'https://telemetry.elastic.co/ping',
duration: 50,
});
},
});
},
});
_.db({
name: 'GET apm-*/_search',
type: 'elasticsearch',
duration: 400,
});
_.db({ name: 'GET', type: 'redis', duration: 500 });
_.db({
name: 'SELECT * FROM users',
type: 'sqlite',
duration: 600,
});
},
});

_.service({
serviceInstance: synthNode,
transactionName: 'GET /nodejs/users',
latency: 100,
repeat: 5 * repeaterFactor,
children: (_) => {
_.service({
serviceInstance: synthGo,
transactionName: 'GET /go/security',
latency: 50,
children: (_) => {
_.service({
repeat: 5 * repeaterFactor,
serviceInstance: synthDotnet,
transactionName: 'GET /dotnet/cases/4',
latency: 50,
children: (_) =>
_.db({
name: 'GET apm-*/_search',
type: 'elasticsearch',
duration: 600,
statement: JSON.stringify(
{
query: {
query_string: {
query: '(new york city) OR (big apple)',
default_field: 'content',
},
},
},
null,
2
),
}),
});
},
});
},
});
},
}).getTransaction();
});

return synthtrace.index(traces);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { synthtrace } from '../../../../synthtrace';
import { generateLargeTrace } from './generate_large_trace';

const start = '2021-10-10T00:00:00.000Z';
const end = '2021-10-10T00:01:00.000Z';
const rootTransactionName = `Large trace`;

const timeRange = { rangeFrom: start, rangeTo: end };

describe('Large Trace in waterfall', () => {
before(() => {
synthtrace.clean();

generateLargeTrace({
start: new Date(start).getTime(),
end: new Date(end).getTime(),
rootTransactionName,
repeaterFactor: 10,
environment: 'large_trace',
});
});

after(() => {
synthtrace.clean();
});

describe('when navigating to a trace sample with default maxTraceItems', () => {
beforeEach(() => {
cy.loginAsViewerUser();
cy.visitKibana(
`/app/apm/services/synth-rum/transactions/view?${new URLSearchParams({
...timeRange,
transactionName: rootTransactionName,
})}`
);
});

it('renders waterfall items', () => {
cy.getByTestSubj('waterfallItem').should('have.length.greaterThan', 200);
});

it('shows warning about trace size', () => {
cy.getByTestSubj('apmWaterfallSizeWarning').should(
'have.text',
'The number of items in this trace is 15551 which is higher than the current limit of 5000. Please increase the limit via `xpack.apm.ui.maxTraceItems` to see the full trace'
);
});
});

describe('when navigating to a trace sample with maxTraceItems=20000', () => {
beforeEach(() => {
cy.loginAsViewerUser();

cy.intercept('GET', '/internal/apm/traces/**', (req) => {
req.query.maxTraceItems = 20000;
}).as('getTraces');

cy.visitKibana(
`/app/apm/services/synth-rum/transactions/view?${new URLSearchParams({
...timeRange,
transactionName: rootTransactionName,
})}`
);
});

it('renders waterfall items', () => {
cy.getByTestSubj('waterfallItem').should('have.length.greaterThan', 400);
});

it('does not show the warning about trace size', () => {
cy.getByTestSubj('apmWaterfallSizeWarning').should('not.exist');
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const INITIAL_DATA: APIReturnType<'GET /internal/apm/traces/{traceId}'> = {
traceDocs: [],
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: 0,
traceDocsTotal: 0,
maxTraceItems: 0,
},
entryTransaction: undefined,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ export function AccordionWaterfall(props: AccordionWaterfallProps) {

return (
<StyledAccordion
data-test-subj="waterfallItem"
className="waterfall_accordion"
style={{ position: 'relative' }}
buttonClassName={`button_${item.id}`}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,15 @@ export function Waterfall({
<Container>
{waterfall.exceedsMax && (
<EuiCallOut
data-test-subj="apmWaterfallSizeWarning"
color="warning"
size="s"
iconType="warning"
title={i18n.translate('xpack.apm.waterfall.exceedsMax', {
defaultMessage:
'The number of items in this trace is {traceItemCount} which is higher than the current limit of {maxTraceItems}. Please increase the limit to see the full trace',
'The number of items in this trace is {traceDocsTotal} which is higher than the current limit of {maxTraceItems}. Please increase the limit via `xpack.apm.ui.maxTraceItems` to see the full trace',
values: {
traceItemCount: waterfall.traceItemCount,
traceDocsTotal: waterfall.traceDocsTotal,
maxTraceItems: waterfall.maxTraceItems,
},
})}
Expand Down Expand Up @@ -161,9 +162,9 @@ export function Waterfall({
) => toggleFlyout({ history, item, flyoutDetailTab })}
showCriticalPath={showCriticalPath}
maxLevelOpen={
waterfall.traceItemCount > 500
waterfall.traceDocsTotal > 500
? maxLevelOpen
: waterfall.traceItemCount
: waterfall.traceDocsTotal
}
/>
)}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ describe('waterfall_helpers', () => {
errorDocs,
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: hits.length,
traceDocsTotal: hits.length,
maxTraceItems: 5000,
},
entryTransaction: {
Expand Down Expand Up @@ -168,7 +168,7 @@ describe('waterfall_helpers', () => {
errorDocs,
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: hits.length,
traceDocsTotal: hits.length,
maxTraceItems: 5000,
},
entryTransaction: {
Expand Down Expand Up @@ -271,7 +271,7 @@ describe('waterfall_helpers', () => {
errorDocs: [],
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: traceItems.length,
traceDocsTotal: traceItems.length,
maxTraceItems: 5000,
},
entryTransaction: {
Expand Down Expand Up @@ -390,7 +390,7 @@ describe('waterfall_helpers', () => {
errorDocs: [],
exceedsMax: false,
spanLinksCountById: {},
traceItemCount: traceItems.length,
traceDocsTotal: traceItems.length,
maxTraceItems: 5000,
},
entryTransaction: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export interface IWaterfall {
errorItems: IWaterfallError[];
exceedsMax: boolean;
totalErrorsCount: number;
traceItemCount: number;
traceDocsTotal: number;
maxTraceItems: number;
}

Expand Down Expand Up @@ -427,7 +427,7 @@ export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
getErrorCount: () => 0,
exceedsMax: false,
totalErrorsCount: 0,
traceItemCount: 0,
traceDocsTotal: 0,
maxTraceItems: 0,
};
}
Expand Down Expand Up @@ -476,7 +476,7 @@ export function getWaterfall(apiResponse: TraceAPIResponse): IWaterfall {
getErrorCount: (parentId: string) => errorCountByParentId[parentId] ?? 0,
exceedsMax: traceItems.exceedsMax,
totalErrorsCount: traceItems.errorDocs.length,
traceItemCount: traceItems.traceItemCount,
traceDocsTotal: traceItems.traceDocsTotal,
maxTraceItems: traceItems.maxTraceItems,
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export const Example: Story<any> = () => {
traceDocs,
errorDocs: errorDocs.map((error) => dedot(error, {}) as WaterfallError),
spanLinksCountById: {},
traceItemCount: traceDocs.length,
traceDocsTotal: traceDocs.length,
maxTraceItems: 5000,
};

Expand Down
Loading

0 comments on commit 93ba71c

Please sign in to comment.