From 9dd5f32d524169dbcbf3bc059310685886177e9f Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 15 Mar 2024 00:53:30 +0100 Subject: [PATCH] Use USING in joins - rename id columns to use same name as foreign keys - use critId instead of criterion in Measurement Signed-off-by: Stefan Marr --- src/backend/db/db.sql | 24 ++++----- src/backend/db/db.ts | 61 ++++++++++++---------- src/backend/db/types.ts | 6 +-- src/backend/main/main.ts | 2 +- src/backend/project/data-export.ts | 2 +- src/shared/data-format.ts | 2 +- tests/backend/compare/compare-view.test.ts | 2 +- tests/backend/db/db-setup.test.ts | 8 +-- tests/shared/data-format.test.ts | 2 +- 9 files changed, 57 insertions(+), 52 deletions(-) diff --git a/src/backend/db/db.sql b/src/backend/db/db.sql index 047496b6..3f067780 100644 --- a/src/backend/db/db.sql +++ b/src/backend/db/db.sql @@ -1,7 +1,7 @@ -- A specific software version, possibly used by multiple environments -- or versions of environments. CREATE TABLE SoftwareVersionInfo ( - id serial primary key, + softId serial primary key, name varchar, version varchar, unique (name, version) @@ -10,7 +10,7 @@ CREATE TABLE SoftwareVersionInfo ( -- Identifies the specific state of an environment, including -- the relevant software versions. CREATE TABLE Environment ( - id serial primary key, + envId serial primary key, hostname varchar unique, osType varchar, -- total number of bytes of memory provided by the system @@ -26,7 +26,7 @@ CREATE TABLE Environment ( -- This can be anything, from total time over memory consumption -- to other things or parts worth measuring. CREATE TABLE Criterion ( - id serial primary key, + critId serial primary key, name varchar, unit varchar, @@ -112,7 +112,7 @@ CREATE TABLE Trial ( unique (username, envId, expId, startTime), foreign key (expId) references Experiment (id), - foreign key (envId) references Environment (id), + foreign key (envId) references Environment (envId), foreign key (sourceId) references Source (id) ); @@ -122,8 +122,8 @@ CREATE TABLE SoftwareUse ( softId smallint, primary key (envId, softId), - foreign key (envId) references Environment (id), - foreign key (softId) references SoftwareVersionInfo (id) + foreign key (envId) references Environment (envId), + foreign key (softId) references SoftwareVersionInfo (softId) ); -- A concrete execution of a benchmark by a specific executor. @@ -152,16 +152,16 @@ CREATE TABLE Run ( CREATE TABLE Measurement ( runId smallint, trialId smallint, - criterion smallint, + critId smallint, invocation smallint, iteration smallint, value float4 NOT NULL, - primary key (iteration, invocation, runId, trialId, criterion), + primary key (iteration, invocation, runId, trialId, critId), foreign key (trialId) references Trial (id), foreign key (runId) references Run (id), - foreign key (criterion) references Criterion (id) + foreign key (critId) references Criterion (critId) ); CREATE TABLE ProfileData ( @@ -181,7 +181,7 @@ CREATE TABLE ProfileData ( CREATE TABLE Timeline ( runId smallint, trialId smallint, - criterion smallint, + critId smallint, numSamples int, @@ -195,8 +195,8 @@ CREATE TABLE Timeline ( bci95low float4, bci95up float4, - primary key (runId, trialId, criterion), + primary key (runId, trialId, critId), foreign key (trialId) references Trial (id), foreign key (runId) references Run (id), - foreign key (criterion) references Criterion (id) + foreign key (critId) references Criterion (critId) ); diff --git a/src/backend/db/db.ts b/src/backend/db/db.ts index 916e8b42..d300d276 100644 --- a/src/backend/db/db.ts +++ b/src/backend/db/db.ts @@ -63,7 +63,7 @@ const measurementDataTableJoins = ` JOIN Trial ON trialId = Trial.id JOIN Experiment ON expId = Experiment.id JOIN Source ON source.id = sourceId - JOIN Criterion ON criterion = criterion.id + JOIN Criterion USING (critId) JOIN Run ON runId = run.id`; function filterCommitMessage(msg) { @@ -113,7 +113,7 @@ export abstract class Database { this.statsValid = new TimedCacheValidity(cacheInvalidationDelay); this.queries.insertMeasurementBatchedN = `INSERT INTO Measurement - (runId, trialId, invocation, iteration, criterion, value) + (runId, trialId, invocation, iteration, critId, value) VALUES ${this.generateBatchInsert(Database.batchN, 6)} ON CONFLICT DO NOTHING`; @@ -338,7 +338,7 @@ export abstract class Database { JOIN Experiment exp ON exp.projectId = p.id JOIN Trial t ON t.expId = exp.id JOIN Source src ON t.sourceId = src.id - JOIN Environment env ON t.envId = env.id + JOIN Environment env USING (envId) JOIN Timeline tl ON tl.trialId = t.id JOIN Run r ON tl.runId = r.id WHERE p.id = $1 @@ -443,7 +443,7 @@ export abstract class Database { exp: Experiment ): Promise { const e = data.env; - const cacheKey = `${e.userName}-${env.id}-${data.startTime}-${exp.id}`; + const cacheKey = `${e.userName}-${env.envid}-${data.startTime}-${exp.id}`; if (this.trials.has(cacheKey)) { return this.trials.get(cacheKey); @@ -458,7 +458,7 @@ export abstract class Database { text: `SELECT * FROM Trial WHERE username = $1 AND envId = $2 AND startTime = $3 AND expId = $4`, - values: [e.userName, env.id, data.startTime, exp.id] + values: [e.userName, env.envid, data.startTime, exp.id] }, { name: 'insertTrial', @@ -470,7 +470,7 @@ export abstract class Database { data.startTime, exp.id, e.userName, - env.id, + env.envid, source.id, e.denoise ] @@ -749,7 +749,7 @@ export abstract class Database { FROM ${measurementDataTableJoins} WHERE (commitId = $2 OR commitid = $3) AND Experiment.projectId = $4 - ORDER BY runId, expId, trialId, criterion, invocation, iteration`, + ORDER BY runId, expId, trialId, critId, invocation, iteration`, values: [minDistinctLength, commitHash1, commitHash2, projectId] }); return result.rows; @@ -768,7 +768,7 @@ export abstract class Database { FROM Source src JOIN Trial t ON t.sourceId = src.id JOIN Experiment exp ON exp.id = t.expId - JOIN Environment env ON t.envId = env.id + JOIN Environment env USING (envId) WHERE (commitId = $1 OR commitid = $2) AND exp.projectId = $3`, values: [commitHash1, commitHash2, projectId] }); @@ -787,7 +787,7 @@ export abstract class Database { WHERE Experiment.id = $2 ORDER BY - runId, trialId, cmdline, invocation, iteration, criterion`, + runId, trialId, cmdline, invocation, iteration, critId`, values: [6, expId] }); return result.rows; @@ -865,32 +865,32 @@ export abstract class Database { const results = await this.query({ name: 'fetchAvailableMeasurements', text: `SELECT - runId, criterion, invocation as inv, max(iteration) as ite + runId, critId, invocation as inv, max(iteration) as ite FROM Measurement WHERE trialId = $1 - GROUP BY runId, criterion, invocation - ORDER BY runId, inv, ite, criterion`, + GROUP BY runId, critId, invocation + ORDER BY runId, inv, ite, critId`, values: [trialId] }); const measurements = {}; for (const r of results.rows) { - // runid, criterion, inv, ite + // runid, critId, inv, ite if (!(r.runid in measurements)) { measurements[r.runid] = {}; } const run = measurements[r.runid]; - if (!(r.criterion in run)) { - run[r.criterion] = {}; + if (!(r.critid in run)) { + run[r.critid] = {}; } - const crit = run[r.criterion]; + const crit = run[r.critid]; assert( !(r.inv in crit), - `${r.runid}, ${r.criterion}, ${r.inv} in ${JSON.stringify(crit)}` + `${r.runid}, ${r.critid}, ${r.inv} in ${JSON.stringify(crit)}` ); crit[r.inv] = r.ite; } @@ -976,14 +976,19 @@ export abstract class Database { // batched inserts are much faster // so let's do this const criterion = criteria.get(m.c); - const values = [run.id, trial.id, d.in, d.it, criterion.id, m.v]; + const values = [run.id, trial.id, d.in, d.it, criterion.critid, m.v]; if (this.alreadyRecorded(availableMs, values)) { // then,just skip this one. continue; } if (this.timelineUpdater && criterion.name === TotalCriterion) { - this.timelineUpdater.addValue(run.id, trial.id, criterion.id, m.v); + this.timelineUpdater.addValue( + run.id, + trial.id, + criterion.critid, + m.v + ); } batchedMs += 1; batchedValues = batchedValues.concat(values); @@ -1106,7 +1111,7 @@ export abstract class Database { const q = { name: 'insertMeasurement10', text: `INSERT INTO Measurement - (runId, trialId, invocation, iteration, criterion, value) + (runId, trialId, invocation, iteration, critId, value) VALUES ($1, $2, $3, $4, $5, $6), ($7, $8, $9, $10, $11, $12), @@ -1142,7 +1147,7 @@ export abstract class Database { const q = { name: 'insertMeasurement', text: `INSERT INTO Measurement - (runId, trialId, invocation, iteration, criterion, value) + (runId, trialId, invocation, iteration, critId, value) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, // [runId, trialId, invocation, iteration, critId, value]; @@ -1155,18 +1160,18 @@ export abstract class Database { public async recordTimeline( runId: number, trialId: number, - criterion: number, + critId: number, stats: SummaryStatistics ): Promise { const q = { name: 'insertTimelineStats', text: `INSERT INTO timeline - (runid, trialid, criterion, + (runid, trialid, critid, minval, maxval, sdval, mean, median, numsamples, bci95low, bci95up) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - ON CONFLICT (runid, trialid, criterion) DO UPDATE + ON CONFLICT (runid, trialid, critid) DO UPDATE SET minval = EXCLUDED.minval, maxval = EXCLUDED.maxval, @@ -1179,7 +1184,7 @@ export abstract class Database { values: [ runId, trialId, - criterion, + critId, stats.min, stats.max, stats.standardDeviation, @@ -1325,7 +1330,7 @@ export abstract class Database { JOIN Experiment exp ON exp.projectId = p.id JOIN Trial t ON t.expId = exp.id JOIN Source src ON t.sourceId = src.id - JOIN Environment env ON t.envId = env.id + JOIN Environment env USING (envId) JOIN Timeline tl ON tl.trialId = t.id JOIN Run r ON tl.runId = r.id JOIN LatestExperiment le ON exp.id = le.expId @@ -1506,7 +1511,7 @@ export abstract class Database { JOIN Experiment e ON tr.expId = e.id JOIN Project p ON p.id = e.projectId JOIN Run r ON r.id = ti.runId - JOIN Criterion c ON ti.criterion = c.id + JOIN Criterion c USING (critId) WHERE s.branchOrTag = p.baseBranch AND p.id = $1 AND @@ -1538,7 +1543,7 @@ export abstract class Database { JOIN Experiment e ON tr.expId = e.id JOIN Project p ON p.id = e.projectId JOIN Run r ON r.id = ti.runId - JOIN Criterion c ON ti.criterion = c.id + JOIN Criterion c USING (critId) WHERE s.branchOrTag IN ($3, $4) AND p.name = $5 AND diff --git a/src/backend/db/types.ts b/src/backend/db/types.ts index 60e6dbb7..e2b8c8fa 100644 --- a/src/backend/db/types.ts +++ b/src/backend/db/types.ts @@ -28,13 +28,13 @@ export interface Benchmark { } export interface SoftwareVersionInfo { - id: number; + softId: number; name: string; version: string; } export interface Environment { - id: number; + envid: number; hostname: string; ostype: string; memory: number; @@ -50,7 +50,7 @@ export interface Unit { } export interface Criterion { - id: number; + critid: number; name: string; unit: string; } diff --git a/src/backend/main/main.ts b/src/backend/main/main.ts index 8c5747e4..3c632f9a 100644 --- a/src/backend/main/main.ts +++ b/src/backend/main/main.ts @@ -75,7 +75,7 @@ export async function getLast100Measurements( JOIN Trial t ON m.trialId = t.id JOIN Experiment e ON t.expId = e.id JOIN Run r ON m.runId = r.id - JOIN Criterion c ON m.criterion = c.id + JOIN Criterion c USING (critId) WHERE projectId = $1 AND c.name = '${TotalCriterion}' ORDER BY t.startTime, m.invocation, m.iteration diff --git a/src/backend/project/data-export.ts b/src/backend/project/data-export.ts index cabe8730..b1e89796 100644 --- a/src/backend/project/data-export.ts +++ b/src/backend/project/data-export.ts @@ -110,7 +110,7 @@ export async function getDataOverview( FROM experiment exp JOIN Trial t ON exp.id = t.expId JOIN Source src ON t.sourceId = src.id - JOIN Environment env ON env.id = t.envId + JOIN Environment env USING (envId) --JOIN Measurement m ON m.trialId = t.id JOIN Timeline tl ON tl.trialId = t.id diff --git a/src/shared/data-format.ts b/src/shared/data-format.ts index cf5eb9df..2e30eac7 100644 --- a/src/shared/data-format.ts +++ b/src/shared/data-format.ts @@ -75,7 +75,7 @@ export function formatEnvironment( envId: number, environments: Environment[] ): string | undefined { - const env = environments.find((e) => e.id === envId); + const env = environments.find((e) => e.envid === envId); if (env === undefined) { return undefined; } diff --git a/tests/backend/compare/compare-view.test.ts b/tests/backend/compare/compare-view.test.ts index 71d2d20a..960d1014 100644 --- a/tests/backend/compare/compare-view.test.ts +++ b/tests/backend/compare/compare-view.test.ts @@ -64,7 +64,7 @@ const benchId = { const environments: Environment[] = [ { - id: 1, + envid: 1, hostname: 'MyHost', ostype: 'Linux', memory: 123456, diff --git a/tests/backend/db/db-setup.test.ts b/tests/backend/db/db-setup.test.ts index 6ddf2bd6..9b261e8a 100644 --- a/tests/backend/db/db-setup.test.ts +++ b/tests/backend/db/db-setup.test.ts @@ -121,7 +121,7 @@ describe('Recording a ReBench execution data fragments', () => { const result = await db.recordTrial(basicTestData, env, exp); expect(e.userName).toEqual(result.username); expect(e.manualRun).toEqual(result.manualrun); - expect(env.id).toEqual(result.envid); + expect(env.envid).toEqual(result.envid); }); it('should accept trial denoise info', async () => { @@ -146,7 +146,7 @@ describe('Recording a ReBench execution data fragments', () => { const result = await db.recordTrial(testData, env, exp); expect(e.userName).toEqual(result.username); expect(e.manualRun).toEqual(result.manualrun); - expect(env.id).toEqual(result.envid); + expect(env.envid).toEqual(result.envid); expect(e.denoise.scaling_governor).toEqual('performance'); expect(e.denoise).toEqual(result.denoise); }); @@ -165,8 +165,8 @@ describe('Recording a ReBench execution data fragments', () => { const criterion = await db.recordCriterion(c); expect(c.c).toEqual(criterion.name); expect(c.u).toEqual(criterion.unit); - expect(typeof criterion.id).toEqual('number'); - expect(criterion.id).toBeGreaterThanOrEqual(0); + expect(typeof criterion.critid).toEqual('number'); + expect(criterion.critid).toBeGreaterThanOrEqual(0); }); }); diff --git a/tests/shared/data-format.test.ts b/tests/shared/data-format.test.ts index 055a7d93..e8da3135 100644 --- a/tests/shared/data-format.test.ts +++ b/tests/shared/data-format.test.ts @@ -133,7 +133,7 @@ describe('Format Functions for Numerical Values', () => { describe('formatEnvironment - for display to user', () => { const envs: Environment[] = [ { - id: 1, + envid: 1, hostname: 'host', ostype: 'Linux', memory: 453454333,