Map error positions against the original source (sourcemapping)

packages/compiler/src/compile.ts

@@ -13,6 +13,7 @@ const defaultLogger = createLogger();
 // }
 
 export type Options = TransformOptions & {
+  name?: string;
   logger?: Logger;
   logCompiledSource?: boolean;
 };
@@ -27,15 +28,20 @@ export default function compile(pathOrSource: string, options: Options = {}) {
   } else {
     //logger.debug('Starting compilation from string');
   }
-  const ast = parse(source);
+
+  const name = options.name ?? 'src';
+  const ast = parse(source, { name });
 
   const transformedAst = transform(ast, undefined, options);
 
-  const compiledSource = print(transformedAst).code;
+  const { code, map } = print(transformedAst, {
+    sourceMapName: `${name}.map.js`,
+  });
+
   if (options.logCompiledSource) {
     logger.debug('Compiled source:');
-    logger.debug(compiledSource); // TODO indent or something
+    logger.debug(code);
   }
 
-  return compiledSource;
+  return { code, map };
 }

packages/compiler/src/parse.ts

@@ -11,11 +11,17 @@
 import recast from 'recast';
 import * as acorn from 'acorn';
 
-export default function parse(source: string) {
+type Options = {
+  /** Name of the source job (no file extension). This triggers source map generation */
+  name?: string;
+};
+
+export default function parse(source: string, options: Options = {}) {
   // This is copied from v1 but I am unsure the usecase
   const escaped = source.replace(/\ $/, '');
 
   const ast = recast.parse(escaped, {
+    sourceFileName: options.name && `${options.name}.js`,
     tolerant: true,
     range: true,
     parser: {

packages/compiler/test/compile.test.ts

@@ -3,53 +3,63 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import compile from '../src/compile';
 
+// Not doing deep testing on this because recast does the heavy lifting
+// This is just to ensure the map is actually generated
+test('generate a source map if a file name is passed', (t) => {
+  const source = 'fn();';
+  const { map } = compile(source, { name: 'job' });
+  t.truthy(map);
+  t.deepEqual(map.sources, ['job.js']);
+  t.is(map.file, 'job.map.js');
+});
+
 test('ensure default exports is created', (t) => {
   const source = '';
   const expected = 'export default [];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('do not add default exports if exports exist', (t) => {
   const source = 'export const x = 10;';
   const expected = 'export const x = 10;';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile a single operation', (t) => {
   const source = 'fn();';
   const expected = 'export default [fn()];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile a single namespaced operation', (t) => {
   const source = 'http.get();';
   const expected = 'export default [http.get()];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile a const assignment with single method call', (t) => {
   const source = 'const x = dateFns.parse()';
   const expected = `const x = dateFns.parse()
 export default [];`;
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile a single operation without being fussy about semicolons', (t) => {
   const source = 'fn()';
   const expected = 'export default [fn()];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile multiple operations', (t) => {
   const source = 'fn();fn();fn();';
   const expected = 'export default [fn(), fn(), fn()];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
@@ -66,7 +76,7 @@ test('add imports', (t) => {
   };
   const source = 'fn();';
   const expected = `import { fn } from "@openfn/language-common";\nexport default [fn()];`;
-  const result = compile(source, options);
+  const { code: result } = compile(source, options);
   t.is(result, expected);
 });
 
@@ -84,7 +94,7 @@ test('do not add imports', (t) => {
   // This example already has the correct imports declared, so add-imports should do nothing
   const source = "import { fn } from '@openfn/language-common'; fn();";
   const expected = `import { fn } from '@openfn/language-common';\nexport default [fn()];`;
-  const result = compile(source, options);
+  const { code: result } = compile(source, options);
   t.is(result, expected);
 });
 
@@ -101,7 +111,7 @@ test('dumbly add imports', (t) => {
   // This example already has the correct imports declared, so add-imports should do nothing
   const source = "import { jam } from '@openfn/language-common'; jam(state);";
   const expected = `import { jam } from '@openfn/language-common';\nexport default [jam(state)];`;
-  const result = compile(source, options);
+  const { code: result } = compile(source, options);
   t.is(result, expected);
 });
 
@@ -119,7 +129,7 @@ test('add imports with export all', (t) => {
   };
   const source = 'fn();';
   const expected = `import { fn } from "@openfn/language-common";\nexport * from "@openfn/language-common";\nexport default [fn()];`;
-  const result = compile(source, options);
+  const { code: result } = compile(source, options);
   t.is(result, expected);
 });
 
@@ -134,28 +144,28 @@ test('twitter example', async (t) => {
     path.resolve('test/jobs/twitter.compiled.js'),
     'utf8'
   );
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.deepEqual(result, expected);
 });
 
 test('compile with optional chaining', (t) => {
   const source = 'fn(a.b?.c);';
   const expected = 'export default [fn(a.b?.c)];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile with nullish coalescence', (t) => {
   const source = 'fn(a ?? b);';
   const expected = 'export default [fn(a ?? b)];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
 test('compile a lazy state ($) expression', (t) => {
   const source = 'get($.data.endpoint);';
   const expected = 'export default [get(state => state.data.endpoint)];';
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
@@ -175,7 +185,7 @@ test('compile a lazy state ($) expression with dumb imports', (t) => {
 export * from "@openfn/language-common";
 export default [get(state => state.data.endpoint)];`;
 
-  const result = compile(source, options);
+  const { code: result } = compile(source, options);
   t.is(result, expected);
 });
 
@@ -190,7 +200,7 @@ export default [_defer(
   p => p.then((s => { console.log(s.data); return state;} ))
 )];`;
 
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });
 
@@ -207,6 +217,6 @@ export default [each(
   _defer(post("/upsert", (state) => state.data), p => p.then((s) => s))
 )];`;
 
-  const result = compile(source);
+  const { code: result } = compile(source);
   t.is(result, expected);
 });

packages/compiler/test/parse.test.ts

@@ -6,6 +6,9 @@ import parse from '../src/parse';
 
 import { loadAst } from './util';
 
+// Note that these tests do not include source mappings
+// just because the ast changes and develops circular structures
+
 test('parse a simple statement', (t) => {
   const source = 'const x = 10;';
 

packages/runtime/package.json

@@ -26,13 +26,16 @@
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",
   "devDependencies": {
+    "@openfn/compiler": "workspace:^",
     "@openfn/language-common": "2.0.0-rc3",
     "@openfn/lexicon": "workspace:^",
     "@types/mock-fs": "^4.13.1",
     "@types/node": "^18.15.13",
     "@types/semver": "^7.5.0",
     "ava": "5.3.1",
     "mock-fs": "^5.4.1",
+    "recast": "^0.21.5",
+    "ts-node": "^10.9.1",
     "tslib": "^2.4.0",
     "tsup": "^7.2.0",
     "typescript": "^5.1.6"
@@ -45,6 +48,7 @@
   "dependencies": {
     "@openfn/logger": "workspace:*",
     "fast-safe-stringify": "^2.1.1",
-    "semver": "^7.5.4"
+    "semver": "^7.5.4",
+    "source-map": "^0.7.4"
   }
 }

packages/runtime/src/runtime.ts

@@ -5,29 +5,26 @@ import type { LinkerOptions } from './modules/linker';
 import executePlan from './execute/plan';
 import { defaultState, parseRegex, clone } from './util/index';
 
+// TODO we should be able to get a proper typing for this from somewherewhere
+type SourceMap = any;
+
 export type Options = {
   logger?: Logger;
   jobLogger?: Logger;
-
   // Treat state as immutable (likely to break in legacy jobs)
   immutableState?: boolean;
-
   // TODO currently unused
   // Ensure that all incoming jobs are sandboxed / loaded as text
   // In practice this means throwing if someone tries to pass live js
   forceSandbox?: boolean;
-
   linker?: LinkerOptions;
-
   callbacks?: ExecutionCallbacks;
-
   // inject globals into the environment
   // TODO leaving this here for now, but maybe its actually on the xplan?
   globals?: any;
-
   statePropsToRemove?: string[];
-
   defaultRunTimeoutMs?: number;
+  sourceMap?: SourceMap;
 };
 
 type RawOptions = Omit<Options, 'linker'> & {

packages/runtime/src/util/get-mapped-position.ts

@@ -0,0 +1,18 @@
+// TODO rename the file to source mapping
+// 1 function to get the position
+// 1 function to extract the line from the source
+
+import { SourceMapConsumer } from 'source-map';
+
+const getMappedPosition = async (map, line, column) => {
+  const smc = await new SourceMapConsumer(map);
+  const pos = smc.originalPositionFor({
+    line,
+    column,
+  });
+
+  //return { line, col, src };
+  return pos;
+};
+
+export default getMappedPosition;

packages/runtime/test/errors.test.ts

@@ -1,6 +1,7 @@
 import test from 'ava';
 import path from 'node:path';
 import type { WorkflowOptions } from '@openfn/lexicon';
+import compile from '@openfn/compiler';
 
 import run from '../src/runtime';
 
@@ -47,13 +48,18 @@ test('crash on runtime error with SyntaxError', async (t) => {
   t.is(error.message, 'SyntaxError: Invalid or unexpected token');
 });
 
-test('crash on runtime error with ReferenceError', async (t) => {
+test.only('crash on runtime error with ReferenceError', async (t) => {
   const expression = 'export default [(s) => x]';
 
+  // compile the code so we get a source map
+  const { code, map } = compile(expression, { name: 'src' });
+  console.log({ code, map });
+
   let error: any;
   try {
-    await run(expression);
+    await run(code, { map });
   } catch (e) {
+    console.log(e);
     error = e;
   }
   t.log(error);

packages/runtime/test/util/get-mapped-position.test.ts

@@ -0,0 +1,43 @@
+import test from 'ava';
+import recast from 'recast';
+import getMappedPosition from '../../src/util/get-mapped-position';
+
+const b = recast.types.builders;
+
+// compile an expression into a function
+const compile = (src: string) => {
+  const ast = recast.parse(src, {
+    sourceFileName: 'src.js',
+  });
+
+  // take the expression and wrap it in a function declaration
+  const [{ expression }] = ast.program.body;
+  const fn = b.functionDeclaration(
+    b.identifier('fn'),
+    [],
+    b.blockStatement([b.returnStatement(expression)])
+  );
+
+  ast.program.body.push(fn);
+
+  return recast.print(fn, {
+    sourceMapName: 'src.map.js',
+  });
+};
+
+test('should return a sourcemapped position', async (t) => {
+  // Compile this simple expression into a function
+  const { code, map } = compile('x + 1');
+
+  // Now work out where x is
+  const lines = code.split('\n');
+  const line = 2; // line is 1 based
+  const column = lines[1].indexOf('x');
+
+  // now get the uncompiled position for x
+  const result = await getMappedPosition(map, line, column);
+
+  // now check the position
+  t.is(result.line, 1);
+  t.is(result.column, 0);
+});