Consensys · cd1m0 · Jan 18, 2024 · Jan 11, 2024 · Jan 11, 2024 · Jan 15, 2024
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -20,6 +20,6 @@
     "eslint.alwaysShowStatus": true,
     "eslint.validate": ["javascript", "typescript"],
     "editor.codeActionsOnSave": {
-        "source.fixAll.eslint": true
+        "source.fixAll.eslint": "explicit"
     }
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -12,14 +12,14 @@
         "scribble": "dist/bin/scribble.js"
     },
     "dependencies": {
-        "big-integer": "^1.6.51",
+        "big-integer": "^1.6.52",
         "command-line-args": "^5.2.1",
         "command-line-usage": "^7.0.1",
         "findup-sync": "^5.0.0",
         "fs-extra": "^11.1.1",
         "logplease": "^1.2.15",
         "semver": "^7.5.4",
-        "solc-typed-ast": "^17.0.2",
+        "solc-typed-ast": "^18.0.0",
         "src-location": "^1.1.0",
         "yaml": "^1.10.2"
     },
@@ -39,23 +39,23 @@
     },
     "devDependencies": {
         "@ethereumjs/vm": "^6.5.0",
-        "@types/bn.js": "^5.1.4",
-        "@types/fs-extra": "^11.0.3",
-        "@types/mocha": "^10.0.3",
-        "@types/node": "^16.18.60",
-        "@types/semver": "^7.5.4",
-        "@typescript-eslint/eslint-plugin": "^6.9.1",
-        "@typescript-eslint/parser": "^6.9.1",
-        "eslint": "^8.52.0",
-        "eslint-config-prettier": "^9.0.0",
-        "eslint-plugin-prettier": "^5.0.1",
+        "@types/bn.js": "^5.1.5",
+        "@types/fs-extra": "^11.0.4",
+        "@types/mocha": "^10.0.6",
+        "@types/node": "^16.18.71",
+        "@types/semver": "^7.5.6",
+        "@typescript-eslint/eslint-plugin": "^6.18.1",
+        "@typescript-eslint/parser": "^6.18.1",
+        "eslint": "^8.56.0",
+        "eslint-config-prettier": "^9.1.0",
+        "eslint-plugin-prettier": "^5.1.3",
         "ethereumjs-abi": "^0.6.8",
         "expect": "^29.7.0",
         "mocha": "^10.2.0",
         "nyc": "^15.1.0",
         "peggy": "^2.0.1",
-        "prettier": "^3.0.3",
-        "ts-node": "^10.9.1",
+        "prettier": "^3.2.2",
+        "ts-node": "^10.9.2",
         "ts-pegjs": "^3.1.0",
         "typescript": "^5.2.2"
     },

diff --git a/src/ast_to_source_printer.ts b/src/ast_to_source_printer.ts
@@ -1,5 +1,4 @@
 import {
-    assert,
     ASTNodeFactory,
     ASTWriter,
     DefaultASTWriterMapping,
@@ -8,11 +7,14 @@ import {
     PrettyFormatter,
     SourceUnit,
     SrcRangeMap,
-    SymbolAlias
+    SymbolAlias,
+    assert,
+    toUTF8
 } from "solc-typed-ast";
 import { ImportDirectiveDesc } from "./rewriter/import_directive_header";
 import { parse as parseImportDirective } from "./rewriter/import_directive_parser";
 import { SourceMap } from "./util/sources";
+import { strUTF8Len } from "./util";
 
 /**
  * Find an import named `name` imported from source unit `from`. This will
@@ -103,8 +105,8 @@ export function rewriteImports(
 
         assert(source !== undefined, `Missing source for ${sourceUnit.absolutePath}`);
 
-        const importDirSrc = importDir.extractSourceFragment(source.contents);
-        const importDesc: ImportDirectiveDesc = parseImportDirective(importDirSrc);
+        const importDirSrc = importDir.extractSourceFragment(source.rawContents);
+        const importDesc: ImportDirectiveDesc = parseImportDirective(toUTF8(importDirSrc));
 
         assert(
             importDesc.symbolAliases.length === importDir.symbolAliases.length,
@@ -164,6 +166,7 @@ export function print(
 ): Map<SourceUnit, string> {
     const writer = getWriter(compilerVersion);
     const result = new Map<SourceUnit, string>();
+    const markerLen = instrumentationMarker === undefined ? 0 : strUTF8Len(instrumentationMarker);
 
     for (const unit of sourceUnits) {
         const source = writer.write(unit, srcMap);
@@ -183,7 +186,7 @@ export function print(
                 const src = srcMap.get(node);
 
                 if (src !== undefined) {
-                    src[0] += instrumentationMarker.length;
+                    src[0] += markerLen;
                 }
             }
 
@@ -193,7 +196,7 @@ export function print(
             const src = srcMap.get(unit);
 
             if (src !== undefined) {
-                src[1] += instrumentationMarker.length;
+                src[1] += markerLen;
             }
         }
     }

diff --git a/src/bin/scribble.ts b/src/bin/scribble.ts
@@ -10,6 +10,7 @@ import {
     CompilerKind,
     ContractDefinition,
     ContractKind,
+    FileMap,
     FunctionDefinition,
     FunctionKind,
     FunctionStateMutability,
@@ -30,7 +31,8 @@ import {
     downloadSupportedCompilers,
     getCompilerPrefixForOs,
     isVisiblityExternallyCallable,
-    parsePathRemapping
+    parsePathRemapping,
+    toUTF8
 } from "solc-typed-ast";
 import { rewriteImports } from "../ast_to_source_printer";
 import {
@@ -103,10 +105,12 @@ function error(msg: string): never {
 function getSrcLine(l: Range | Location): string {
     const startLoc = "start" in l ? l.start : l;
     const lineStart = startLoc.offset - startLoc.column;
-    let lineEnd = startLoc.file.contents.indexOf("\n", lineStart);
-    lineEnd = lineEnd == -1 ? startLoc.file.contents.length : lineEnd;
 
-    return startLoc.file.contents.slice(lineStart, lineEnd);
+    let lineEnd = startLoc.file.rawContents.indexOf("\n".charCodeAt(0), lineStart);
+
+    lineEnd = lineEnd == -1 ? startLoc.file.rawContents.length : lineEnd;
+
+    return toUTF8(startLoc.file.rawContents.slice(lineStart, lineEnd));
 }
 
 /// TODO: Eventually make this support underlining a range spanning multiple liens
@@ -303,7 +307,7 @@ function detectMacroDefinitions(
     const fileNames = searchRecursive(path, (fileName) => fileName.endsWith(".scribble.yaml"));
 
     for (const fileName of fileNames) {
-        const data = fse.readFileSync(fileName, { encoding: "utf-8" });
+        const data = fse.readFileSync(fileName);
         const macroFile = new MacroFile(fileName, data);
 
         sources.set(fileName, macroFile);
@@ -778,7 +782,7 @@ function loadInstrMetaData(fileName: string): InstrumentationMetaData {
         let units: SourceUnit[];
         let astCtx: ASTContext;
         let compilerVersionUsed: string;
-        let files: Map<string, string>;
+        let files: FileMap;
         let resolvedFilesMap: Map<string, string>;
 
         /**
@@ -895,7 +899,7 @@ function loadInstrMetaData(fileName: string): InstrumentationMetaData {
 
                 contentsMap.set(
                     unit.absolutePath,
-                    new SolFile(resolvedPath, files.get(unit.sourceEntryKey) as string)
+                    new SolFile(resolvedPath, files.get(unit.sourceEntryKey) as Uint8Array)
                 );
             }
         }

diff --git a/src/instrumenter/annotations.ts b/src/instrumenter/annotations.ts
@@ -10,6 +10,7 @@ import {
     Statement,
     StatementWithChildren,
     StructuredDocumentation,
+    toUTF8,
     TryCatchClause,
     VariableDeclaration
 } from "solc-typed-ast";
@@ -28,7 +29,14 @@ import {
 } from "../spec-lang/ast";
 import { PeggySyntaxError as ExprPEGSSyntaxError, parseAnnotation } from "../spec-lang/expr_parser";
 import { PPAbleError } from "../util/errors";
-import { makeRange, Range, rangeToLocRange } from "../util/location";
+import {
+    adjustRange,
+    makeIdxToOffMap,
+    makeRange,
+    Range,
+    rangeToLocRange,
+    strUTF16IndexToUTF8Offset
+} from "../util";
 import { getOr, getScopeUnit, zip } from "../util/misc";
 import { SourceFile, SourceMap } from "../util/sources";
 
@@ -101,7 +109,7 @@ export class AnnotationMetaData<T extends SAnnotation = SAnnotation> {
                 : target.name;
 
         this.original = parsedAnnot.getSourceFragment(
-            definitionSource ? definitionSource.contents : source.contents
+            definitionSource ? definitionSource.rawContents : source.rawContents
         );
 
         this.id = numAnnotations++;
@@ -215,6 +223,7 @@ function makeAnnotationFromMatch(
     let matchIdx = match.index;
     while (meta.text[matchIdx].match(/[\n\r]/)) matchIdx++;
 
+    const matchUTF8Offset = strUTF16IndexToUTF8Offset(meta.text, matchIdx);
     const slice = meta.text.slice(matchIdx);
 
     let annotation: SAnnotation;
@@ -225,12 +234,16 @@ function makeAnnotationFromMatch(
             meta.target,
             ctx.inference,
             source,
-            meta.docFileOffset + matchIdx
+            meta.docFileOffset + matchUTF8Offset
         );
     } catch (e) {
         if (e instanceof ExprPEGSSyntaxError) {
             // Compute the syntax error offset relative to the start of the file
-            const errStartOff = e.location.start.offset + meta.docFileOffset + matchIdx;
+            const errStartOff =
+                meta.docFileOffset +
+                matchUTF8Offset +
+                strUTF16IndexToUTF8Offset(slice, e.location.start.offset);
+
             const errLength = e.location.end.offset - e.location.start.offset;
 
             const errRange = rangeToLocRange(errStartOff, errLength, source);
@@ -397,7 +410,7 @@ function findAnnotations(
     const meta: RawMetaData = {
         target: target,
         node: raw,
-        text: raw.extractSourceFragment(source.contents),
+        text: toUTF8(raw.extractSourceFragment(source.rawContents)),
         docFileOffset: sourceInfo.offset
     };
 
@@ -654,18 +667,17 @@ function processMacroAnnotations(
                                 offset
                             );
 
-                            throw new SyntaxError(
-                                e.message,
-                                expression,
-                                /// TODO: Remove or fix
-                                makeRange(e.location, {
-                                    file: meta.definitionFile,
-                                    baseOff: offset,
-                                    baseLine: line - 1,
-                                    baseCol: column
-                                }),
-                                target
-                            );
+                            const locOpts = {
+                                file: meta.definitionFile,
+                                baseOff: offset,
+                                baseLine: line - 1,
+                                baseCol: column,
+                                idxToOffMap: makeIdxToOffMap(expression)
+                            };
+                            const range = makeRange(e.location);
+                            adjustRange(range, locOpts);
+
+                            throw new SyntaxError(e.message, expression, range, target);
                         }
 
                         throw e;

diff --git a/src/instrumenter/deprecated_warnings.ts b/src/instrumenter/deprecated_warnings.ts
@@ -8,6 +8,7 @@ import {
     Statement,
     StatementWithChildren,
     StructuredDocumentation,
+    toUTF8,
     VariableDeclaration
 } from "solc-typed-ast";
 import { SAnnotation } from "../spec-lang/ast/declarations/annotation";
@@ -34,8 +35,10 @@ function findAnnotationsInStr(
     fixer: (match: RegExpExecArray, text: string) => string
 ): Array<Range | Location> {
     const res: Array<Range | Location> = [];
-    const text = doc.extractSourceFragment(file.contents);
+    const text = toUTF8(doc.extractSourceFragment(file.rawContents));
+
     let match = rx.exec(text);
+
     const nodeOff = doc.sourceInfo.offset;
 
     while (match !== null) {

diff --git a/src/instrumenter/instrument.ts b/src/instrumenter/instrument.ts
@@ -47,6 +47,7 @@ import {
     PPAbleError,
     SourceMap,
     filterByType,
+    isASCII,
     isChangingState,
     isExternallyVisible,
     parseSrcTriple,
@@ -359,8 +360,9 @@ function emitAssert(
     let userAssertionHit: Statement | undefined;
 
     const messageStr = `000000:0000:000 ${annotation.id}: ${annotation.message}`;
-    const messageLit = factory.makeLiteral("<missing>", LiteralKind.String, "", messageStr);
-    const hitLit = factory.makeLiteral("<missing>", LiteralKind.String, "", `HIT: ${messageStr}`);
+    const litKind = isASCII(annotation.message) ? LiteralKind.String : LiteralKind.UnicodeString;
+    const messageLit = factory.makeLiteral("<missing>", litKind, "", messageStr);
+    const hitLit = factory.makeLiteral("<missing>", litKind, "", `HIT: ${messageStr}`);
 
     if (instrCtx.assertionMode === "log") {
         const event = gt(instrCtx.compilerVersion, "0.6.2")

diff --git a/src/instrumenter/instrumentation_context.ts b/src/instrumenter/instrumentation_context.ts
@@ -7,6 +7,7 @@ import {
     EnumDefinition,
     EventDefinition,
     Expression,
+    fromUTF8,
     FunctionCallKind,
     FunctionDefinition,
     ImportDirective,
@@ -20,6 +21,7 @@ import {
     SrcRangeMap,
     Statement,
     StructDefinition,
+    toUTF8,
     TypeNode,
     VariableDeclaration
 } from "solc-typed-ast";
@@ -54,6 +56,8 @@ import { generateUtilsLibrary, makeEqBytesFun } from "./utils_library";
 
 export type AssertionMode = "log" | "mstore" | "hardhat";
 
+const utf8enc = new TextEncoder();
+
 /**
  * Gather all named nodes in the provided source units.
  * @param units list of source units
@@ -500,6 +504,7 @@ export class InstrumentationContext {
         srcMap: SrcRangeMap
     ): string {
         const replaceMarker = "000000:0000:000";
+        const byteContents = fromUTF8(contents);
 
         for (const [lit, targetNode] of this.litAdjustMap) {
             if (lit.getClosestParentByType(SourceUnit) !== unit) {
@@ -515,10 +520,10 @@ export class InstrumentationContext {
             const startAt = strLoc[0];
             const endAt = strLoc[0] + strLoc[1];
 
-            let litStr = contents.slice(startAt, endAt);
+            let litStr = toUTF8(byteContents.slice(startAt, endAt));
 
             assert(
-                litStr.startsWith("'") || litStr.startsWith('"'),
+                litStr.startsWith("'") || litStr.startsWith('"') || litStr.startsWith("unicode"),
                 `Expected {0} (at {1}) to have a string start at the beginning`,
                 litStr,
                 startAt
@@ -538,10 +543,10 @@ export class InstrumentationContext {
 
             litStr = litStr.replace(replaceMarker, newLocStr);
 
-            contents = contents.slice(0, startAt) + litStr + contents.slice(endAt);
+            utf8enc.encodeInto(litStr, byteContents.subarray(startAt));
         }
 
-        return contents;
+        return toUTF8(byteContents);
     }
 
     needsImport(unit: SourceUnit, importPath: string): void {

diff --git a/src/spec-lang/ast/declarations/user_function_definition.ts b/src/spec-lang/ast/declarations/user_function_definition.ts
@@ -25,7 +25,7 @@ export class SUserFunctionDefinition extends SAnnotation {
     }
 
     getFields(): any[] {
-        return [this.name, ...this.parameters, this.returnType, this.body];
+        return [this.name, ...this.parameters.map((x) => x[0]), this.returnType, this.body];
     }
 
     pp(): string {