Support unreleased solc-typed-ast where Solidity sources are represen…

…ted by uint8 arrays

package.json

@@ -19,7 +19,7 @@
         "fs-extra": "^11.1.1",
         "logplease": "^1.2.15",
         "semver": "^7.5.4",
-        "solc-typed-ast": "^17.0.2",
+        "solc-typed-ast": "../solc-typed-ast",
         "src-location": "^1.1.0",
         "yaml": "^1.10.2"
     },

src/ast_to_source_printer.ts

@@ -1,5 +1,4 @@
 import {
-    assert,
     ASTNodeFactory,
     ASTWriter,
     DefaultASTWriterMapping,
@@ -8,7 +7,9 @@ import {
     PrettyFormatter,
     SourceUnit,
     SrcRangeMap,
-    SymbolAlias
+    SymbolAlias,
+    assert,
+    toUTF8
 } from "solc-typed-ast";
 import { ImportDirectiveDesc } from "./rewriter/import_directive_header";
 import { parse as parseImportDirective } from "./rewriter/import_directive_parser";
@@ -104,7 +105,7 @@ export function rewriteImports(
         assert(source !== undefined, `Missing source for ${sourceUnit.absolutePath}`);
 
         const importDirSrc = importDir.extractSourceFragment(source.contents);
-        const importDesc: ImportDirectiveDesc = parseImportDirective(importDirSrc);
+        const importDesc: ImportDirectiveDesc = parseImportDirective(toUTF8(importDirSrc));
 
         assert(
             importDesc.symbolAliases.length === importDir.symbolAliases.length,

src/bin/scribble.ts

@@ -10,6 +10,7 @@ import {
     CompilerKind,
     ContractDefinition,
     ContractKind,
+    FileMap,
     FunctionDefinition,
     FunctionKind,
     FunctionStateMutability,
@@ -30,7 +31,8 @@ import {
     downloadSupportedCompilers,
     getCompilerPrefixForOs,
     isVisiblityExternallyCallable,
-    parsePathRemapping
+    parsePathRemapping,
+    toUTF8
 } from "solc-typed-ast";
 import { rewriteImports } from "../ast_to_source_printer";
 import {
@@ -103,10 +105,12 @@ function error(msg: string): never {
 function getSrcLine(l: Range | Location): string {
     const startLoc = "start" in l ? l.start : l;
     const lineStart = startLoc.offset - startLoc.column;
-    let lineEnd = startLoc.file.contents.indexOf("\n", lineStart);
+
+    let lineEnd = startLoc.file.contents.indexOf("\n".charCodeAt(0), lineStart);
+
     lineEnd = lineEnd == -1 ? startLoc.file.contents.length : lineEnd;
 
-    return startLoc.file.contents.slice(lineStart, lineEnd);
+    return toUTF8(startLoc.file.contents.slice(lineStart, lineEnd));
 }
 
 /// TODO: Eventually make this support underlining a range spanning multiple liens
@@ -303,7 +307,7 @@ function detectMacroDefinitions(
     const fileNames = searchRecursive(path, (fileName) => fileName.endsWith(".scribble.yaml"));
 
     for (const fileName of fileNames) {
-        const data = fse.readFileSync(fileName, { encoding: "utf-8" });
+        const data = fse.readFileSync(fileName);
         const macroFile = new MacroFile(fileName, data);
 
         sources.set(fileName, macroFile);
@@ -778,7 +782,7 @@ function loadInstrMetaData(fileName: string): InstrumentationMetaData {
         let units: SourceUnit[];
         let astCtx: ASTContext;
         let compilerVersionUsed: string;
-        let files: Map<string, string>;
+        let files: FileMap;
         let resolvedFilesMap: Map<string, string>;
 
         /**
@@ -895,7 +899,7 @@ function loadInstrMetaData(fileName: string): InstrumentationMetaData {
 
                 contentsMap.set(
                     unit.absolutePath,
-                    new SolFile(resolvedPath, files.get(unit.sourceEntryKey) as string)
+                    new SolFile(resolvedPath, files.get(unit.sourceEntryKey) as Uint8Array)
                 );
             }
         }

src/instrumenter/annotations.ts

@@ -10,6 +10,7 @@ import {
     Statement,
     StatementWithChildren,
     StructuredDocumentation,
+    toUTF8,
     TryCatchClause,
     VariableDeclaration
 } from "solc-typed-ast";
@@ -101,7 +102,7 @@ export class AnnotationMetaData<T extends SAnnotation = SAnnotation> {
                 : target.name;
 
         this.original = parsedAnnot.getSourceFragment(
-            definitionSource ? definitionSource.contents : source.contents
+            toUTF8(definitionSource ? definitionSource.contents : source.contents)
         );
 
         this.id = numAnnotations++;
@@ -397,7 +398,7 @@ function findAnnotations(
     const meta: RawMetaData = {
         target: target,
         node: raw,
-        text: raw.extractSourceFragment(source.contents),
+        text: toUTF8(raw.extractSourceFragment(source.contents)),
         docFileOffset: sourceInfo.offset
     };
 

src/instrumenter/deprecated_warnings.ts

@@ -8,6 +8,7 @@ import {
     Statement,
     StatementWithChildren,
     StructuredDocumentation,
+    toUTF8,
     VariableDeclaration
 } from "solc-typed-ast";
 import { SAnnotation } from "../spec-lang/ast/declarations/annotation";
@@ -34,8 +35,10 @@ function findAnnotationsInStr(
     fixer: (match: RegExpExecArray, text: string) => string
 ): Array<Range | Location> {
     const res: Array<Range | Location> = [];
-    const text = doc.extractSourceFragment(file.contents);
+    const text = toUTF8(doc.extractSourceFragment(file.contents));
+
     let match = rx.exec(text);
+
     const nodeOff = doc.sourceInfo.offset;
 
     while (match !== null) {

src/macros/utils.ts

@@ -2,6 +2,7 @@ import YAML from "yaml";
 import { Scalar, YAMLMap, YAMLSeq } from "yaml/types";
 import { SourceFile } from "../util/sources";
 import { checkYamlSchema, makeYamlRange, YamlSchemaError } from "../util/yaml";
+import { toUTF8 } from "solc-typed-ast";
 
 export interface MacroVariable {
     name: string;
@@ -37,7 +38,7 @@ const yamlMacroSchema = {
 };
 
 export function readMacroDefinitions(source: SourceFile, defs: Map<string, MacroDefinition>): void {
-    const document = YAML.parseDocument(source.contents);
+    const document = YAML.parseDocument(toUTF8(source.contents));
 
     if (document.contents === null) {
         throw new YamlSchemaError(

src/util/sources.ts

@@ -1,7 +1,7 @@
 export abstract class SourceFile {
     constructor(
         public fileName: string,
-        public contents: string
+        public contents: Uint8Array
     ) {}
 }
 
@@ -10,13 +10,13 @@ export class MacroFile extends SourceFile {}
 
 export class UtilsSolFile extends SourceFile {
     constructor(fileName: string) {
-        super(fileName, "");
+        super(fileName, new Uint8Array());
     }
 }
 
 export class DummySourceFile extends SourceFile {
     constructor() {
-        super("", "");
+        super("", new Uint8Array());
     }
 }
 

test/integration/src2srcmap.spec.ts

@@ -26,7 +26,10 @@ import {
     TupleExpression,
     TypeName,
     UnaryOperation,
-    VariableDeclaration
+    VariableDeclaration,
+    FileMap,
+    fromUTF8,
+    toUTF8
 } from "solc-typed-ast";
 import {
     contains,
@@ -83,10 +86,10 @@ function buildSrc2NodeMap(units: SourceUnit[], newSrcList?: string[]): Src2NodeM
     return res;
 }
 
-function fragment(src: string, contents: string) {
+function fragment(src: string, contents: Uint8Array) {
     const [off, len] = parseSrcTriple(src);
 
-    return contents.slice(off, off + len);
+    return toUTF8(contents.slice(off, off + len));
 }
 
 describe("Src2src map test", () => {
@@ -107,8 +110,8 @@ describe("Src2src map test", () => {
     for (const sample of samples) {
         describe(sample, () => {
             let inAst: SourceUnit[];
-            let contents: string;
-            let instrContents: string;
+            let contents: Uint8Array;
+            let instrContents: Uint8Array;
             let outJSON: any;
             let propMap: PropertyMap;
             let outAST: SourceUnit;
@@ -125,7 +128,7 @@ describe("Src2src map test", () => {
                 }
 
                 inAst = result.units;
-                contents = result.files.get(sample) as string;
+                contents = result.files.get(sample) as Uint8Array;
 
                 const args = [sample, "--output-mode", "json"];
 
@@ -134,9 +137,9 @@ describe("Src2src map test", () => {
                 }
                 outJSON = JSON.parse(scrSample(args[0], ...args.slice(1)));
 
-                instrContents = outJSON["sources"]["flattened.sol"]["source"];
+                instrContents = fromUTF8(outJSON["sources"]["flattened.sol"]["source"]);
 
-                const contentsMap = new Map<string, string>([["flattened.sol", instrContents]]);
+                const contentsMap: FileMap = new Map([["flattened.sol", instrContents]]);
                 const reader = new ASTReader();
 
                 [outAST] = reader.read(outJSON, ASTKind.Modern, contentsMap);

test/integration/utils.ts

@@ -16,6 +16,7 @@ import {
     PossibleCompilerKinds,
     SourceUnit,
     Statement,
+    toUTF8,
     VariableDeclaration,
     XPath
 } from "solc-typed-ast";
@@ -70,7 +71,7 @@ export function makeArtefact(result: CompileResult): string {
             delete entry.legacyAST;
         }
 
-        artefact.sources[removeProcWd(name)] = { ...entry, source };
+        artefact.sources[removeProcWd(name)] = { ...entry, source: toUTF8(source) };
     }
 
     return removeProcWd(JSON.stringify(artefact));

test/unit/custom_maps.spec.ts

@@ -1177,13 +1177,7 @@ describe("Interposing on a map", () => {
             version = res.compilerVersion;
 
             const factory = new ScribbleFactory(version, ctx);
-            const instrCtx = makeInstrumentationCtx(
-                [unit],
-                factory,
-                new Map([[name, sample]]),
-                "log",
-                version
-            );
+            const instrCtx = makeInstrumentationCtx([unit], factory, res.files, "log", version);
 
             const writer: ASTWriter = new ASTWriter(
                 DefaultASTWriterMapping,

test/unit/interposing.spec.ts

@@ -4,6 +4,7 @@ import {
     ASTContext,
     ASTNodeFactory,
     ContractDefinition,
+    fromUTF8,
     FunctionDefinition,
     SourceUnit,
     TupleExpression,
@@ -28,7 +29,10 @@ import { makeInstrumentationCtx } from "./utils";
 
 function print(units: SourceUnit[], contents: string[], version: string): Map<SourceUnit, string> {
     const contentMap = new Map(
-        units.map((unit, idx) => [unit.absolutePath, new SolFile(unit.absolutePath, contents[idx])])
+        units.map((unit, idx) => [
+            unit.absolutePath,
+            new SolFile(unit.absolutePath, fromUTF8(contents[idx]))
+        ])
     );
     const context = new ASTContext(...units);
     const factory = new ASTNodeFactory(context);

test/unit/sc.spec.ts

@@ -3,6 +3,7 @@ import {
     BoolType,
     ContractDefinition,
     eq,
+    fromUTF8,
     FunctionDefinition,
     InferType,
     IntLiteralType,
@@ -370,7 +371,7 @@ describe("SemanticChecker Expression Unit Tests", () => {
 
                 units = result.units;
                 inference = new InferType(compilerVersion);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc, expectedType, expectedInfo] of testCases) {
@@ -418,7 +419,7 @@ describe("SemanticChecker Expression Unit Tests", () => {
 
                 units = result.units;
                 inference = new InferType(compilerVersion);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc] of testCases) {
@@ -531,7 +532,7 @@ describe("SemanticChecker Annotation Unit Tests", () => {
 
                 units = result.units;
                 inference = new InferType(compilerVersion);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc] of testCases) {
@@ -573,7 +574,7 @@ describe("SemanticChecker Annotation Unit Tests", () => {
 
                 units = result.units;
                 inference = new InferType(compilerVersion);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc] of testCases) {

test/unit/tc.spec.ts

@@ -11,6 +11,7 @@ import {
     EnumDefinition,
     eq,
     FixedBytesType,
+    fromUTF8,
     FunctionStateMutability,
     FunctionType,
     FunctionVisibility,
@@ -987,7 +988,7 @@ contract UserDefinedValueTypes {
                 compilerVersion = result.compilerVersion;
 
                 inference = new InferType(compilerVersion);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc, expected] of testCases) {
@@ -1024,7 +1025,7 @@ contract UserDefinedValueTypes {
                 const inference = new InferType(compilerVersion);
 
                 typeEnv = new TypeEnv(inference);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc] of testCases) {
@@ -1693,7 +1694,7 @@ contract Statements08 {
 
                 inference = new InferType(compilerVersion);
                 typeEnv = new TypeEnv(inference);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
             });
 
             for (const [specString, loc, expectedType, clearFunsBefore] of testCases) {
@@ -1743,7 +1744,7 @@ contract Statements08 {
                 const inference = new InferType(compilerVersion);
 
                 typeEnv = new TypeEnv(inference);
-                sourceFile = new SolFile(fileName, content);
+                sourceFile = new SolFile(fileName, fromUTF8(content));
 
                 // Setup any definitions
                 for (const [specString, loc] of setupSteps) {

test/unit/utils.ts

@@ -12,7 +12,7 @@ import { SolFile } from "../../src/util/sources";
 export function makeInstrumentationCtx(
     sources: SourceUnit[],
     factory: ScribbleFactory,
-    files: Map<string, string>,
+    files: Map<string, Uint8Array>,
     assertionMode: AssertionMode,
     compilerVersion: string
 ): InstrumentationContext {