From 81eb3cb81c127e22d93fa96f6bf4d4c64d2475e2 Mon Sep 17 00:00:00 2001 From: Breck Yunits Date: Fri, 29 Nov 2024 05:42:58 -1000 Subject: [PATCH] Implement Fusion async file system (#193) --- fusion/Fusion.test.ts | 57 +++-- fusion/Fusion.ts | 376 +++++++++++++++++++++++---------- package.json | 2 +- particle/Particle.ts | 2 +- products.scroll | 2 +- products/Fusion.browser.js | 328 +++++++++++++++++++--------- products/Fusion.js | 331 ++++++++++++++++++++--------- products/Particle.browser.js | 2 +- products/Particle.js | 2 +- products/SandboxApp.browser.js | 9 +- releaseNotes.scroll | 6 + sandbox/SandboxApp.ts | 11 +- 12 files changed, 793 insertions(+), 335 deletions(-) diff --git a/fusion/Fusion.test.ts b/fusion/Fusion.test.ts index 9d94c3b6f..0853b969d 100755 --- a/fusion/Fusion.test.ts +++ b/fusion/Fusion.test.ts @@ -1,17 +1,17 @@ #!/usr/bin/env ts-node - const { Particle } = require("../products/Particle.js") -const { Fusion } = require("../products/Fusion.js") +const { Fusion, FusionFile } = require("../products/Fusion.js") const { TestRacer } = require("../products/TestRacer.js") const path = require("path") import { particlesTypes } from "../products/particlesTypes" const testParticles: particlesTypes.testParticles = {} -testParticles.disk = equal => { +testParticles.disk = async equal => { const tfs = new Fusion() // Arrange/Act/Assert - equal(tfs.fuseFile(path.join(__dirname, "..", "readme.scroll")).fused.length > 0, true) + const result = await tfs.fuseFile(path.join(__dirname, "..", "readme.scroll")) + equal(result.fused.length > 0, true) } const stripImported = (str: string) => { @@ -20,7 +20,7 @@ const stripImported = (str: string) => { return particle.toString() } -testParticles.inMemory = equal => { +testParticles.inMemory = async equal => { // Arrange/Act/Assert const files = { "/hello": "world", @@ -30,29 +30,30 @@ testParticles.inMemory = equal => { } const tfs = new Fusion(files) equal(tfs.dirname("/"), "/") - equal(stripImported(tfs.fuseFile("/main").fused), "world\nciao") - equal(stripImported(tfs.fuseFile("/nested/deep/relative").fused), "world\nciao") - equal(tfs.fuseFile("/main").exists, true) + const mainResult = await tfs.fuseFile("/main") + equal(stripImported(mainResult.fused), "world\nciao") + + const relativeResult = await tfs.fuseFile("/nested/deep/relative") + equal(stripImported(relativeResult.fused), "world\nciao") + equal(mainResult.exists, true) } -testParticles.nonExistant = equal => { +testParticles.nonExistant = async equal => { // Arrange/Act/Assert const files = { "/main": "import env" } const tfs = new Fusion(files) - const result = tfs.fuseFile("/main") + const result = await tfs.fuseFile("/main") equal(stripImported(result.fused), "") equal(result.exists, false) } -testParticles.footers = equal => { +testParticles.footers = async equal => { // Arrange/Act/Assert const files = { "/hello.scroll": `headerAndFooter.scroll - title Hello world - This is my content `, "/headerAndFooter.scroll": "header.scroll\nfooter.scroll\n footer", @@ -60,13 +61,13 @@ This is my content "/footer.scroll": "The end." } const tfs = new Fusion(files) - const result = tfs.fuseFile("/hello.scroll") + const result = await tfs.fuseFile("/hello.scroll") equal(result.fused.includes("This is my content"), true) equal(result.fused.includes("The end"), false) equal(result.footers[0], "The end.") } -testParticles.quickImports = equal => { +testParticles.quickImports = async equal => { // Arrange/Act/Assert const files = { "/hello.scroll": "world", @@ -77,11 +78,29 @@ testParticles.quickImports = equal => { } const tfs = new Fusion(files) equal(tfs.dirname("/"), "/") - equal(stripImported(tfs.fuseFile("/nested/a").fused), "ciao") - equal(stripImported(tfs.fuseFile("/main").fused), "world\nciao") - equal(stripImported(tfs.fuseFile("/nested/deep/relative").fused), "world\nciao") + + const [aResult, mainResult, relativeResult] = await Promise.all([tfs.fuseFile("/nested/a"), tfs.fuseFile("/main"), tfs.fuseFile("/nested/deep/relative")]) + + equal(stripImported(aResult.fused), "ciao") + equal(stripImported(mainResult.fused), "world\nciao") + equal(stripImported(relativeResult.fused), "world\nciao") + + // FileAPI + // Arrange + const file = new FusionFile(files["/main"], "/main", tfs) + equal(file.fusedCode, undefined) + // Act + await file.fuse() + // Assert + equal(stripImported(file.fusedCode), "world\nciao") } -/*NODE_JS_ONLY*/ if (!module.parent) TestRacer.testSingleFile(__filename, testParticles) +/*NODE_JS_ONLY*/ if (!module.parent) { + // Update TestRacer to handle async tests + const runTests = async () => { + await TestRacer.testSingleFile(__filename, testParticles) + } + runTests() +} export { testParticles } diff --git a/fusion/Fusion.ts b/fusion/Fusion.ts index 0b64fc4c6..d03288b37 100644 --- a/fusion/Fusion.ts +++ b/fusion/Fusion.ts @@ -1,4 +1,4 @@ -const fs = require("fs") +const fs = require("fs").promises // Change to use promises version const path = require("path") import { particlesTypes } from "../products/particlesTypes" @@ -29,12 +29,12 @@ interface FusedFile { } interface Storage { - read(absolutePath: string): string - exists(absolutePath: string): boolean - list(absolutePath: string): string[] - write(absolutePath: string, content: string): void - getMTime(absolutePath: string): number - getCTime(absolutePath: string): number + read(absolutePath: string): Promise + exists(absolutePath: string): Promise + list(absolutePath: string): Promise + write(absolutePath: string, content: string): Promise + getMTime(absolutePath: string): Promise + getCTime(absolutePath: string): Promise dirname(absolutePath: string): string join(...absolutePath: string[]): string } @@ -46,38 +46,50 @@ const importOnlyRegex = /^importOnly/ class DiskWriter implements Storage { fileCache: { [filepath: string]: OpenedFile } = {} - _read(absolutePath: particlesTypes.filepath) { + + async _read(absolutePath: particlesTypes.filepath) { const { fileCache } = this if (!fileCache[absolutePath]) { - const exists = fs.existsSync(absolutePath) - if (exists) fileCache[absolutePath] = { absolutePath, exists: true, content: Disk.read(absolutePath).replace(/\r/g, ""), stats: fs.statSync(absolutePath) } - else fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + const exists = await fs + .access(absolutePath) + .then(() => true) + .catch(() => false) + if (exists) { + const [content, stats] = await Promise.all([fs.readFile(absolutePath, "utf8").then(content => content.replace(/\r/g, "")), fs.stat(absolutePath)]) + fileCache[absolutePath] = { absolutePath, exists: true, content, stats } + } else { + fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + } } return fileCache[absolutePath] } - exists(absolutePath: string) { - return this._read(absolutePath).exists + async exists(absolutePath: string) { + const file = await this._read(absolutePath) + return file.exists } - read(absolutePath: string) { - return this._read(absolutePath).content + async read(absolutePath: string) { + const file = await this._read(absolutePath) + return file.content } - list(folder: string) { + async list(folder: string) { return Disk.getFiles(folder) } - write(fullPath: string, content: string) { + async write(fullPath: string, content: string) { Disk.writeIfChanged(fullPath, content) } - getMTime(absolutePath: string) { - return this._read(absolutePath).stats.mtimeMs + async getMTime(absolutePath: string) { + const file = await this._read(absolutePath) + return file.stats.mtimeMs } - getCTime(absolutePath: string) { - return this._read(absolutePath).stats.ctimeMs + async getCTime(absolutePath: string) { + const file = await this._read(absolutePath) + return file.stats.ctimeMs } dirname(absolutePath: string) { @@ -96,7 +108,7 @@ class MemoryWriter implements Storage { inMemoryFiles: particlesTypes.diskMap - read(absolutePath: particlesTypes.filepath) { + async read(absolutePath: particlesTypes.filepath) { const value = this.inMemoryFiles[absolutePath] if (value === undefined) { return "" @@ -104,23 +116,23 @@ class MemoryWriter implements Storage { return value } - exists(absolutePath: string) { + async exists(absolutePath: string) { return this.inMemoryFiles[absolutePath] !== undefined } - write(absolutePath: particlesTypes.filepath, content: string) { + async write(absolutePath: particlesTypes.filepath, content: string) { this.inMemoryFiles[absolutePath] = content } - list(absolutePath: particlesTypes.filepath) { + async list(absolutePath: particlesTypes.filepath) { return Object.keys(this.inMemoryFiles).filter(filePath => filePath.startsWith(absolutePath) && !filePath.replace(absolutePath, "").includes("/")) } - getMTime() { + async getMTime() { return 1 } - getCTime() { + async getCTime() { return 1 } @@ -133,26 +145,90 @@ class MemoryWriter implements Storage { } } +class FusionFile { + constructor(codeAtStart: string, absoluteFilePath = "", fileSystem = new Fusion({})) { + this.fileSystem = fileSystem + this.filePath = absoluteFilePath + this.filename = posix.basename(absoluteFilePath) + this.folderPath = posix.dirname(absoluteFilePath) + "/" + this.codeAtStart = codeAtStart + this.timeIndex = 0 + this.timestamp = 0 + this.importOnly = false + } + + async readCodeFromStorage() { + if (this.codeAtStart !== undefined) return this // Code provided + const { filePath } = this + if (!filePath) { + this.codeAtStart = "" + return this + } + this.codeAtStart = await this.fileSystem.read(filePath) + } + + get isFused() { + return this.fusedCode !== undefined + } + + async fuse() { + // PASS 1: READ FULL FILE + await this.readCodeFromStorage() + const { codeAtStart, fileSystem, filePath, defaultParserCode } = this + // PASS 2: READ AND REPLACE IMPORTs + let fusedCode = codeAtStart + if (filePath) { + this.timestamp = await fileSystem.getCTime(filePath) + const fusedFile = await fileSystem.fuseFile(filePath, defaultParserCode) + this.importOnly = fusedFile.isImportOnly + fusedCode = fusedFile.fused + if (fusedFile.footers.length) fusedCode += "\n" + fusedFile.footers.join("\n") + this.dependencies = fusedFile.importFilePaths + this.fusedFile = fusedFile + } + this.fusedCode = fusedCode + this.parseCode() + return this + } + + parseCode() {} + + get formatted() { + return this.codeAtStart + } + + async formatAndSave() { + const { codeAtStart, formatted } = this + if (codeAtStart === formatted) return false + await this.fileSystem.write(this.filePath, formatted) + return true + } + + defaultParserCode = "" +} +let fusionIdNumber = 0 class Fusion implements Storage { constructor(inMemoryFiles: particlesTypes.diskMap) { if (inMemoryFiles) this._storage = new MemoryWriter(inMemoryFiles) else this._storage = new DiskWriter() + fusionIdNumber = fusionIdNumber + 1 + this.fusionId = fusionIdNumber } - read(absolutePath: particlesTypes.filepath) { - return this._storage.read(absolutePath) + async read(absolutePath: particlesTypes.filepath) { + return await this._storage.read(absolutePath) } - exists(absolutePath: particlesTypes.filepath) { - return this._storage.exists(absolutePath) + async exists(absolutePath: particlesTypes.filepath) { + return await this._storage.exists(absolutePath) } - write(absolutePath: particlesTypes.filepath, content: string) { - return this._storage.write(absolutePath, content) + async write(absolutePath: particlesTypes.filepath, content: string) { + return await this._storage.write(absolutePath, content) } - list(absolutePath: particlesTypes.filepath) { - return this._storage.list(absolutePath) + async list(absolutePath: particlesTypes.filepath) { + return await this._storage.list(absolutePath) } dirname(absolutePath: string) { @@ -163,12 +239,18 @@ class Fusion implements Storage { return this._storage.join(...segments) } - getMTime(absolutePath: string) { - return this._storage.getMTime(absolutePath) + async getMTime(absolutePath: string) { + return await this._storage.getMTime(absolutePath) } - getCTime(absolutePath: string) { - return this._storage.getMTime(absolutePath) + async getCTime(absolutePath: string) { + return await this._storage.getCTime(absolutePath) + } + + productCache = {} + async writeProduct(absolutePath, content) { + this.productCache[absolutePath] = content + return await this.write(absolutePath, content) } private _storage: Storage @@ -177,20 +259,20 @@ class Fusion implements Storage { private _expandedImportCache: { [filepath: string]: FusedFile } = {} private _parsersExpandersCache: { [filepath: string]: boolean } = {} - private _getFileAsParticles(absoluteFilePath: string) { + private async _getFileAsParticles(absoluteFilePath: string) { const { _particleCache } = this if (_particleCache[absoluteFilePath] === undefined) { - _particleCache[absoluteFilePath] = new Particle(this._storage.read(absoluteFilePath)) + const content = await this._storage.read(absoluteFilePath) + _particleCache[absoluteFilePath] = new Particle(content) } return _particleCache[absoluteFilePath] } - private _fuseFile(absoluteFilePath: string) { + private async _fuseFile(absoluteFilePath: string): Promise { const { _expandedImportCache } = this if (_expandedImportCache[absoluteFilePath]) return _expandedImportCache[absoluteFilePath] - let code = this.read(absoluteFilePath) - const exists = this.exists(absoluteFilePath) + const [code, exists] = await Promise.all([this.read(absoluteFilePath), this.exists(absoluteFilePath)]) const isImportOnly = importOnlyRegex.test(code) @@ -198,116 +280,194 @@ class Fusion implements Storage { // If its a parsers file, it will have no content, just parsers (and maybe imports). // The parsers will already have been processed. We can skip them const stripParsers = absoluteFilePath.endsWith(PARSERS_EXTENSION) - if (stripParsers) - code = code - .split("\n") - .filter(line => importRegex.test(line)) - .join("\n") + const processedCode = stripParsers + ? code + .split("\n") + .filter(line => importRegex.test(line)) + .join("\n") + : code const filepathsWithParserDefinitions = [] - if (this._doesFileHaveParsersDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath) + if (await this._doesFileHaveParsersDefinitions(absoluteFilePath)) { + filepathsWithParserDefinitions.push(absoluteFilePath) + } - if (!importRegex.test(code)) - return { - fused: code, + if (!importRegex.test(processedCode)) { + return { + fused: processedCode, footers: [], isImportOnly, importFilePaths: [], filepathsWithParserDefinitions, exists } + } + + const particle = new Particle(processedCode) + const folder = this.dirname(absoluteFilePath) + + // Fetch all imports in parallel + const importParticles = particle.filter(particle => particle.getLine().match(importRegex)) + const importResults = importParticles.map(async importParticle => { + const relativeFilePath = importParticle.getLine().replace("import ", "") + const absoluteImportFilePath = this.join(folder, relativeFilePath) + + // todo: race conditions + const [expandedFile, exists] = await Promise.all([this._fuseFile(absoluteImportFilePath), this.exists(absoluteImportFilePath)]) + return { + expandedFile, + exists, + relativeFilePath, + absoluteImportFilePath, + importParticle + } + }) + const imported = await Promise.all(importResults) + + // Assemble all imports let importFilePaths: string[] = [] let footers: string[] = [] - const particle = new Particle(code) - const folder = this.dirname(absoluteFilePath) - particle - .filter(particle => particle.getLine().match(importRegex)) - .forEach(importParticle => { - const relativeFilePath = importParticle.getLine().replace("import ", "") - const absoluteImportFilePath = this.join(folder, relativeFilePath) - const expandedFile = this._fuseFile(absoluteImportFilePath) - importFilePaths.push(absoluteImportFilePath) - importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) - const exists = this.exists(absoluteImportFilePath) - importParticle.setLine("imported " + relativeFilePath) - importParticle.set("exists", `${exists}`) - footers = footers.concat(expandedFile.footers) - if (importParticle.has("footer")) footers.push(expandedFile.fused) - else importParticle.insertLinesAfter(expandedFile.fused) - }) + imported.forEach(importResults => { + const { importParticle, absoluteImportFilePath, expandedFile, relativeFilePath, exists } = importResults + importFilePaths.push(absoluteImportFilePath) + importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) + + importParticle.setLine("imported " + relativeFilePath) + importParticle.set("exists", `${exists}`) + + footers = footers.concat(expandedFile.footers) + if (importParticle.has("footer")) footers.push(expandedFile.fused) + else importParticle.insertLinesAfter(expandedFile.fused) + }) + + const existStates = await Promise.all(importFilePaths.map(file => this.exists(file))) + + const allImportsExist = !existStates.some(exists => !exists) _expandedImportCache[absoluteFilePath] = { importFilePaths, isImportOnly, fused: particle.toString(), footers, - exists: !importFilePaths.some(file => !this.exists(file)), - filepathsWithParserDefinitions: importFilePaths.filter((filename: string) => this._doesFileHaveParsersDefinitions(filename)).concat(filepathsWithParserDefinitions) + exists: allImportsExist, + filepathsWithParserDefinitions: ( + await Promise.all( + importFilePaths.map(async filename => ({ + filename, + hasParser: await this._doesFileHaveParsersDefinitions(filename) + })) + ) + ) + .filter(result => result.hasParser) + .map(result => result.filename) + .concat(filepathsWithParserDefinitions) } return _expandedImportCache[absoluteFilePath] } - private _doesFileHaveParsersDefinitions(absoluteFilePath: particlesTypes.filepath) { + private async _doesFileHaveParsersDefinitions(absoluteFilePath: particlesTypes.filepath) { if (!absoluteFilePath) return false const { _parsersExpandersCache } = this - if (_parsersExpandersCache[absoluteFilePath] === undefined) _parsersExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(parserRegex) - + if (_parsersExpandersCache[absoluteFilePath] === undefined) { + const content = await this._storage.read(absoluteFilePath) + _parsersExpandersCache[absoluteFilePath] = !!content.match(parserRegex) + } return _parsersExpandersCache[absoluteFilePath] } - private _getOneParsersParserFromFiles(filePaths: string[], baseParsersCode: string) { - const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ - const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ - const asOneFile = filePaths - .map(filePath => { - const content = this._storage.read(filePath) - if (filePath.endsWith(PARSERS_EXTENSION)) return content - // Strip scroll content - return new Particle(content) - .filter((particle: particlesTypes.particle) => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) - .map((particle: particlesTypes.particle) => particle.asString) - .join("\n") - }) - .join("\n") - .trim() - - // todo: clean up scrollsdk so we are using supported methods (perhaps add a formatOptions that allows you to tell Parsers not to run prettier on js particles) - return new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() - } - - get parsers() { - return Object.values(this._parserCache).map(parser => parser.parsersParser) + private async _getOneParsersParserFromFiles(filePaths: string[], baseParsersCode: string) { + const fileContents = await Promise.all(filePaths.map(async filePath => await this._storage.read(filePath))) + return Fusion.combineParsers(filePaths, fileContents, baseParsersCode) } - getParser(filePaths: string[], baseParsersCode = "") { + async getParser(filePaths: string[], baseParsersCode = "") { const { _parserCache } = this const key = filePaths .filter(fp => fp) .sort() .join("\n") + const hit = _parserCache[key] if (hit) return hit - const parsersParser = this._getOneParsersParserFromFiles(filePaths, baseParsersCode) - const parsersCode = parsersParser.asString - _parserCache[key] = { - parsersParser, + + _parserCache[key] = await this._getOneParsersParserFromFiles(filePaths, baseParsersCode) + return _parserCache[key] + } + + static combineParsers(filePaths: string[], fileContents: string[], baseParsersCode = "") { + const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ + const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ + + const mapped = fileContents.map((content, index) => { + const filePath = filePaths[index] + if (filePath.endsWith(PARSERS_EXTENSION)) return content + + return new Particle(content) + .filter((particle: particlesTypes.particle) => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) + .map((particle: particlesTypes.particle) => particle.asString) + .join("\n") + }) + + const asOneFile = mapped.join("\n").trim() + const sorted = new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() + const parsersCode = sorted.asString + return { + parsersParser: sorted, parsersCode, parser: new HandParsersProgram(parsersCode).compileAndReturnRootParser() } - return _parserCache[key] } - fuseFile(absoluteFilePath: string, defaultParserCode?: string): FusedFile { - const fusedFile = this._fuseFile(absoluteFilePath) + get parsers() { + return Object.values(this._parserCache).map(parser => parser.parsersParser) + } + + async fuseFile(absoluteFilePath: string, defaultParserCode?: string): Promise { + const fusedFile = await this._fuseFile(absoluteFilePath) if (!defaultParserCode) return fusedFile - // BUILD CUSTOM COMPILER, IF THERE ARE CUSTOM PARSERS NODES DEFINED - if (fusedFile.filepathsWithParserDefinitions.length) fusedFile.parser = this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode).parser + if (fusedFile.filepathsWithParserDefinitions.length) { + const parser = await this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode) + fusedFile.parser = parser.parser + } return fusedFile } + + defaultFileClass = FusionFile + async getLoadedFile(filePath) { + return await this._getLoadedFile(filePath, this.defaultFileClass) + } + + parsedFiles = {} + async _getLoadedFile(absolutePath, parser) { + if (this.parsedFiles[absolutePath]) return this.parsedFiles[absolutePath] + const file = new parser(undefined, absolutePath, this) + await file.fuse() + this.parsedFiles[absolutePath] = file + return file + } + + getCachedLoadedFilesInFolder(folderPath, requester) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + const hit = this.folderCache[folderPath] + if (!hit) console.log(`Warning: '${folderPath}' not yet loaded in '${this.fusionId}'. Requested by '${requester.filePath}'`) + return hit || [] + } + + folderCache = {} + async getLoadedFilesInFolder(folderPath, extension) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + if (this.folderCache[folderPath]) return this.folderCache[folderPath] + const allFiles = await this.list(folderPath) + const loadedFiles = await Promise.all(allFiles.filter(file => file.endsWith(extension)).map(filePath => this.getLoadedFile(filePath))) + const sorted = loadedFiles.sort((a, b) => b.timestamp - a.timestamp) + sorted.forEach((file, index) => (file.timeIndex = index)) + this.folderCache[folderPath] = sorted + return this.folderCache[folderPath] + } } -export { Fusion } +export { Fusion, FusionFile } diff --git a/package.json b/package.json index 7483effa9..ac32d4da5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scrollsdk", - "version": "96.0.0", + "version": "97.0.0", "description": "This npm package includes the Particles class, the Parsers compiler-compiler, a Parsers IDE, and more, all implemented in Particles, Parsers, and TypeScript.", "types": "./built/scrollsdk.node.d.ts", "main": "./products/Particle.js", diff --git a/particle/Particle.ts b/particle/Particle.ts index 8e59a4ff8..9771680c0 100644 --- a/particle/Particle.ts +++ b/particle/Particle.ts @@ -3094,7 +3094,7 @@ class Particle extends AbstractParticle { return str ? indent + str.replace(/\n/g, indent) : "" } - static getVersion = () => "96.0.0" + static getVersion = () => "97.0.0" static fromDisk(path: string): Particle { const format = this._getFileFormat(path) diff --git a/products.scroll b/products.scroll index 73bedb529..f76b66c7d 100644 --- a/products.scroll +++ b/products.scroll @@ -32,7 +32,7 @@ nodeProduct combineTypeScriptFiles utils/Utils.ts nodeProduct outputFileName Fusion.js - insertLastLine module.exports = {Fusion} + insertLastLine module.exports = {Fusion, FusionFile} combineTypeScriptFiles fusion/Fusion.ts nodeProduct outputFileName Kitchen.node.js diff --git a/products/Fusion.browser.js b/products/Fusion.browser.js index 44661bcbe..88deaf5cb 100644 --- a/products/Fusion.browser.js +++ b/products/Fusion.browser.js @@ -7,32 +7,43 @@ class DiskWriter { constructor() { this.fileCache = {} } - _read(absolutePath) { + async _read(absolutePath) { const { fileCache } = this if (!fileCache[absolutePath]) { - const exists = fs.existsSync(absolutePath) - if (exists) fileCache[absolutePath] = { absolutePath, exists: true, content: Disk.read(absolutePath).replace(/\r/g, ""), stats: fs.statSync(absolutePath) } - else fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + const exists = await fs + .access(absolutePath) + .then(() => true) + .catch(() => false) + if (exists) { + const [content, stats] = await Promise.all([fs.readFile(absolutePath, "utf8").then(content => content.replace(/\r/g, "")), fs.stat(absolutePath)]) + fileCache[absolutePath] = { absolutePath, exists: true, content, stats } + } else { + fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + } } return fileCache[absolutePath] } - exists(absolutePath) { - return this._read(absolutePath).exists + async exists(absolutePath) { + const file = await this._read(absolutePath) + return file.exists } - read(absolutePath) { - return this._read(absolutePath).content + async read(absolutePath) { + const file = await this._read(absolutePath) + return file.content } - list(folder) { + async list(folder) { return Disk.getFiles(folder) } - write(fullPath, content) { + async write(fullPath, content) { Disk.writeIfChanged(fullPath, content) } - getMTime(absolutePath) { - return this._read(absolutePath).stats.mtimeMs + async getMTime(absolutePath) { + const file = await this._read(absolutePath) + return file.stats.mtimeMs } - getCTime(absolutePath) { - return this._read(absolutePath).stats.ctimeMs + async getCTime(absolutePath) { + const file = await this._read(absolutePath) + return file.stats.ctimeMs } dirname(absolutePath) { return path.dirname(absolutePath) @@ -45,26 +56,26 @@ class MemoryWriter { constructor(inMemoryFiles) { this.inMemoryFiles = inMemoryFiles } - read(absolutePath) { + async read(absolutePath) { const value = this.inMemoryFiles[absolutePath] if (value === undefined) { return "" } return value } - exists(absolutePath) { + async exists(absolutePath) { return this.inMemoryFiles[absolutePath] !== undefined } - write(absolutePath, content) { + async write(absolutePath, content) { this.inMemoryFiles[absolutePath] = content } - list(absolutePath) { + async list(absolutePath) { return Object.keys(this.inMemoryFiles).filter(filePath => filePath.startsWith(absolutePath) && !filePath.replace(absolutePath, "").includes("/")) } - getMTime() { + async getMTime() { return 1 } - getCTime() { + async getCTime() { return 1 } dirname(path) { @@ -74,26 +85,87 @@ class MemoryWriter { return posix.join(...arguments) } } +class FusionFile { + constructor(codeAtStart, absoluteFilePath = "", fileSystem = new Fusion({})) { + this.defaultParserCode = "" + this.fileSystem = fileSystem + this.filePath = absoluteFilePath + this.filename = posix.basename(absoluteFilePath) + this.folderPath = posix.dirname(absoluteFilePath) + "/" + this.codeAtStart = codeAtStart + this.timeIndex = 0 + this.timestamp = 0 + this.importOnly = false + } + async readCodeFromStorage() { + if (this.codeAtStart !== undefined) return this // Code provided + const { filePath } = this + if (!filePath) { + this.codeAtStart = "" + return this + } + this.codeAtStart = await this.fileSystem.read(filePath) + } + get isFused() { + return this.fusedCode !== undefined + } + async fuse() { + // PASS 1: READ FULL FILE + await this.readCodeFromStorage() + const { codeAtStart, fileSystem, filePath, defaultParserCode } = this + // PASS 2: READ AND REPLACE IMPORTs + let fusedCode = codeAtStart + if (filePath) { + this.timestamp = await fileSystem.getCTime(filePath) + const fusedFile = await fileSystem.fuseFile(filePath, defaultParserCode) + this.importOnly = fusedFile.isImportOnly + fusedCode = fusedFile.fused + if (fusedFile.footers.length) fusedCode += "\n" + fusedFile.footers.join("\n") + this.dependencies = fusedFile.importFilePaths + this.fusedFile = fusedFile + } + this.fusedCode = fusedCode + this.parseCode() + return this + } + parseCode() {} + get formatted() { + return this.codeAtStart + } + async formatAndSave() { + const { codeAtStart, formatted } = this + if (codeAtStart === formatted) return false + await this.fileSystem.write(this.filePath, formatted) + return true + } +} +let fusionIdNumber = 0 class Fusion { constructor(inMemoryFiles) { + this.productCache = {} this._particleCache = {} this._parserCache = {} this._expandedImportCache = {} this._parsersExpandersCache = {} + this.defaultFileClass = FusionFile + this.parsedFiles = {} + this.folderCache = {} if (inMemoryFiles) this._storage = new MemoryWriter(inMemoryFiles) else this._storage = new DiskWriter() + fusionIdNumber = fusionIdNumber + 1 + this.fusionId = fusionIdNumber } - read(absolutePath) { - return this._storage.read(absolutePath) + async read(absolutePath) { + return await this._storage.read(absolutePath) } - exists(absolutePath) { - return this._storage.exists(absolutePath) + async exists(absolutePath) { + return await this._storage.exists(absolutePath) } - write(absolutePath, content) { - return this._storage.write(absolutePath, content) + async write(absolutePath, content) { + return await this._storage.write(absolutePath, content) } - list(absolutePath) { - return this._storage.list(absolutePath) + async list(absolutePath) { + return await this._storage.list(absolutePath) } dirname(absolutePath) { return this._storage.dirname(absolutePath) @@ -101,102 +173,120 @@ class Fusion { join(...segments) { return this._storage.join(...segments) } - getMTime(absolutePath) { - return this._storage.getMTime(absolutePath) + async getMTime(absolutePath) { + return await this._storage.getMTime(absolutePath) } - getCTime(absolutePath) { - return this._storage.getMTime(absolutePath) + async getCTime(absolutePath) { + return await this._storage.getCTime(absolutePath) } - _getFileAsParticles(absoluteFilePath) { + async writeProduct(absolutePath, content) { + this.productCache[absolutePath] = content + return await this.write(absolutePath, content) + } + async _getFileAsParticles(absoluteFilePath) { const { _particleCache } = this if (_particleCache[absoluteFilePath] === undefined) { - _particleCache[absoluteFilePath] = new Particle(this._storage.read(absoluteFilePath)) + const content = await this._storage.read(absoluteFilePath) + _particleCache[absoluteFilePath] = new Particle(content) } return _particleCache[absoluteFilePath] } - _fuseFile(absoluteFilePath) { + async _fuseFile(absoluteFilePath) { const { _expandedImportCache } = this if (_expandedImportCache[absoluteFilePath]) return _expandedImportCache[absoluteFilePath] - let code = this.read(absoluteFilePath) - const exists = this.exists(absoluteFilePath) + const [code, exists] = await Promise.all([this.read(absoluteFilePath), this.exists(absoluteFilePath)]) const isImportOnly = importOnlyRegex.test(code) // Perf hack // If its a parsers file, it will have no content, just parsers (and maybe imports). // The parsers will already have been processed. We can skip them const stripParsers = absoluteFilePath.endsWith(PARSERS_EXTENSION) - if (stripParsers) - code = code - .split("\n") - .filter(line => importRegex.test(line)) - .join("\n") + const processedCode = stripParsers + ? code + .split("\n") + .filter(line => importRegex.test(line)) + .join("\n") + : code const filepathsWithParserDefinitions = [] - if (this._doesFileHaveParsersDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath) - if (!importRegex.test(code)) + if (await this._doesFileHaveParsersDefinitions(absoluteFilePath)) { + filepathsWithParserDefinitions.push(absoluteFilePath) + } + if (!importRegex.test(processedCode)) { return { - fused: code, + fused: processedCode, footers: [], isImportOnly, importFilePaths: [], filepathsWithParserDefinitions, exists } + } + const particle = new Particle(processedCode) + const folder = this.dirname(absoluteFilePath) + // Fetch all imports in parallel + const importParticles = particle.filter(particle => particle.getLine().match(importRegex)) + const importResults = importParticles.map(async importParticle => { + const relativeFilePath = importParticle.getLine().replace("import ", "") + const absoluteImportFilePath = this.join(folder, relativeFilePath) + // todo: race conditions + const [expandedFile, exists] = await Promise.all([this._fuseFile(absoluteImportFilePath), this.exists(absoluteImportFilePath)]) + return { + expandedFile, + exists, + relativeFilePath, + absoluteImportFilePath, + importParticle + } + }) + const imported = await Promise.all(importResults) + // Assemble all imports let importFilePaths = [] let footers = [] - const particle = new Particle(code) - const folder = this.dirname(absoluteFilePath) - particle - .filter(particle => particle.getLine().match(importRegex)) - .forEach(importParticle => { - const relativeFilePath = importParticle.getLine().replace("import ", "") - const absoluteImportFilePath = this.join(folder, relativeFilePath) - const expandedFile = this._fuseFile(absoluteImportFilePath) - importFilePaths.push(absoluteImportFilePath) - importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) - const exists = this.exists(absoluteImportFilePath) - importParticle.setLine("imported " + relativeFilePath) - importParticle.set("exists", `${exists}`) - footers = footers.concat(expandedFile.footers) - if (importParticle.has("footer")) footers.push(expandedFile.fused) - else importParticle.insertLinesAfter(expandedFile.fused) - }) + imported.forEach(importResults => { + const { importParticle, absoluteImportFilePath, expandedFile, relativeFilePath, exists } = importResults + importFilePaths.push(absoluteImportFilePath) + importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) + importParticle.setLine("imported " + relativeFilePath) + importParticle.set("exists", `${exists}`) + footers = footers.concat(expandedFile.footers) + if (importParticle.has("footer")) footers.push(expandedFile.fused) + else importParticle.insertLinesAfter(expandedFile.fused) + }) + const existStates = await Promise.all(importFilePaths.map(file => this.exists(file))) + const allImportsExist = !existStates.some(exists => !exists) _expandedImportCache[absoluteFilePath] = { importFilePaths, isImportOnly, fused: particle.toString(), footers, - exists: !importFilePaths.some(file => !this.exists(file)), - filepathsWithParserDefinitions: importFilePaths.filter(filename => this._doesFileHaveParsersDefinitions(filename)).concat(filepathsWithParserDefinitions) + exists: allImportsExist, + filepathsWithParserDefinitions: ( + await Promise.all( + importFilePaths.map(async filename => ({ + filename, + hasParser: await this._doesFileHaveParsersDefinitions(filename) + })) + ) + ) + .filter(result => result.hasParser) + .map(result => result.filename) + .concat(filepathsWithParserDefinitions) } return _expandedImportCache[absoluteFilePath] } - _doesFileHaveParsersDefinitions(absoluteFilePath) { + async _doesFileHaveParsersDefinitions(absoluteFilePath) { if (!absoluteFilePath) return false const { _parsersExpandersCache } = this - if (_parsersExpandersCache[absoluteFilePath] === undefined) _parsersExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(parserRegex) + if (_parsersExpandersCache[absoluteFilePath] === undefined) { + const content = await this._storage.read(absoluteFilePath) + _parsersExpandersCache[absoluteFilePath] = !!content.match(parserRegex) + } return _parsersExpandersCache[absoluteFilePath] } - _getOneParsersParserFromFiles(filePaths, baseParsersCode) { - const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ - const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ - const asOneFile = filePaths - .map(filePath => { - const content = this._storage.read(filePath) - if (filePath.endsWith(PARSERS_EXTENSION)) return content - // Strip scroll content - return new Particle(content) - .filter(particle => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) - .map(particle => particle.asString) - .join("\n") - }) - .join("\n") - .trim() - // todo: clean up scrollsdk so we are using supported methods (perhaps add a formatOptions that allows you to tell Parsers not to run prettier on js particles) - return new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() + async _getOneParsersParserFromFiles(filePaths, baseParsersCode) { + const fileContents = await Promise.all(filePaths.map(async filePath => await this._storage.read(filePath))) + return Fusion.combineParsers(filePaths, fileContents, baseParsersCode) } - get parsers() { - return Object.values(this._parserCache).map(parser => parser.parsersParser) - } - getParser(filePaths, baseParsersCode = "") { + async getParser(filePaths, baseParsersCode = "") { const { _parserCache } = this const key = filePaths .filter(fp => fp) @@ -204,21 +294,67 @@ class Fusion { .join("\n") const hit = _parserCache[key] if (hit) return hit - const parsersParser = this._getOneParsersParserFromFiles(filePaths, baseParsersCode) - const parsersCode = parsersParser.asString - _parserCache[key] = { - parsersParser, + _parserCache[key] = await this._getOneParsersParserFromFiles(filePaths, baseParsersCode) + return _parserCache[key] + } + static combineParsers(filePaths, fileContents, baseParsersCode = "") { + const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ + const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ + const mapped = fileContents.map((content, index) => { + const filePath = filePaths[index] + if (filePath.endsWith(PARSERS_EXTENSION)) return content + return new Particle(content) + .filter(particle => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) + .map(particle => particle.asString) + .join("\n") + }) + const asOneFile = mapped.join("\n").trim() + const sorted = new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() + const parsersCode = sorted.asString + return { + parsersParser: sorted, parsersCode, parser: new HandParsersProgram(parsersCode).compileAndReturnRootParser() } - return _parserCache[key] } - fuseFile(absoluteFilePath, defaultParserCode) { - const fusedFile = this._fuseFile(absoluteFilePath) + get parsers() { + return Object.values(this._parserCache).map(parser => parser.parsersParser) + } + async fuseFile(absoluteFilePath, defaultParserCode) { + const fusedFile = await this._fuseFile(absoluteFilePath) if (!defaultParserCode) return fusedFile - // BUILD CUSTOM COMPILER, IF THERE ARE CUSTOM PARSERS NODES DEFINED - if (fusedFile.filepathsWithParserDefinitions.length) fusedFile.parser = this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode).parser + if (fusedFile.filepathsWithParserDefinitions.length) { + const parser = await this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode) + fusedFile.parser = parser.parser + } return fusedFile } + async getLoadedFile(filePath) { + return await this._getLoadedFile(filePath, this.defaultFileClass) + } + async _getLoadedFile(absolutePath, parser) { + if (this.parsedFiles[absolutePath]) return this.parsedFiles[absolutePath] + const file = new parser(undefined, absolutePath, this) + await file.fuse() + this.parsedFiles[absolutePath] = file + return file + } + getCachedLoadedFilesInFolder(folderPath, requester) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + const hit = this.folderCache[folderPath] + if (!hit) console.log(`Warning: '${folderPath}' not yet loaded in '${this.fusionId}'. Requested by '${requester.filePath}'`) + return hit || [] + } + async getLoadedFilesInFolder(folderPath, extension) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + if (this.folderCache[folderPath]) return this.folderCache[folderPath] + const allFiles = await this.list(folderPath) + const loadedFiles = await Promise.all(allFiles.filter(file => file.endsWith(extension)).map(filePath => this.getLoadedFile(filePath))) + const sorted = loadedFiles.sort((a, b) => b.timestamp - a.timestamp) + sorted.forEach((file, index) => (file.timeIndex = index)) + this.folderCache[folderPath] = sorted + return this.folderCache[folderPath] + } } window.Fusion = Fusion +window.FusionFile = FusionFile diff --git a/products/Fusion.js b/products/Fusion.js index e8532658e..a390fdd1a 100644 --- a/products/Fusion.js +++ b/products/Fusion.js @@ -1,4 +1,4 @@ -const fs = require("fs") +const fs = require("fs").promises // Change to use promises version const path = require("path") const { Disk } = require("../products/Disk.node.js") const { Utils } = require("../products/Utils.js") @@ -15,32 +15,43 @@ class DiskWriter { constructor() { this.fileCache = {} } - _read(absolutePath) { + async _read(absolutePath) { const { fileCache } = this if (!fileCache[absolutePath]) { - const exists = fs.existsSync(absolutePath) - if (exists) fileCache[absolutePath] = { absolutePath, exists: true, content: Disk.read(absolutePath).replace(/\r/g, ""), stats: fs.statSync(absolutePath) } - else fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + const exists = await fs + .access(absolutePath) + .then(() => true) + .catch(() => false) + if (exists) { + const [content, stats] = await Promise.all([fs.readFile(absolutePath, "utf8").then(content => content.replace(/\r/g, "")), fs.stat(absolutePath)]) + fileCache[absolutePath] = { absolutePath, exists: true, content, stats } + } else { + fileCache[absolutePath] = { absolutePath, exists: false, content: "", stats: { mtimeMs: 0, ctimeMs: 0 } } + } } return fileCache[absolutePath] } - exists(absolutePath) { - return this._read(absolutePath).exists + async exists(absolutePath) { + const file = await this._read(absolutePath) + return file.exists } - read(absolutePath) { - return this._read(absolutePath).content + async read(absolutePath) { + const file = await this._read(absolutePath) + return file.content } - list(folder) { + async list(folder) { return Disk.getFiles(folder) } - write(fullPath, content) { + async write(fullPath, content) { Disk.writeIfChanged(fullPath, content) } - getMTime(absolutePath) { - return this._read(absolutePath).stats.mtimeMs + async getMTime(absolutePath) { + const file = await this._read(absolutePath) + return file.stats.mtimeMs } - getCTime(absolutePath) { - return this._read(absolutePath).stats.ctimeMs + async getCTime(absolutePath) { + const file = await this._read(absolutePath) + return file.stats.ctimeMs } dirname(absolutePath) { return path.dirname(absolutePath) @@ -53,26 +64,26 @@ class MemoryWriter { constructor(inMemoryFiles) { this.inMemoryFiles = inMemoryFiles } - read(absolutePath) { + async read(absolutePath) { const value = this.inMemoryFiles[absolutePath] if (value === undefined) { return "" } return value } - exists(absolutePath) { + async exists(absolutePath) { return this.inMemoryFiles[absolutePath] !== undefined } - write(absolutePath, content) { + async write(absolutePath, content) { this.inMemoryFiles[absolutePath] = content } - list(absolutePath) { + async list(absolutePath) { return Object.keys(this.inMemoryFiles).filter(filePath => filePath.startsWith(absolutePath) && !filePath.replace(absolutePath, "").includes("/")) } - getMTime() { + async getMTime() { return 1 } - getCTime() { + async getCTime() { return 1 } dirname(path) { @@ -82,26 +93,87 @@ class MemoryWriter { return posix.join(...arguments) } } +class FusionFile { + constructor(codeAtStart, absoluteFilePath = "", fileSystem = new Fusion({})) { + this.defaultParserCode = "" + this.fileSystem = fileSystem + this.filePath = absoluteFilePath + this.filename = posix.basename(absoluteFilePath) + this.folderPath = posix.dirname(absoluteFilePath) + "/" + this.codeAtStart = codeAtStart + this.timeIndex = 0 + this.timestamp = 0 + this.importOnly = false + } + async readCodeFromStorage() { + if (this.codeAtStart !== undefined) return this // Code provided + const { filePath } = this + if (!filePath) { + this.codeAtStart = "" + return this + } + this.codeAtStart = await this.fileSystem.read(filePath) + } + get isFused() { + return this.fusedCode !== undefined + } + async fuse() { + // PASS 1: READ FULL FILE + await this.readCodeFromStorage() + const { codeAtStart, fileSystem, filePath, defaultParserCode } = this + // PASS 2: READ AND REPLACE IMPORTs + let fusedCode = codeAtStart + if (filePath) { + this.timestamp = await fileSystem.getCTime(filePath) + const fusedFile = await fileSystem.fuseFile(filePath, defaultParserCode) + this.importOnly = fusedFile.isImportOnly + fusedCode = fusedFile.fused + if (fusedFile.footers.length) fusedCode += "\n" + fusedFile.footers.join("\n") + this.dependencies = fusedFile.importFilePaths + this.fusedFile = fusedFile + } + this.fusedCode = fusedCode + this.parseCode() + return this + } + parseCode() {} + get formatted() { + return this.codeAtStart + } + async formatAndSave() { + const { codeAtStart, formatted } = this + if (codeAtStart === formatted) return false + await this.fileSystem.write(this.filePath, formatted) + return true + } +} +let fusionIdNumber = 0 class Fusion { constructor(inMemoryFiles) { + this.productCache = {} this._particleCache = {} this._parserCache = {} this._expandedImportCache = {} this._parsersExpandersCache = {} + this.defaultFileClass = FusionFile + this.parsedFiles = {} + this.folderCache = {} if (inMemoryFiles) this._storage = new MemoryWriter(inMemoryFiles) else this._storage = new DiskWriter() + fusionIdNumber = fusionIdNumber + 1 + this.fusionId = fusionIdNumber } - read(absolutePath) { - return this._storage.read(absolutePath) + async read(absolutePath) { + return await this._storage.read(absolutePath) } - exists(absolutePath) { - return this._storage.exists(absolutePath) + async exists(absolutePath) { + return await this._storage.exists(absolutePath) } - write(absolutePath, content) { - return this._storage.write(absolutePath, content) + async write(absolutePath, content) { + return await this._storage.write(absolutePath, content) } - list(absolutePath) { - return this._storage.list(absolutePath) + async list(absolutePath) { + return await this._storage.list(absolutePath) } dirname(absolutePath) { return this._storage.dirname(absolutePath) @@ -109,102 +181,120 @@ class Fusion { join(...segments) { return this._storage.join(...segments) } - getMTime(absolutePath) { - return this._storage.getMTime(absolutePath) + async getMTime(absolutePath) { + return await this._storage.getMTime(absolutePath) } - getCTime(absolutePath) { - return this._storage.getMTime(absolutePath) + async getCTime(absolutePath) { + return await this._storage.getCTime(absolutePath) } - _getFileAsParticles(absoluteFilePath) { + async writeProduct(absolutePath, content) { + this.productCache[absolutePath] = content + return await this.write(absolutePath, content) + } + async _getFileAsParticles(absoluteFilePath) { const { _particleCache } = this if (_particleCache[absoluteFilePath] === undefined) { - _particleCache[absoluteFilePath] = new Particle(this._storage.read(absoluteFilePath)) + const content = await this._storage.read(absoluteFilePath) + _particleCache[absoluteFilePath] = new Particle(content) } return _particleCache[absoluteFilePath] } - _fuseFile(absoluteFilePath) { + async _fuseFile(absoluteFilePath) { const { _expandedImportCache } = this if (_expandedImportCache[absoluteFilePath]) return _expandedImportCache[absoluteFilePath] - let code = this.read(absoluteFilePath) - const exists = this.exists(absoluteFilePath) + const [code, exists] = await Promise.all([this.read(absoluteFilePath), this.exists(absoluteFilePath)]) const isImportOnly = importOnlyRegex.test(code) // Perf hack // If its a parsers file, it will have no content, just parsers (and maybe imports). // The parsers will already have been processed. We can skip them const stripParsers = absoluteFilePath.endsWith(PARSERS_EXTENSION) - if (stripParsers) - code = code - .split("\n") - .filter(line => importRegex.test(line)) - .join("\n") + const processedCode = stripParsers + ? code + .split("\n") + .filter(line => importRegex.test(line)) + .join("\n") + : code const filepathsWithParserDefinitions = [] - if (this._doesFileHaveParsersDefinitions(absoluteFilePath)) filepathsWithParserDefinitions.push(absoluteFilePath) - if (!importRegex.test(code)) + if (await this._doesFileHaveParsersDefinitions(absoluteFilePath)) { + filepathsWithParserDefinitions.push(absoluteFilePath) + } + if (!importRegex.test(processedCode)) { return { - fused: code, + fused: processedCode, footers: [], isImportOnly, importFilePaths: [], filepathsWithParserDefinitions, exists } + } + const particle = new Particle(processedCode) + const folder = this.dirname(absoluteFilePath) + // Fetch all imports in parallel + const importParticles = particle.filter(particle => particle.getLine().match(importRegex)) + const importResults = importParticles.map(async importParticle => { + const relativeFilePath = importParticle.getLine().replace("import ", "") + const absoluteImportFilePath = this.join(folder, relativeFilePath) + // todo: race conditions + const [expandedFile, exists] = await Promise.all([this._fuseFile(absoluteImportFilePath), this.exists(absoluteImportFilePath)]) + return { + expandedFile, + exists, + relativeFilePath, + absoluteImportFilePath, + importParticle + } + }) + const imported = await Promise.all(importResults) + // Assemble all imports let importFilePaths = [] let footers = [] - const particle = new Particle(code) - const folder = this.dirname(absoluteFilePath) - particle - .filter(particle => particle.getLine().match(importRegex)) - .forEach(importParticle => { - const relativeFilePath = importParticle.getLine().replace("import ", "") - const absoluteImportFilePath = this.join(folder, relativeFilePath) - const expandedFile = this._fuseFile(absoluteImportFilePath) - importFilePaths.push(absoluteImportFilePath) - importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) - const exists = this.exists(absoluteImportFilePath) - importParticle.setLine("imported " + relativeFilePath) - importParticle.set("exists", `${exists}`) - footers = footers.concat(expandedFile.footers) - if (importParticle.has("footer")) footers.push(expandedFile.fused) - else importParticle.insertLinesAfter(expandedFile.fused) - }) + imported.forEach(importResults => { + const { importParticle, absoluteImportFilePath, expandedFile, relativeFilePath, exists } = importResults + importFilePaths.push(absoluteImportFilePath) + importFilePaths = importFilePaths.concat(expandedFile.importFilePaths) + importParticle.setLine("imported " + relativeFilePath) + importParticle.set("exists", `${exists}`) + footers = footers.concat(expandedFile.footers) + if (importParticle.has("footer")) footers.push(expandedFile.fused) + else importParticle.insertLinesAfter(expandedFile.fused) + }) + const existStates = await Promise.all(importFilePaths.map(file => this.exists(file))) + const allImportsExist = !existStates.some(exists => !exists) _expandedImportCache[absoluteFilePath] = { importFilePaths, isImportOnly, fused: particle.toString(), footers, - exists: !importFilePaths.some(file => !this.exists(file)), - filepathsWithParserDefinitions: importFilePaths.filter(filename => this._doesFileHaveParsersDefinitions(filename)).concat(filepathsWithParserDefinitions) + exists: allImportsExist, + filepathsWithParserDefinitions: ( + await Promise.all( + importFilePaths.map(async filename => ({ + filename, + hasParser: await this._doesFileHaveParsersDefinitions(filename) + })) + ) + ) + .filter(result => result.hasParser) + .map(result => result.filename) + .concat(filepathsWithParserDefinitions) } return _expandedImportCache[absoluteFilePath] } - _doesFileHaveParsersDefinitions(absoluteFilePath) { + async _doesFileHaveParsersDefinitions(absoluteFilePath) { if (!absoluteFilePath) return false const { _parsersExpandersCache } = this - if (_parsersExpandersCache[absoluteFilePath] === undefined) _parsersExpandersCache[absoluteFilePath] = !!this._storage.read(absoluteFilePath).match(parserRegex) + if (_parsersExpandersCache[absoluteFilePath] === undefined) { + const content = await this._storage.read(absoluteFilePath) + _parsersExpandersCache[absoluteFilePath] = !!content.match(parserRegex) + } return _parsersExpandersCache[absoluteFilePath] } - _getOneParsersParserFromFiles(filePaths, baseParsersCode) { - const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ - const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ - const asOneFile = filePaths - .map(filePath => { - const content = this._storage.read(filePath) - if (filePath.endsWith(PARSERS_EXTENSION)) return content - // Strip scroll content - return new Particle(content) - .filter(particle => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) - .map(particle => particle.asString) - .join("\n") - }) - .join("\n") - .trim() - // todo: clean up scrollsdk so we are using supported methods (perhaps add a formatOptions that allows you to tell Parsers not to run prettier on js particles) - return new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() + async _getOneParsersParserFromFiles(filePaths, baseParsersCode) { + const fileContents = await Promise.all(filePaths.map(async filePath => await this._storage.read(filePath))) + return Fusion.combineParsers(filePaths, fileContents, baseParsersCode) } - get parsers() { - return Object.values(this._parserCache).map(parser => parser.parsersParser) - } - getParser(filePaths, baseParsersCode = "") { + async getParser(filePaths, baseParsersCode = "") { const { _parserCache } = this const key = filePaths .filter(fp => fp) @@ -212,22 +302,67 @@ class Fusion { .join("\n") const hit = _parserCache[key] if (hit) return hit - const parsersParser = this._getOneParsersParserFromFiles(filePaths, baseParsersCode) - const parsersCode = parsersParser.asString - _parserCache[key] = { - parsersParser, + _parserCache[key] = await this._getOneParsersParserFromFiles(filePaths, baseParsersCode) + return _parserCache[key] + } + static combineParsers(filePaths, fileContents, baseParsersCode = "") { + const parserDefinitionRegex = /^[a-zA-Z0-9_]+Parser$/ + const atomDefinitionRegex = /^[a-zA-Z0-9_]+Atom/ + const mapped = fileContents.map((content, index) => { + const filePath = filePaths[index] + if (filePath.endsWith(PARSERS_EXTENSION)) return content + return new Particle(content) + .filter(particle => particle.getLine().match(parserDefinitionRegex) || particle.getLine().match(atomDefinitionRegex)) + .map(particle => particle.asString) + .join("\n") + }) + const asOneFile = mapped.join("\n").trim() + const sorted = new parsersParser(baseParsersCode + "\n" + asOneFile)._sortParticlesByInScopeOrder()._sortWithParentParsersUpTop() + const parsersCode = sorted.asString + return { + parsersParser: sorted, parsersCode, parser: new HandParsersProgram(parsersCode).compileAndReturnRootParser() } - return _parserCache[key] } - fuseFile(absoluteFilePath, defaultParserCode) { - const fusedFile = this._fuseFile(absoluteFilePath) + get parsers() { + return Object.values(this._parserCache).map(parser => parser.parsersParser) + } + async fuseFile(absoluteFilePath, defaultParserCode) { + const fusedFile = await this._fuseFile(absoluteFilePath) if (!defaultParserCode) return fusedFile - // BUILD CUSTOM COMPILER, IF THERE ARE CUSTOM PARSERS NODES DEFINED - if (fusedFile.filepathsWithParserDefinitions.length) fusedFile.parser = this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode).parser + if (fusedFile.filepathsWithParserDefinitions.length) { + const parser = await this.getParser(fusedFile.filepathsWithParserDefinitions, defaultParserCode) + fusedFile.parser = parser.parser + } return fusedFile } + async getLoadedFile(filePath) { + return await this._getLoadedFile(filePath, this.defaultFileClass) + } + async _getLoadedFile(absolutePath, parser) { + if (this.parsedFiles[absolutePath]) return this.parsedFiles[absolutePath] + const file = new parser(undefined, absolutePath, this) + await file.fuse() + this.parsedFiles[absolutePath] = file + return file + } + getCachedLoadedFilesInFolder(folderPath, requester) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + const hit = this.folderCache[folderPath] + if (!hit) console.log(`Warning: '${folderPath}' not yet loaded in '${this.fusionId}'. Requested by '${requester.filePath}'`) + return hit || [] + } + async getLoadedFilesInFolder(folderPath, extension) { + folderPath = Utils.ensureFolderEndsInSlash(folderPath) + if (this.folderCache[folderPath]) return this.folderCache[folderPath] + const allFiles = await this.list(folderPath) + const loadedFiles = await Promise.all(allFiles.filter(file => file.endsWith(extension)).map(filePath => this.getLoadedFile(filePath))) + const sorted = loadedFiles.sort((a, b) => b.timestamp - a.timestamp) + sorted.forEach((file, index) => (file.timeIndex = index)) + this.folderCache[folderPath] = sorted + return this.folderCache[folderPath] + } } -module.exports = { Fusion } +module.exports = { Fusion, FusionFile } diff --git a/products/Particle.browser.js b/products/Particle.browser.js index f636f8136..069e9b7dd 100644 --- a/products/Particle.browser.js +++ b/products/Particle.browser.js @@ -2598,7 +2598,7 @@ Particle.iris = `sepal_length,sepal_width,petal_length,petal_width,species 4.9,2.5,4.5,1.7,virginica 5.1,3.5,1.4,0.2,setosa 5,3.4,1.5,0.2,setosa` -Particle.getVersion = () => "96.0.0" +Particle.getVersion = () => "97.0.0" class AbstractExtendibleParticle extends Particle { _getFromExtended(cuePath) { const hit = this._getParticleFromExtended(cuePath) diff --git a/products/Particle.js b/products/Particle.js index 38352c0be..af72877d1 100644 --- a/products/Particle.js +++ b/products/Particle.js @@ -2588,7 +2588,7 @@ Particle.iris = `sepal_length,sepal_width,petal_length,petal_width,species 4.9,2.5,4.5,1.7,virginica 5.1,3.5,1.4,0.2,setosa 5,3.4,1.5,0.2,setosa` -Particle.getVersion = () => "96.0.0" +Particle.getVersion = () => "97.0.0" class AbstractExtendibleParticle extends Particle { _getFromExtended(cuePath) { const hit = this._getParticleFromExtended(cuePath) diff --git a/products/SandboxApp.browser.js b/products/SandboxApp.browser.js index f4ca2767f..afebeec4d 100644 --- a/products/SandboxApp.browser.js +++ b/products/SandboxApp.browser.js @@ -76,10 +76,11 @@ This is my content "/footer.scroll": "The end.", "/main": particle.toString() } - const fusion = new Fusion(files) - const result = fusion.fuseFile("/main") - this.fused = result - willowBrowser.setHtmlOfElementWithIdHack("fusionConsole", result.fused) + const fs = new Fusion(files) + const file = new FusionFile(particle.toString(), "/main", fs) + await file.fuse() + this.file = file + willowBrowser.setHtmlOfElementWithIdHack("fusionConsole", file.fusedCode) } async particleComponentDidMount() { // todo: refactor!!! split these into components diff --git a/releaseNotes.scroll b/releaseNotes.scroll index 656f99fd0..673a629a9 100644 --- a/releaseNotes.scroll +++ b/releaseNotes.scroll @@ -20,6 +20,12 @@ node_modules/scroll-cli/microlangs/changes.parsers thinColumns 4 +📦 97.0.0 2024-11-29 +# Fusion v2 Release +🎉 Fusion is now async. Needed to support in-browser imports. +🎉 new FusionFile class +⚠️ BREAKING: fusion API is now async. update all fuseFile to await fuseFile. + 📦 96.0.0 2024-11-28 # Fusion v1 Release 🎉 added Fusion console to sandbox diff --git a/sandbox/SandboxApp.ts b/sandbox/SandboxApp.ts index c830dec63..f5337a3c3 100644 --- a/sandbox/SandboxApp.ts +++ b/sandbox/SandboxApp.ts @@ -2,7 +2,7 @@ const { AbstractParticleComponentParser, ParticleComponentFrameworkDebuggerComponent, AbstractGithubTriangleComponent } = require("../products/ParticleComponentFramework.node.js") const { Particle } = require("../products/Particle.js") -const { Fusion } = require("../products/Fusion.js") +const { Fusion, FusionFile } = require("../products/Fusion.js") // Todo: add inputs at the top to change the edge, particle, and atom delimiters. @@ -94,10 +94,11 @@ This is my content "/footer.scroll": "The end.", "/main": particle.toString() } - const fusion = new Fusion(files) - const result = fusion.fuseFile("/main") - this.fused = result - willowBrowser.setHtmlOfElementWithIdHack("fusionConsole", result.fused) + const fs = new Fusion(files) + const file = new FusionFile(particle.toString(), "/main", fs) + await file.fuse() + this.file = file + willowBrowser.setHtmlOfElementWithIdHack("fusionConsole", file.fusedCode) } async particleComponentDidMount() {