diff --git a/changelog.md b/changelog.md index ddc5e123..01a25d6d 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,11 @@ When contributing a fix, feature or example please add a new line to briefly exp ## 0.3.x * _add next change here_ +## 0.3.5 +* codeAsInSource has been reworked to work better with templates and uses of `nbCode` in different files. +* If you don't use any nbJs, now nimib won't build an empty nim file in those cases. +* The temporary js files generated by nbJs now has unique names to allow parallel builds. + ## 0.3.4 * added `nbCodeDisplay` and `nbCodeAnd` (#158). diff --git a/nimib.nimble b/nimib.nimble index 1f252f29..f7d2ab5f 100644 --- a/nimib.nimble +++ b/nimib.nimble @@ -1,7 +1,7 @@ # Package -version = "0.3.4" -author = "Pietro Peterlongo" +version = "0.3.5" +author = "Pietro Peterlongo & Hugo Granström" description = "nimib 🐳 - nim 👑 driven ⛵ publishing ✍" license = "MIT" srcDir = "src" diff --git a/src/nimib/jsutils.nim b/src/nimib/jsutils.nim index 9f795fec..2a05ccb7 100644 --- a/src/nimib/jsutils.nim +++ b/src/nimib/jsutils.nim @@ -1,4 +1,4 @@ -import std / [macros, macrocache, tables, strutils, strformat, sequtils, sugar, os] +import std / [macros, macrocache, tables, strutils, strformat, sequtils, sugar, os, hashes] import ./types proc contains(tab: CacheTable, keyToCheck: string): bool = @@ -142,7 +142,7 @@ proc compileNimToJs*(doc: var NbDoc, blk: var NbBlock) = createDir(tempdir) let (dir, filename, ext) = doc.thisFile.splitFile() let nimfile = dir / (filename & "_nbCodeToJs_" & $doc.newId() & ext).RelativeFile - let jsfile = tempdir / "out.js" + let jsfile = tempdir / &"out{hash(doc.thisFile)}.js" var codeText = blk.context["transformedCode"].vString let nbJsCounter = doc.nbJsCounter doc.nbJsCounter += 1 @@ -178,8 +178,9 @@ proc nbCollectAllNbJs*(doc: var NbDoc) = code.add "\n" & blk.context["transformedCode"].vString code = topCode & "\n" & code - # Create block which which will compile the code when rendered (nbJsFromJsOwnFile) - var blk = NbBlock(command: "nbJsFromCodeOwnFile", code: code, context: newContext(searchDirs = @[], partials = doc.partials), output: "") - blk.context["transformedCode"] = code - doc.blocks.add blk - doc.blk = blk + if not code.isEmptyOrWhitespace: + # Create block which which will compile the code when rendered (nbJsFromJsOwnFile) + var blk = NbBlock(command: "nbJsFromCodeOwnFile", code: code, context: newContext(searchDirs = @[], partials = doc.partials), output: "") + blk.context["transformedCode"] = code + doc.blocks.add blk + doc.blk = blk diff --git a/src/nimib/sources.nim b/src/nimib/sources.nim index ceb16e0e..81799e15 100644 --- a/src/nimib/sources.nim +++ b/src/nimib/sources.nim @@ -6,29 +6,40 @@ import std/[ import types - # Credits to @haxscramper for sharing his code on reading the line info # And credits to @Yardanico for making a previous attempt which @hugogranstrom have taken much inspiration from # when implementing this. type Pos* = object + filename*: string line*: int column*: int +proc `<`*(p1, p2: Pos): bool = + doAssert p1.filename == p2.filename, """ + Code from two different files were found in the same nbCode! + If you want to mix code from different files in nbCode, use -d:nimibCodeFromAst instead. + If you are not mixing code from different files, please open an issue on nimib's Github with a minimal reproducible example.""" + (p1.line, p1.column) < (p2.line, p2.column) + proc toPos*(info: LineInfo): Pos = - Pos(line: info.line, column: info.column) + Pos(line: info.line, column: info.column, filename: info.filename) + +proc startPos(node: NimNode): Pos = + case node.kind + of nnkStmtList: + return node[0].startPos() + else: + result = toPos(node.lineInfoObj()) + for child in node.children: + let childPos = child.startPos() + # If we can't get the line info for some reason, skip it! + if childPos.line == 0: continue + + if childPos < result: + result = childPos -proc startPos*(node: NimNode): Pos = - ## Get the starting position of a NimNode. Corrections will be needed for certains cases though. - # Has column info - case node.kind: - of nnkNone .. nnkNilLit, nnkDiscardStmt, nnkCommentStmt: - result = toPos(node.lineInfoObj()) - of nnkBlockStmt: - result = node[1].startPos() - else: - result = node[0].startPos() proc finishPos*(node: NimNode): Pos = ## Get the ending position of a NimNode. Corrections will be needed for certains cases though. @@ -56,18 +67,24 @@ proc finishPos*(node: NimNode): Pos = proc isCommandLine*(s: string, command: string): bool = nimIdentNormalize(s.strip()).startsWith(nimIdentNormalize(command)) +proc isCommentLine*(s: string): bool = + s.strip.startsWith('#') + +proc findStartLine*(source: seq[string], startPos: Pos): int = + let line = source[startPos.line - 1] + let preline = line[0 ..< startPos.column - 1] + # Multiline, we need to check further up for comments + if preline.isEmptyOrWhitespace: + result = startPos.line - 1 + # Make sure we catch all comments + while source[result-1].isCommentLine() or source[result-1].isEmptyOrWhitespace() or source[result-1].nimIdentNormalize.strip() == "type": + dec result + # Now remove all empty lines + while source[result].isEmptyOrWhitespace(): + inc result + else: # starts on same line as command + return startPos.line - 1 -proc findStartLine*(source: seq[string], command: string, startPos: int): int = - if source[startPos].isCommandLine(command): - return startPos - # The code is starting on a line below the command - # Decrease result until it is on the line below the command - result = startPos - while not source[result-1].isCommandLine(command): - dec result - # Remove empty lines at the beginning of the block - while source[result].isEmptyOrWhitespace: - inc result proc findEndLine*(source: seq[string], command: string, startLine, endPos: int): int = result = endPos @@ -90,49 +107,32 @@ proc findEndLine*(source: seq[string], command: string, startLine, endPos: int): while result < source.high and (source[result+1].startsWith(baseIndentStr) or source[result+1].isEmptyOrWhitespace): inc result - proc getCodeBlock*(source, command: string, startPos, endPos: Pos): string = ## Extracts the code in source from startPos to endPos with additional processing to get the entire code block. - let rawLines = source.split("\n") - var startLine = findStartLine(rawLines, command, startPos.line - 1) + let rawLines = source.splitLines() + let rawStartLine = startPos.line - 1 + let rawStartCol = startPos.column - 1 + var startLine = findStartLine(rawLines, startPos) var endLine = findEndLine(rawLines, command, startLine, endPos.line - 1) var lines = rawLines[startLine .. endLine] - let baseIndent = skipWhile(rawLines[startLine], {' '}) - - let startsOnCommandLine = lines[0].isCommandLine(command) # is it nbCode: code or nbCode: code - if startsOnCommandLine: # remove the command - var startColumn = startPos.column - # the "import"-part is not included in the startPos - let startsWithImport = lines[0].find("import") - if startsWithImport != -1: - startColumn = startsWithImport - lines[0] = lines[0][startColumn .. ^1].strip() - - var codeText: string - if startLine == endLine and startsOnCommandLine: # single-line expression - # remove eventual unneccerary parenthesis - let line = rawLines[startLine] # includes command and eventual opening parethesises - var extractedLine = lines[0] # doesn't include command - if extractedLine.endsWith(")"): - # check if the ending ")" has a matching "(", otherwise remove it. - var nOpen: int - var i = startPos.column - # count the number of opening brackets before code starts. - while line[i-1] in Whitespace or line[i-1] == '(': - if line[i-1] == '(': - nOpen += 1 - i -= 1 - var nRemoved: int - while nRemoved < nOpen: # remove last char until we have removed correct number of parentesis - # We assume we are given correct Nim code and thus won't have to check what we remove, it should either be Whitespace or ')' - assert extractedLine[^1] in Whitespace or extractedLine[^1] == ')', "Unexpected ending of string during parsing. Single line expression ended with character that wasn't whitespace of ')'." - if extractedLine[^1] == ')': - nRemoved += 1 - extractedLine.setLen(extractedLine.len-1) - codeText = extractedLine + let startsOnCommandLine = block: + let preline = lines[0][0 ..< rawStartCol] + startLine == rawStartLine and (not preline.isEmptyOrWhitespace) and (not (preline.nimIdentNormalize.strip() in ["for", "type"])) + + if startsOnCommandLine: + lines[0] = lines[0][rawStartCol .. ^1].strip() + + if startLine == endLine and startsOnCommandLine: + # single line expression + var line = lines[0] # doesn't include command, but includes opening parenthesis + while line.startsWith('(') and line.endsWith(')'): + line = line[1 .. ^2].strip() + + result = line else: # multi-line expression + let baseIndent = skipWhile(rawLines[startLine], {' '}) var preserveIndent: bool = false for i in 0 .. lines.high: let line = lines[i] @@ -141,93 +141,23 @@ proc getCodeBlock*(source, command: string, startPos, endPos: Pos): string = lines[i] = line.substr(baseIndent) if nonMatching: # there is a non-matching triple-quote string preserveIndent = not preserveIndent - codeText = lines.join("\n") - result = codeText - - - - -func getCodeBlockOld*(source: string, command: string, startPos, endPos: Pos): string = - ## Extracts the code in source from startPos to endPos with additional processing to get the entire code block. - let lines = source.split("\n") - var startLine = startPos.line - 1 - var endLine = endPos.line - 1 - debugecho "Start line: ", startLine + 1, startPos - debugecho "End line: ", endLine + 1, endPos - var codeText: string - if not lines[startLine].isCommandLine(command): # multiline case - while 0 < startLine and not lines[startLine-1].isCommandLine(command): - #[ cases like this reports the third line instead of the second line: - nbCode: - let # this is the line we want - x = 1 # but this is the one we get - ]# - dec startLine - - let indent = skipWhile(lines[startLine], {' '}) - let indentStr = " ".repeat(indent) - - if lines[endLine].count("\"\"\"") == 1: # only opening of triple quoted string found. Rest is below it. - inc endLine # bump it to not trigger the loop to immediately break - while endLine < lines.high and "\"\"\"" notin lines[endLine]: - inc endLine - debugecho "Triple quote: ", lines[endLine] - - while endLine < lines.high and (lines[endLine+1].startsWith(indentStr) or lines[endLine+1].isEmptyOrWhitespace):# and lines[endLine+1].strip().startsWith("#"): - # Ending Comments should be included as well, but they won't be included in the AST -> endLine doesn't take them into account. - # Block comments must be properly indented (including the content) - inc endLine - - var codeLines = lines[startLine .. endLine] - - var notIndentLines: seq[int] # these lines are not to be adjusted for indentation. Eg content of triple quoted strings. - var i: int - while i < codeLines.len: - if codeLines[i].count("\"\"\"") == 1: - # We must do the identification of triple quoted string separatly from the endLine bumping because the triple strings - # might not be the last expression in the code block. - inc i # bump it to not trigger the loop to immediately break on the initial """ - notIndentLines.add i - while i < codeLines.len and "\"\"\"" notin codeLines[i]: - inc i - notIndentLines.add i - inc i - - let parsedLines = collect(newSeqOfCap(codeLines.len)): - for i in 0 .. codeLines.high: - if i in notIndentLines: - codeLines[i] - else: - codeLines[i].substr(indent) - codeText = parsedLines.join("\n") - elif lines[startLine].isCommandLine(command) and "\"\"\"" in lines[startLine]: # potentially multiline string - discard - else: # single line case, eg `nbCode: echo "Hello World"` - let line = lines[startLine] - var extractedLine = line[startPos.column .. ^1].strip() - if extractedLine.strip().endsWith(")"): - # check if the ending ")" has a matching "(", otherwise remove it. - var nOpen: int - var i = startPos.column - # count the number of opening brackets before code starts. - while line[i-1] in Whitespace or line[i-1] == '(': - if line[i-1] == '(': - nOpen += 1 - i -= 1 - var nRemoved: int - while nRemoved < nOpen: # remove last char until we have removed correct number of parentesis - # We assume we are given correct Nim code and thus won't have to check what we remove, it should either be Whitespace or ')' - assert extractedLine[^1] in Whitespace or extractedLine[^1] == ')', "Unexpected ending of string during parsing. Single line expression ended with character that wasn't whitespace of ')'." - if extractedLine[^1] == ')': - nRemoved += 1 - extractedLine.setLen(extractedLine.len-1) - codeText = extractedLine - return codeText + result = lines.join("\n") macro getCodeAsInSource*(source: string, command: static string, body: untyped): string = ## Returns string for the code in body from source. # substitute for `toStr` in blocks.nim let startPos = startPos(body) + let filename = startPos.filename.newLit let endPos = finishPos(body) + let endFilename = endPos.filename.newLit + result = quote do: - getCodeBlock(`source`, `command`, `startPos`, `endPos`) \ No newline at end of file + if `filename` notin nb.sourceFiles: + nb.sourceFiles[`filename`] = readFile(`filename`) + + doAssert `endFilename` == `filename`, """ + Code from two different files were found in the same nbCode! + If you want to mix code from different files in nbCode, use -d:nimibCodeFromAst instead. + If you are not mixing code from different files, please open an issue on nimib's Github with a minimal reproducible example.""" + + getCodeBlock(nb.sourceFiles[`filename`], `command`, `startPos`, `endPos`) \ No newline at end of file diff --git a/src/nimib/types.nim b/src/nimib/types.nim index a3341c71..a3c31928 100644 --- a/src/nimib/types.nim +++ b/src/nimib/types.nim @@ -20,6 +20,7 @@ type thisFile*: AbsoluteFile filename*: string source*: string + sourceFiles*: Table[string, string] initDir*: AbsoluteDir options*: NbOptions cfg*: NbConfig diff --git a/tests/tsources.nim b/tests/tsources.nim index 4b4d3e06..d2af6ce2 100644 --- a/tests/tsources.nim +++ b/tests/tsources.nim @@ -7,8 +7,9 @@ suite "test sources": # the replace stuff needed on windows where the lines read from file will have windows native new lines test $currentTest: actual = nbBlock.code - echo &"===\n---actual:\n{actual.repr}\n---expected\n{expected.repr}\n---\n===" check actual.nbNormalize == expected.nbNormalize + if actual.nbNormalize != expected.nbNormalize: + echo &"===\n---actual:\n{actual.repr}\n---expected\n{expected.repr}\n---\n===" currentTest += 1 var currentTest: int @@ -107,4 +108,59 @@ end""" expected = "echo y" check + nbCode: + block: + let + b = 1 + + expected = "block:\n let\n b = 1" + check + + template notNbCode(body: untyped) = + nbCode: + body + + notNbCode: + echo y + + expected = "echo y" + check + + template `&`(a,b: int) = discard + + nbCode: + 1 & + 2 + + expected = "1 &\n 2" + check + + nbCode: + nb.context["no_source"] = true + + expected = "nb.context[\"no_source\"] = true" + check + + nbCode: discard + expected = "discard" + check + + nbCode: + for n in 0 .. 1: + discard + expected = "for n in 0 .. 1:\n discard" + check + + template nbCodeInTemplate = + nbCode: + nb.renderPlans["nbText"] = @["mdOutputToHtml"] + + nbCodeInTemplate() + expected = """nb.renderPlans["nbText"] = @["mdOutputToHtml"]""" + check + + nbCode: + type A = object + expected = "type A = object" + check \ No newline at end of file