-
-
Notifications
You must be signed in to change notification settings - Fork 395
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(transformers): introduce
matchAlgorithm
option for new matchin…
…g algorithm (#835) Co-authored-by: Anthony Fu <[email protected]>
- Loading branch information
Showing
21 changed files
with
448 additions
and
105 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
import type { Element, Text } from 'hast' | ||
import type { ShikiTransformer, ShikiTransformerContext } from 'shiki' | ||
import { parseComments, type ParsedComments, v1ClearEndCommentPrefix } from './parse-comments' | ||
|
||
export type MatchAlgorithm = 'v1' | 'v3' | ||
|
||
export interface MatchAlgorithmOptions { | ||
/** | ||
* Match algorithm to use | ||
* | ||
* @see https://shiki.style/packages/transformers#matching-algorithm | ||
* @default 'v1' | ||
*/ | ||
matchAlgorithm?: MatchAlgorithm | ||
} | ||
|
||
export function createCommentNotationTransformer( | ||
name: string, | ||
regex: RegExp, | ||
onMatch: ( | ||
this: ShikiTransformerContext, | ||
match: string[], | ||
line: Element, | ||
commentNode: Element, | ||
lines: Element[], | ||
index: number | ||
) => boolean, | ||
matchAlgorithm: MatchAlgorithm = 'v1', | ||
): ShikiTransformer { | ||
return { | ||
name, | ||
code(code) { | ||
const lines = code.children.filter(i => i.type === 'element') | ||
const linesToRemove: (Element | Text)[] = [] | ||
|
||
code.data ??= {} as any | ||
const data = code.data as { | ||
_shiki_notation?: ParsedComments | ||
} | ||
|
||
data._shiki_notation ??= parseComments(lines, ['jsx', 'tsx'].includes(this.options.lang), matchAlgorithm) | ||
const parsed = data._shiki_notation | ||
|
||
for (const comment of parsed) { | ||
if (comment.info[1].length === 0) | ||
continue | ||
|
||
const isLineCommentOnly = comment.line.children.length === (comment.isJsxStyle ? 3 : 1) | ||
let lineIdx = lines.indexOf(comment.line) | ||
if (isLineCommentOnly && matchAlgorithm !== 'v1') | ||
lineIdx++ | ||
|
||
let replaced = false | ||
comment.info[1] = comment.info[1].replace(regex, (...match) => { | ||
if (onMatch.call(this, match, comment.line, comment.token, lines, lineIdx)) { | ||
replaced = true | ||
return '' | ||
} | ||
|
||
return match[0] | ||
}) | ||
|
||
if (!replaced) | ||
continue | ||
|
||
if (matchAlgorithm === 'v1') { | ||
comment.info[1] = v1ClearEndCommentPrefix(comment.info[1]) | ||
} | ||
|
||
const isEmpty = comment.info[1].trim().length === 0 | ||
// ignore comment node | ||
if (isEmpty) | ||
comment.info[1] = '' | ||
|
||
if (isEmpty && isLineCommentOnly) { | ||
linesToRemove.push(comment.line) | ||
} | ||
else if (isEmpty && comment.isJsxStyle) { | ||
comment.line.children.splice(comment.line.children.indexOf(comment.token) - 1, 3) | ||
} | ||
else if (isEmpty) { | ||
comment.line.children.splice(comment.line.children.indexOf(comment.token), 1) | ||
} | ||
else { | ||
const head = comment.token.children[0] | ||
|
||
if (head.type === 'text') { | ||
head.value = comment.info.join('') | ||
} | ||
} | ||
} | ||
|
||
for (const line of linesToRemove) | ||
code.children.splice(code.children.indexOf(line), 1) | ||
}, | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
import type { Element, ElementContent } from 'hast' | ||
import type { MatchAlgorithm } from './notation-transformer' | ||
|
||
export type ParsedComments = { | ||
line: Element | ||
token: Element | ||
info: [prefix: string, content: string, suffix?: string] | ||
isJsxStyle: boolean | ||
}[] | ||
|
||
/** | ||
* some comment formats have to be located at the end of line | ||
* hence we can skip matching them for other tokens | ||
*/ | ||
const matchers: [re: RegExp, endOfLine: boolean][] = [ | ||
[/^(<!--)(.+)(-->)$/, false], | ||
[/^(\/\*)(.+)(\*\/)$/, false], | ||
[/^(\/\/|["'#]|;{1,2}|%{1,2}|--)(.*)$/, true], | ||
/** | ||
* for multi-line comments like this | ||
*/ | ||
[/^(\*)(.+)$/, true], | ||
] | ||
|
||
/** | ||
* @param lines line tokens | ||
* @param jsx enable JSX parsing | ||
* @param matchAlgorithm matching algorithm | ||
*/ | ||
export function parseComments( | ||
lines: Element[], | ||
jsx: boolean, | ||
matchAlgorithm: MatchAlgorithm, | ||
): ParsedComments { | ||
const out: ParsedComments = [] | ||
|
||
for (const line of lines) { | ||
const elements = line.children | ||
let start = elements.length - 1 | ||
if (matchAlgorithm === 'v1') | ||
start = 0 | ||
else if (jsx) | ||
// one step further for JSX as comment is inside curly brackets | ||
start = elements.length - 2 | ||
|
||
for (let i = Math.max(start, 0); i < elements.length; i++) { | ||
const token = elements[i] | ||
if (token.type !== 'element') | ||
continue | ||
const head = token.children.at(0) | ||
if (head?.type !== 'text') | ||
continue | ||
|
||
const isLast = i === elements.length - 1 | ||
const match = matchToken(head.value, isLast) | ||
if (!match) | ||
continue | ||
|
||
if (jsx && !isLast && i !== 0) { | ||
out.push({ | ||
info: match, | ||
line, | ||
token, | ||
isJsxStyle: isValue(elements[i - 1], '{') && isValue(elements[i + 1], '}'), | ||
}) | ||
} | ||
else { | ||
out.push({ | ||
info: match, | ||
line, | ||
token, | ||
isJsxStyle: false, | ||
}) | ||
} | ||
} | ||
} | ||
|
||
return out | ||
} | ||
|
||
function isValue(element: ElementContent, value: string): boolean { | ||
if (element.type !== 'element') | ||
return false | ||
const text = element.children[0] | ||
if (text.type !== 'text') | ||
return false | ||
|
||
return text.value.trim() === value | ||
} | ||
|
||
/** | ||
* @param text text value of comment node | ||
* @param isLast whether the token is located at the end of line | ||
*/ | ||
function matchToken(text: string, isLast: boolean): [prefix: string, content: string, suffix?: string] | undefined { | ||
// no leading and trailing spaces allowed for matchers | ||
// we extract the spaces | ||
let trimmed = text.trimStart() | ||
const spaceFront = text.length - trimmed.length | ||
|
||
trimmed = trimmed.trimEnd() | ||
const spaceEnd = text.length - trimmed.length - spaceFront | ||
|
||
for (const [matcher, endOfLine] of matchers) { | ||
if (endOfLine && !isLast) | ||
continue | ||
|
||
const result = matcher.exec(trimmed) | ||
if (!result) | ||
continue | ||
|
||
return [ | ||
' '.repeat(spaceFront) + result[1], | ||
result[2], | ||
result[3] ? result[3] + ' '.repeat(spaceEnd) : undefined, | ||
] | ||
} | ||
} | ||
|
||
/** | ||
* Remove empty comment prefixes at line end, e.g. `// ` | ||
* | ||
* For matchAlgorithm v1 | ||
*/ | ||
export function v1ClearEndCommentPrefix(text: string): string { | ||
const regex = /(?:\/\/|["'#]|;{1,2}|%{1,2}|--)(.*)$/ | ||
const result = regex.exec(text) | ||
|
||
if (result && result[1].trim().length === 0) { | ||
return text.slice(0, result.index) | ||
} | ||
|
||
return text | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.