diff --git a/markdowndb.config.js b/markdowndb.config.js new file mode 100644 index 0000000..8984816 --- /dev/null +++ b/markdowndb.config.js @@ -0,0 +1,3 @@ +export default { + customFields: [] +}; diff --git a/package-lock.json b/package-lock.json index c97855c..3e716bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@portaljs/remark-wiki-link": "^1.0.4", "gray-matter": "^4.0.3", "knex": "^2.4.2", + "micromatch": "^4.0.5", "react-markdown": "^9.0.1", "remark-gfm": "^3.0.1", "remark-parse": "^10.0.1", @@ -25,6 +26,7 @@ "@changesets/changelog-github": "^0.4.8", "@changesets/cli": "^2.26.1", "@types/jest": "^29.5.1", + "@types/micromatch": "^4.0.6", "@typescript-eslint/eslint-plugin": "^5.59.5", "@typescript-eslint/parser": "^5.59.5", "eslint": "^8.40.0", @@ -2154,6 +2156,12 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/braces": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz", + "integrity": "sha512-0WR3b8eaISjEW7RpZnclONaLFDf7buaowRHdqLp4vLj54AsSAYWfh3DRbfiYJY9XDxMgx1B4sE1Afw2PGpuHOA==", + "dev": true + }, "node_modules/@types/debug": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", @@ -2236,6 +2244,15 @@ "@types/unist": "*" } }, + "node_modules/@types/micromatch": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.6.tgz", + "integrity": "sha512-2eulCHWqjEpk9/vyic4tBhI8a9qQEl6DaK2n/sF7TweX9YESlypgKyhXMDGt4DAOy/jhLPvVrZc8pTDAMsplJA==", + "dev": true, + "dependencies": { + "@types/braces": "*" + } + }, "node_modules/@types/minimist": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", @@ -2930,7 +2947,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -4210,7 +4226,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -5182,7 +5197,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -7634,7 +7648,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", - "dev": true, "dependencies": { "braces": "^3.0.2", "picomatch": "^2.3.1" @@ -8307,7 +8320,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -10537,7 +10549,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "dependencies": { "is-number": "^7.0.0" }, diff --git a/package.json b/package.json index 110c8a7..7f157e5 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@portaljs/remark-wiki-link": "^1.0.4", "gray-matter": "^4.0.3", "knex": "^2.4.2", + "micromatch": "^4.0.5", "react-markdown": "^9.0.1", "remark-gfm": "^3.0.1", "remark-parse": "^10.0.1", @@ -57,6 +58,7 @@ "@changesets/changelog-github": "^0.4.8", "@changesets/cli": "^2.26.1", "@types/jest": "^29.5.1", + "@types/micromatch": "^4.0.6", "@typescript-eslint/eslint-plugin": "^5.59.5", "@typescript-eslint/parser": "^5.59.5", "eslint": "^8.40.0", @@ -69,4 +71,4 @@ "ts-node": "^10.9.1", "typescript": "^5.0.4" } -} \ No newline at end of file +} diff --git a/src/bin/index.js b/src/bin/index.js index 6459ccc..b3afc77 100755 --- a/src/bin/index.js +++ b/src/bin/index.js @@ -1,21 +1,41 @@ #!/usr/bin/env node - +import * as path from 'path'; import { MarkdownDB } from "../lib/markdowndb.js"; +async function loadConfig(configFilePath) { + const normalizedPath = path.resolve(configFilePath || 'markdowndb.config.js'); + const fileUrl = new URL(`file://${normalizedPath}`); + + try { + // Import the module using the file URL + const configModule = await import(fileUrl.href); + return configModule.default; + } catch (error) { + if (configFilePath) { + throw new Error(`Error loading configuration file from ${normalizedPath}: ${error.message}`); + } + } +} + // TODO get these from markdowndb.config.js or something const dbPath = "markdown.db"; const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/]; -const [contentPath] = process.argv.slice(2); +const [contentPath, configFilePath] = process.argv.slice(2); if (!contentPath) { throw new Error("Invalid/Missing path to markdown content folder"); } +// Load the configuration file dynamically +const config = await loadConfig(configFilePath); + const client = new MarkdownDB({ client: "sqlite3", connection: { filename: dbPath, }, +}, { + customConfig: config }); await client.init(); diff --git a/src/lib/CustomConfig.ts b/src/lib/CustomConfig.ts new file mode 100644 index 0000000..af52289 --- /dev/null +++ b/src/lib/CustomConfig.ts @@ -0,0 +1,8 @@ +import { FileInfo } from "./process.js"; +import { Root } from "remark-parse/lib/index.js"; + +export interface CustomConfig { + customFields?: ((fileInfo: FileInfo, ast: Root) => any)[]; + include?: string[]; + exclude?: string[]; +} diff --git a/src/lib/indexFolder.ts b/src/lib/indexFolder.ts index 6b28241..ae445a9 100644 --- a/src/lib/indexFolder.ts +++ b/src/lib/indexFolder.ts @@ -1,33 +1,76 @@ import { FileInfo, processFile } from "./process.js"; import { recursiveWalkDir } from "./recursiveWalkDir.js"; +import { CustomConfig } from "./CustomConfig.js"; +import micromatch from "micromatch"; export function indexFolder( folderPath: string, pathToUrlResolver: (filePath: string) => string, + config: CustomConfig, ignorePatterns?: RegExp[] ) { const filePathsToIndex = recursiveWalkDir(folderPath); const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) => - shouldIncludeFile(filePath, ignorePatterns) + shouldIncludeFile({ + filePath, + ignorePatterns, + includeGlob: config.include, + excludeGlob: config.exclude, + }) ); const files: FileInfo[] = []; + + const customFields = config.customFields || []; for (const filePath of filteredFilePathsToIndex) { - const fileObject = processFile( - folderPath, - filePath, + const fileObject = processFile(folderPath, filePath, { pathToUrlResolver, - filePathsToIndex - ); + filePathsToIndex, + customFields, + }); files.push(fileObject); } return files; } -function shouldIncludeFile( - filePath: string, - ignorePatterns?: RegExp[] -): boolean { - return !( - ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath)) - ); +function shouldIncludeFile({ + filePath, + ignorePatterns, + includeGlob, + excludeGlob, +}: { + filePath: string; + ignorePatterns?: RegExp[]; + includeGlob?: string[]; + excludeGlob?: string[]; +}): boolean { + const normalizedFilePath = filePath.replace(/\\/g, "/"); + + if ( + ignorePatterns && + ignorePatterns.some((pattern) => pattern.test(normalizedFilePath)) + ) { + return false; + } + + // Check if the file should be included based on includeGlob + if ( + includeGlob && + !includeGlob.some((pattern) => + micromatch.isMatch(normalizedFilePath, pattern) + ) + ) { + return false; + } + + // Check if the file should be excluded based on excludeGlob + if ( + excludeGlob && + excludeGlob.some((pattern) => + micromatch.isMatch(normalizedFilePath, pattern) + ) + ) { + return false; + } + + return true; } diff --git a/src/lib/markdowndb.ts b/src/lib/markdowndb.ts index a759467..64437c1 100644 --- a/src/lib/markdowndb.ts +++ b/src/lib/markdowndb.ts @@ -12,6 +12,7 @@ import { getUniqueValues, } from "./databaseUtils.js"; import fs from "fs"; +import { CustomConfig } from "./CustomConfig.js"; const defaultFilePathToUrl = (filePath: string) => { let url = filePath @@ -40,14 +41,42 @@ const resolveLinkToUrlPath = (link: string, sourceFilePath?: string) => { */ export class MarkdownDB { config: Knex.Config; + customConfig: CustomConfig = {}; + //@ts-ignore db: Knex; /** * Constructs a new MarkdownDB instance. - * @param {Knex.Config} config - Knex configuration object. + * @param {Knex.Config} knexConfig - Knex configuration object. + * @param {CustomConfig} [customConfig] - Custom configuration object. */ - constructor(config: Knex.Config) { - this.config = config; + constructor( + knexConfig: Knex.Config, + options?: { customConfig?: CustomConfig; configFilePath?: string } + ) { + this.config = knexConfig; + if (options?.customConfig) { + this.customConfig = options.customConfig; + } else { + this.loadConfiguration(options?.configFilePath); + } + } + + private async loadConfiguration(configFilePath?: string) { + const normalizedPath = path.resolve(configFilePath || "markdowndb.config.js"); + const fileUrl = new URL(`file://${normalizedPath}`); + + try { + // Import the module using the file URL + const configModule = await import(fileUrl.href); + this.customConfig = configModule.default; + } catch (error: any) { + if (configFilePath) { + throw new Error( + `Error loading configuration file from ${normalizedPath}: ${error.message}` + ); + } + } } /** @@ -82,6 +111,7 @@ export class MarkdownDB { const fileObjects = indexFolder( folderPath, pathToUrlResolver, + this.customConfig, ignorePatterns ); const filesToInsert = fileObjects.map(mapFileToInsert); diff --git a/src/lib/parseFile.ts b/src/lib/parseFile.ts index 953f7e8..fe207b3 100644 --- a/src/lib/parseFile.ts +++ b/src/lib/parseFile.ts @@ -28,6 +28,7 @@ export function parseFile(source: string, options?: ParsingOptions) { metadata.tasks = tasks; return { + ast, metadata, links, }; diff --git a/src/lib/process.ts b/src/lib/process.ts index 6a8f208..1e9a988 100644 --- a/src/lib/process.ts +++ b/src/lib/process.ts @@ -4,6 +4,7 @@ import path from "path"; import { File } from "./schema.js"; import { WikiLink, parseFile } from "./parseFile.js"; +import { Root } from "remark-parse/lib/index.js"; export interface FileInfo extends File { tags: string[]; @@ -15,8 +16,15 @@ export interface FileInfo extends File { export function processFile( rootFolder: string, filePath: string, - pathToUrlResolver: (filePath: string) => string, - filePathsToIndex: string[] + { + pathToUrlResolver = (filePath) => filePath, + filePathsToIndex = [], + customFields = [(fileInfo: FileInfo, ast: Root) => {}], + }: { + pathToUrlResolver: (filePath: string) => string; + filePathsToIndex: string[]; + customFields: ((fileInfo: FileInfo, ast: Root) => any)[]; + } ) { // Remove rootFolder from filePath const relativePath = path.relative(rootFolder, filePath); @@ -52,7 +60,7 @@ export function processFile( flag: "r", }); - const { metadata, links } = parseFile(source, { + const { ast, metadata, links } = parseFile(source, { from: relativePath, permalinks: filePathsToIndex, }); @@ -66,6 +74,10 @@ export function processFile( const tags = metadata?.tags || []; fileInfo.tags = tags; + for (let index = 0; index < customFields.length; index++) { + const customFieldFunction = customFields[index]; + customFieldFunction(fileInfo, ast); + } return fileInfo; } diff --git a/src/tests/extractWikiLinks.spec.ts b/src/tests/extractWikiLinks.spec.ts index ac9951f..703f7ce 100644 --- a/src/tests/extractWikiLinks.spec.ts +++ b/src/tests/extractWikiLinks.spec.ts @@ -1,4 +1,4 @@ -import { extractWikiLinks, processAST } from "../lib/parseFile"; +import { ParsingOptions, extractWikiLinks, processAST } from "../lib/parseFile"; // TODO test for links with headings and aliases ? // TODO test pdf embeds @@ -7,7 +7,7 @@ import { extractWikiLinks, processAST } from "../lib/parseFile"; // TODO test custom extractors // TODO test with other remark plugins e.g. original wiki links -const getLinksFromSource = (source: string, options?) => { +const getLinksFromSource = (source: string, options?: ParsingOptions) => { const from = "abc/foobar.md"; const ast = processAST(source, options); const links = extractWikiLinks(ast, { from: from, ...options }); diff --git a/src/tests/process.spec.ts b/src/tests/process.spec.ts index 43fd910..2bd7632 100644 --- a/src/tests/process.spec.ts +++ b/src/tests/process.spec.ts @@ -7,12 +7,11 @@ describe("Can parse a file and get file info", () => { test("can parse a file", async () => { const filePath = "index.mdx"; const fullPath = Path.join(pathToContentFixture, filePath); - const fileInfo = processFile( - pathToContentFixture, - fullPath, - (filePath) => filePath, - [] - ); + const fileInfo = processFile(pathToContentFixture, fullPath, { + pathToUrlResolver: (filePath) => filePath, + filePathsToIndex: [], + customFields: [], + }); expect(fileInfo.file_path).toBe(fullPath); expect(fileInfo.url_path).toBe("index.mdx"); diff --git a/tsconfig.json b/tsconfig.json index 98d0126..04f0e5c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,8 @@ "moduleResolution": "node", "esModuleInterop": true, "declaration": true, - "composite": true + "composite": true, + "strict": true }, "include": ["src/**/*.ts"], "references": [ diff --git a/tsconfig.lib.json b/tsconfig.lib.json index e578407..ffa03ee 100644 --- a/tsconfig.lib.json +++ b/tsconfig.lib.json @@ -18,5 +18,5 @@ "src/**/*.spec.ts", "src/**/*.test.ts" ], - "include": ["**/*.js", "**/*.ts"] + "include": ["**/*.js", "**/*.ts", "markdowndb.config.js"] }