From c0324964cbb44557dbfd9b432cf498d9821adb23 Mon Sep 17 00:00:00 2001 From: Ryo <36154873+RyoJerryYu@users.noreply.github.com> Date: Sat, 4 May 2024 17:28:12 +0800 Subject: [PATCH] feat: add cache meta collector --- src/core/indexing/indexing-cache.ts | 2 + src/core/indexing/indexing-settings.ts | 6 +- .../cache-meta-collector.test.ts | 14 ++++ .../meta-collecting/cache-meta-collector.ts | 82 +++++++++++++++++++ .../meta-collecting/meta-collecting.ts | 4 +- src/core/indexing/utils/git-utils.ts | 2 +- 6 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 src/core/indexing/meta-collecting/cache-meta-collector.test.ts create mode 100644 src/core/indexing/meta-collecting/cache-meta-collector.ts diff --git a/src/core/indexing/indexing-cache.ts b/src/core/indexing/indexing-cache.ts index 7cc4523b..96e10e69 100644 --- a/src/core/indexing/indexing-cache.ts +++ b/src/core/indexing/indexing-cache.ts @@ -24,7 +24,9 @@ export const initCache = async () => { if (cache) { return cache; } + console.log("init cache"); cache = await executePipeline(pipeline); + console.log("cache inited"); return cache; }; diff --git a/src/core/indexing/indexing-settings.ts b/src/core/indexing/indexing-settings.ts index f5638e91..c15c0e81 100644 --- a/src/core/indexing/indexing-settings.ts +++ b/src/core/indexing/indexing-settings.ts @@ -34,7 +34,11 @@ export const defaultStaticResourceChain: MetaCollectorChain = { }; export const defaultChain: MetaCollectorChain = { - collectors: [new PostRawMetaCollector(), defaultGitMetaCollector()], + collectors: [ + // new CacheMetaCollector(".", "cache", ["content"]), + new PostRawMetaCollector(), + defaultGitMetaCollector(), + ], defaultMeta: { content: "", title: "", diff --git a/src/core/indexing/meta-collecting/cache-meta-collector.test.ts b/src/core/indexing/meta-collecting/cache-meta-collector.test.ts new file mode 100644 index 00000000..ece4f9f9 --- /dev/null +++ b/src/core/indexing/meta-collecting/cache-meta-collector.test.ts @@ -0,0 +1,14 @@ +import fs from "fs"; +import git from "isomorphic-git"; +describe("test", () => { + it("should work", async () => { + const commits = await git.log({ + fs: fs, + dir: ".", + // depth: 1, + filepath: "public/content/articles/2020-01-27-Building-this-blog.md", + }); + + console.log(commits); + }); +}); diff --git a/src/core/indexing/meta-collecting/cache-meta-collector.ts b/src/core/indexing/meta-collecting/cache-meta-collector.ts new file mode 100644 index 00000000..1aeba786 --- /dev/null +++ b/src/core/indexing/meta-collecting/cache-meta-collector.ts @@ -0,0 +1,82 @@ +import { BaseMeta } from "@/core/types/indexing"; +import dayjs from "dayjs"; +import fs, { existsSync } from "fs"; +import { mkdir, readFile, writeFile } from "fs/promises"; +import git from "isomorphic-git"; +import { dirname } from "path"; +import { dayJsFromCommit } from "../utils/git-utils"; +import { MetaCollector } from "./meta-collecting"; + +export class CacheMetaCollector + implements MetaCollector +{ + constructor( + readonly gitDir: string, + readonly cacheDir: string, + readonly exceptFields: (keyof Meta)[] + ) {} + + filePathToCachePath = (filePath: string) => { + return `${this.cacheDir}/${filePath}.json`; + }; + + handleAbleKeys = (): "*" => { + return "*"; + }; + + updatedAtFromGit = async (filePath: string): Promise => { + const commits = await git.log({ fs, dir: this.gitDir, filepath: filePath }); + return dayJsFromCommit(commits[0]); + }; + + cacheExpired = async (filePath: string): Promise => { + try { + const fileUpdatedAt = await this.updatedAtFromGit(filePath); + const cachePath = this.filePathToCachePath(filePath); + if (!existsSync(cachePath)) { + return true; + } + const cacheUpdatedAt = await this.updatedAtFromGit(cachePath); + return fileUpdatedAt.isAfter(cacheUpdatedAt); + } catch (e) { + // any error occurs, treat as expired + return true; + } + }; + + collectMeta = async (filePath: string): Promise> => { + const cachePath = this.filePathToCachePath(filePath); + if (await this.cacheExpired(filePath)) { + return {}; + } + + console.log(`load cache for ${filePath} from ${cachePath}`); + const file = await readFile(cachePath, { encoding: "utf-8" }); + console.log(file); + const meta = JSON.parse(file); + if (this.exceptFields) { + for (const field of this.exceptFields) { + delete meta[field]; + } + } + return meta; + }; + defer = async (filePath: string, meta: Meta): Promise => { + const cachePath = this.filePathToCachePath(filePath); + if (!existsSync(dirname(cachePath))) { + console.log(`create cache dir for ${filePath} to ${cachePath}`); + await mkdir(dirname(cachePath), { recursive: true }); + } + console.log(`defer write cache for ${filePath} to ${cachePath}`); + const cacheMeta = { ...meta }; + if (this.exceptFields) { + for (const field of this.exceptFields) { + delete cacheMeta[field]; + } + } + await writeFile(cachePath, JSON.stringify(cacheMeta), { + flag: "w+", + encoding: "utf-8", + }); + }; +} diff --git a/src/core/indexing/meta-collecting/meta-collecting.ts b/src/core/indexing/meta-collecting/meta-collecting.ts index 3c752328..19d94b99 100644 --- a/src/core/indexing/meta-collecting/meta-collecting.ts +++ b/src/core/indexing/meta-collecting/meta-collecting.ts @@ -25,7 +25,7 @@ export interface MetaCollector { */ handleAbleKeys(): Array | "*"; collectMeta(filePath: string): Promise>; - defer?(meta: Meta): void; + defer?(filePath: string, meta: Meta): Promise; } /** @@ -77,7 +77,7 @@ export async function collectMetaForFilePath( if (collectorExecuteds[i]) { // only execute defer if the collector is executed const collector = collectors[i]; - collector.defer?.(fullMeta); + await collector.defer?.(filePath, fullMeta); } } diff --git a/src/core/indexing/utils/git-utils.ts b/src/core/indexing/utils/git-utils.ts index 59a4a36f..2121e240 100644 --- a/src/core/indexing/utils/git-utils.ts +++ b/src/core/indexing/utils/git-utils.ts @@ -1,7 +1,7 @@ import dayjs from "dayjs"; import { ReadCommitResult } from "isomorphic-git"; -function dayJsFromCommit(commit: ReadCommitResult): dayjs.Dayjs { +export function dayJsFromCommit(commit: ReadCommitResult): dayjs.Dayjs { return dayjs(commit.commit.committer.timestamp * 1000); }