feat: add cache meta collector

src/core/indexing/indexing-cache.ts

@@ -24,7 +24,9 @@ export const initCache = async () => {
   if (cache) {
     return cache;
   }
+  console.log("init cache");
   cache = await executePipeline(pipeline);
+  console.log("cache inited");
 
   return cache;
 };

src/core/indexing/indexing-settings.ts

@@ -34,7 +34,11 @@ export const defaultStaticResourceChain: MetaCollectorChain<BaseMeta> = {
 };
 
 export const defaultChain: MetaCollectorChain<PostMeta> = {
-  collectors: [new PostRawMetaCollector(), defaultGitMetaCollector()],
+  collectors: [
+    // new CacheMetaCollector(".", "cache", ["content"]),
+    new PostRawMetaCollector(),
+    defaultGitMetaCollector(),
+  ],
   defaultMeta: {
     content: "",
     title: "<No Title>",

src/core/indexing/meta-collecting/cache-meta-collector.test.ts

@@ -0,0 +1,14 @@
+import fs from "fs";
+import git from "isomorphic-git";
+describe("test", () => {
+  it("should work", async () => {
+    const commits = await git.log({
+      fs: fs,
+      dir: ".",
+      // depth: 1,
+      filepath: "public/content/articles/2020-01-27-Building-this-blog.md",
+    });
+
+    console.log(commits);
+  });
+});

src/core/indexing/meta-collecting/cache-meta-collector.ts

@@ -0,0 +1,82 @@
+import { BaseMeta } from "@/core/types/indexing";
+import dayjs from "dayjs";
+import fs, { existsSync } from "fs";
+import { mkdir, readFile, writeFile } from "fs/promises";
+import git from "isomorphic-git";
+import { dirname } from "path";
+import { dayJsFromCommit } from "../utils/git-utils";
+import { MetaCollector } from "./meta-collecting";
+
+export class CacheMetaCollector<Meta extends BaseMeta>
+  implements MetaCollector<Meta>
+{
+  constructor(
+    readonly gitDir: string,
+    readonly cacheDir: string,
+    readonly exceptFields: (keyof Meta)[]
+  ) {}
+
+  filePathToCachePath = (filePath: string) => {
+    return `${this.cacheDir}/${filePath}.json`;
+  };
+
+  handleAbleKeys = (): "*" => {
+    return "*";
+  };
+
+  updatedAtFromGit = async (filePath: string): Promise<dayjs.Dayjs> => {
+    const commits = await git.log({ fs, dir: this.gitDir, filepath: filePath });
+    return dayJsFromCommit(commits[0]);
+  };
+
+  cacheExpired = async (filePath: string): Promise<boolean> => {
+    try {
+      const fileUpdatedAt = await this.updatedAtFromGit(filePath);
+      const cachePath = this.filePathToCachePath(filePath);
+      if (!existsSync(cachePath)) {
+        return true;
+      }
+      const cacheUpdatedAt = await this.updatedAtFromGit(cachePath);
+      return fileUpdatedAt.isAfter(cacheUpdatedAt);
+    } catch (e) {
+      // any error occurs, treat as expired
+      return true;
+    }
+  };
+
+  collectMeta = async (filePath: string): Promise<Partial<Meta>> => {
+    const cachePath = this.filePathToCachePath(filePath);
+    if (await this.cacheExpired(filePath)) {
+      return {};
+    }
+
+    console.log(`load cache for ${filePath} from ${cachePath}`);
+    const file = await readFile(cachePath, { encoding: "utf-8" });
+    console.log(file);
+    const meta = JSON.parse(file);
+    if (this.exceptFields) {
+      for (const field of this.exceptFields) {
+        delete meta[field];
+      }
+    }
+    return meta;
+  };
+  defer = async (filePath: string, meta: Meta): Promise<void> => {
+    const cachePath = this.filePathToCachePath(filePath);
+    if (!existsSync(dirname(cachePath))) {
+      console.log(`create cache dir for ${filePath} to ${cachePath}`);
+      await mkdir(dirname(cachePath), { recursive: true });
+    }
+    console.log(`defer write cache for ${filePath} to ${cachePath}`);
+    const cacheMeta = { ...meta };
+    if (this.exceptFields) {
+      for (const field of this.exceptFields) {
+        delete cacheMeta[field];
+      }
+    }
+    await writeFile(cachePath, JSON.stringify(cacheMeta), {
+      flag: "w+",
+      encoding: "utf-8",
+    });
+  };
+}

src/core/indexing/meta-collecting/meta-collecting.ts

@@ -25,7 +25,7 @@ export interface MetaCollector<Meta extends BaseMeta> {
    */
   handleAbleKeys(): Array<keyof Meta> | "*";
   collectMeta(filePath: string): Promise<Partial<Meta>>;
-  defer?(meta: Meta): void;
+  defer?(filePath: string, meta: Meta): Promise<void>;
 }
 
 /**
@@ -77,7 +77,7 @@ export async function collectMetaForFilePath<Meta extends BaseMeta>(
     if (collectorExecuteds[i]) {
       // only execute defer if the collector is executed
       const collector = collectors[i];
-      collector.defer?.(fullMeta);
+      await collector.defer?.(filePath, fullMeta);
     }
   }
 

src/core/indexing/utils/git-utils.ts

@@ -1,7 +1,7 @@
 import dayjs from "dayjs";
 import { ReadCommitResult } from "isomorphic-git";
 
-function dayJsFromCommit(commit: ReadCommitResult): dayjs.Dayjs {
+export function dayJsFromCommit(commit: ReadCommitResult): dayjs.Dayjs {
   return dayjs(commit.commit.committer.timestamp * 1000);
 }