perf(PostCategory/PostTag): add binary relation index for performance

lib/hexo/index.ts

@@ -40,6 +40,7 @@ import type Box from '../box';
 import type { AssetGenerator, LocalsType, NodeJSLikeCallback, NormalPageGenerator, NormalPostGenerator, PageGenerator, PostGenerator, SiteLocals } from '../types';
 import type { AddSchemaTypeOptions } from 'warehouse/dist/types';
 import type Schema from 'warehouse/dist/schema';
+import BinaryRelationIndex from '../models/binary_relation_index';
 
 const libDir = dirname(__dirname);
 const dbVersion = 1;
@@ -276,6 +277,10 @@ class Hexo extends EventEmitter {
   static lib_dir: string;
   static core_dir: string;
   static version: string;
+  public _binaryRelationIndex: {
+    post_tag: BinaryRelationIndex<'post_id', 'tag_id'>;
+    post_category: BinaryRelationIndex<'post_id', 'category_id'>;
+  };
 
   constructor(base = process.cwd(), args: Args = {}) {
     super();
@@ -354,6 +359,10 @@ class Hexo extends EventEmitter {
     this.theme = new Theme(this);
     this.locals = new Locals();
     this._bindLocals();
+    this._binaryRelationIndex = {
+      post_tag: new BinaryRelationIndex<'post_id', 'tag_id'>('post_id', 'tag_id', 'PostTag', this),
+      post_category: new BinaryRelationIndex<'post_id', 'category_id'>('post_id', 'category_id', 'PostCategory', this)
+    };
   }
 
   _bindLocals(): void {
@@ -493,6 +502,8 @@ class Hexo extends EventEmitter {
 
   load(callback?: NodeJSLikeCallback<any>): Promise<any> {
     return loadDatabase(this).then(() => {
+      this._binaryRelationIndex.post_tag.load();
+      this._binaryRelationIndex.post_category.load();
       this.log.info('Start processing');
 
       return Promise.all([

lib/models/binary_relation_index.ts

@@ -0,0 +1,100 @@
+import type Hexo from '../hexo';
+
+type BinaryRelationType<K extends PropertyKey, V extends PropertyKey> = {
+  [key in K]: PropertyKey;
+} & {
+  [key in V]: PropertyKey;
+};
+
+class BinaryRelationIndex<K extends PropertyKey, V extends PropertyKey> {
+  keyIndex: Map<PropertyKey, Set<PropertyKey>> = new Map();
+  valueIndex: Map<PropertyKey, Set<PropertyKey>> = new Map();
+  key: K;
+  value: V;
+  ctx: Hexo;
+  schemaName: string;
+
+  constructor(key: K, value: V, schemaName: string, ctx: Hexo) {
+    this.key = key;
+    this.value = value;
+    this.schemaName = schemaName;
+    this.ctx = ctx;
+  }
+
+  load() {
+    this.keyIndex.clear();
+    this.valueIndex.clear();
+    const raw = this.ctx.model(this.schemaName).data;
+    for (const _id in raw) {
+      this.saveHook(raw[_id]);
+    }
+  }
+
+  saveHook(data: BinaryRelationType<K, V> & { _id: PropertyKey }) {
+    const _id = data._id;
+    const key = data[this.key];
+    const value = data[this.value];
+    if (!this.keyIndex.has(key)) {
+      this.keyIndex.set(key, new Set());
+    }
+    this.keyIndex.get(key).add(_id);
+
+    if (!this.valueIndex.has(value)) {
+      this.valueIndex.set(value, new Set());
+    }
+    this.valueIndex.get(value).add(_id);
+  }
+
+  removeHook(data: BinaryRelationType<K, V> & { _id: PropertyKey }) {
+    const _id = data._id;
+    const key = data[this.key];
+    const value = data[this.value];
+    this.keyIndex.get(key)?.delete(_id);
+    if (this.keyIndex.get(key)?.size === 0) {
+      this.keyIndex.delete(key);
+    }
+    this.valueIndex.get(value)?.delete(_id);
+    if (this.valueIndex.get(value)?.size === 0) {
+      this.valueIndex.delete(value);
+    }
+  }
+
+  findById(_id: PropertyKey) {
+    const raw = this.ctx.model(this.schemaName).findById(_id, { lean: true });
+    if (!raw) return;
+    return { ...raw };
+  }
+
+  find(query: Partial<BinaryRelationType<K, V>>) {
+    const key = query[this.key];
+    const value = query[this.value];
+
+    if (key && value) {
+      const ids = this.keyIndex.get(key);
+      if (!ids) return [];
+      return Array.from(ids)
+        .map(_id => this.findById(_id))
+        .filter(record => record?.[this.value] === value);
+    }
+
+    if (key) {
+      const ids = this.keyIndex.get(key);
+      if (!ids) return [];
+      return Array.from(ids).map(_id => this.findById(_id));
+    }
+
+    if (value) {
+      const ids = this.valueIndex.get(value);
+      if (!ids) return [];
+      return Array.from(ids).map(_id => this.findById(_id));
+    }
+
+    return [];
+  }
+
+  findOne(query: Partial<BinaryRelationType<K, V>>) {
+    return this.find(query)[0];
+  }
+}
+
+export default BinaryRelationIndex;

lib/models/category.ts

@@ -41,19 +41,19 @@ export = (ctx: Hexo) => {
   });
 
   Category.virtual('posts').get(function() {
-    const PostCategory = ctx.model('PostCategory');
+    const ReadOnlyPostCategory = ctx._binaryRelationIndex.post_category;
 
-    const ids = PostCategory.find({category_id: this._id}).map(item => item.post_id);
+    const ids = ReadOnlyPostCategory.find({category_id: this._id}).map(item => item.post_id);
 
     return ctx.locals.get('posts').find({
       _id: {$in: ids}
     });
   });
 
   Category.virtual('length').get(function() {
-    const PostCategory = ctx.model('PostCategory');
+    const ReadOnlyPostCategory = ctx._binaryRelationIndex.post_category;
 
-    return PostCategory.find({category_id: this._id}).length;
+    return ReadOnlyPostCategory.find({category_id: this._id}).length;
   });
 
   // Check whether a category exists

lib/models/post.ts

@@ -64,10 +64,10 @@ export = (ctx: Hexo) => {
 
   Post.virtual('tags').get(function() {
     return tagsGetterCache.apply(this._id, () => {
-      const PostTag = ctx.model('PostTag');
+      const ReadOnlyPostTag = ctx._binaryRelationIndex.post_tag;
       const Tag = ctx.model('Tag');
 
-      const ids = PostTag.find({post_id: this._id}, {lean: true}).map(item => item.tag_id);
+      const ids = ReadOnlyPostTag.find({post_id: this._id}).map(item => item.tag_id);
 
       return Tag.find({_id: {$in: ids}});
     });
@@ -87,10 +87,11 @@ export = (ctx: Hexo) => {
     tagsGetterCache.flush();
     tags = removeEmptyTag(tags);
 
+    const ReadOnlyPostTag = ctx._binaryRelationIndex.post_tag;
     const PostTag = ctx.model('PostTag');
     const Tag = ctx.model('Tag');
     const id = this._id;
-    const existed = PostTag.find({post_id: id}, {lean: true}).map(pickID);
+    const existed = ReadOnlyPostTag.find({post_id: id}).map(pickID);
 
     return Promise.map(tags, tag => {
       // Find the tag by name
@@ -107,7 +108,7 @@ export = (ctx: Hexo) => {
       });
     }).map(tag => {
       // Find the reference
-      const ref = PostTag.findOne({post_id: id, tag_id: tag._id}, {lean: true});
+      const ref = ReadOnlyPostTag.findOne({post_id: id, tag_id: tag._id});
       if (ref) return ref;
 
       // Insert the reference if not exist
@@ -123,10 +124,10 @@ export = (ctx: Hexo) => {
   });
 
   Post.virtual('categories').get(function() {
-    const PostCategory = ctx.model('PostCategory');
+    const ReadOnlyPostCategory = ctx._binaryRelationIndex.post_category;
     const Category = ctx.model('Category');
 
-    const ids = PostCategory.find({post_id: this._id}, {lean: true}).map(item => item.category_id);
+    const ids = ReadOnlyPostCategory.find({post_id: this._id}).map(item => item.category_id);
 
     return Category.find({_id: {$in: ids}});
   });
@@ -142,11 +143,12 @@ export = (ctx: Hexo) => {
       return Array.isArray(cat) ? removeEmptyTag(cat) : `${cat}`;
     });
 
+    const ReadOnlyPostCategory = ctx._binaryRelationIndex.post_category;
     const PostCategory = ctx.model('PostCategory');
     const Category = ctx.model('Category');
     const id = this._id;
     const allIds = [];
-    const existed = PostCategory.find({post_id: id}, {lean: true}).map(pickID);
+    const existed = ReadOnlyPostCategory.find({post_id: id}).map(pickID);
     const hasHierarchy = cats.filter(Array.isArray).length > 0;
 
     // Add a hierarchy of categories
@@ -192,7 +194,7 @@ export = (ctx: Hexo) => {
     return (hasHierarchy ? Promise.each(cats, addHierarchy) : Promise.resolve(addHierarchy(cats))
     ).then(() => allIds).map(catId => {
       // Find the reference
-      const ref = PostCategory.findOne({post_id: id, category_id: catId}, {lean: true});
+      const ref = ReadOnlyPostCategory.findOne({post_id: id, category_id: catId});
       if (ref) return ref;
 
       // Insert the reference if not exist

lib/models/post_category.ts

@@ -7,5 +7,20 @@ export = (ctx: Hexo) => {
     category_id: {type: warehouse.Schema.Types.CUID, ref: 'Category'}
   });
 
+  PostCategory.pre('save', data => {
+    ctx._binaryRelationIndex.post_category.removeHook(data);
+    return data;
+  });
+
+  PostCategory.post('save', data => {
+    ctx._binaryRelationIndex.post_category.saveHook(data);
+    return data;
+  });
+
+  PostCategory.pre('remove', data => {
+    ctx._binaryRelationIndex.post_category.removeHook(data);
+    return data;
+  });
+
   return PostCategory;
 };

lib/models/post_tag.ts

@@ -7,5 +7,20 @@ export = (ctx: Hexo) => {
     tag_id: {type: warehouse.Schema.Types.CUID, ref: 'Tag'}
   });
 
+  PostTag.pre('save', data => {
+    ctx._binaryRelationIndex.post_tag.removeHook(data);
+    return data;
+  });
+
+  PostTag.post('save', data => {
+    ctx._binaryRelationIndex.post_tag.saveHook(data);
+    return data;
+  });
+
+  PostTag.pre('remove', data => {
+    ctx._binaryRelationIndex.post_tag.removeHook(data);
+    return data;
+  });
+
   return PostTag;
 };

lib/models/tag.ts

@@ -32,9 +32,9 @@ export = (ctx: Hexo) => {
   });
 
   Tag.virtual('posts').get(function() {
-    const PostTag = ctx.model('PostTag');
+    const ReadOnlyPostTag = ctx._binaryRelationIndex.post_tag;
 
-    const ids = PostTag.find({tag_id: this._id}).map(item => item.post_id);
+    const ids = ReadOnlyPostTag.find({tag_id: this._id}).map(item => item.post_id);
 
     return ctx.locals.get('posts').find({
       _id: {$in: ids}
@@ -44,9 +44,9 @@ export = (ctx: Hexo) => {
   Tag.virtual('length').get(function() {
     // Note: this.posts.length is also working
     // But it's slow because `find` has to iterate over all posts
-    const PostTag = ctx.model('PostTag');
+    const ReadOnlyPostTag = ctx._binaryRelationIndex.post_tag;
 
-    return PostTag.find({tag_id: this._id}).length;
+    return ReadOnlyPostTag.find({tag_id: this._id}).length;
   });
 
   // Check whether a tag exists

test/scripts/models/category.ts

@@ -6,6 +6,7 @@ describe('Category', () => {
   const hexo = new Hexo();
   const Category = hexo.model('Category');
   const Post = hexo.model('Post');
+  const ReadOnlyPostCategory = hexo._binaryRelationIndex.post_category;
   const PostCategory = hexo.model('PostCategory');
 
   before(() => hexo.init());
@@ -283,6 +284,7 @@ describe('Category', () => {
     await Category.removeById(cat._id!);
 
     PostCategory.find({category_id: cat._id}).should.have.lengthOf(0);
+    ReadOnlyPostCategory.find({category_id: cat._id}).should.have.lengthOf(0);
 
     await Promise.all(posts.map(post => post.remove()));
   });

test/scripts/models/post.ts

@@ -11,6 +11,8 @@ describe('Post', () => {
   const Post = hexo.model('Post');
   const Tag = hexo.model('Tag');
   const Category = hexo.model('Category');
+  const ReadOnlyPostTag = hexo._binaryRelationIndex.post_tag;
+  const ReadOnlyPostCategory = hexo._binaryRelationIndex.post_category;
   const PostTag = hexo.model('PostTag');
   const PostCategory = hexo.model('PostCategory');
   const Asset = hexo.model('Asset');
@@ -427,6 +429,7 @@ describe('Post', () => {
   }).then(post => post.setTags(['foo', 'bar', 'baz'])
     .thenReturn(Post.findById(post._id))).then(post => Post.removeById(post._id)).then(post => {
     PostTag.find({post_id: post._id}).should.have.lengthOf(0);
+    ReadOnlyPostTag.find({post_id: post._id}).should.have.lengthOf(0);
     Tag.findOne({name: 'foo'}).posts.should.have.lengthOf(0);
     Tag.findOne({name: 'bar'}).posts.should.have.lengthOf(0);
     Tag.findOne({name: 'baz'}).posts.should.have.lengthOf(0);
@@ -438,6 +441,7 @@ describe('Post', () => {
   }).then(post => post.setCategories(['foo', 'bar', 'baz'])
     .thenReturn(Post.findById(post._id))).then(post => Post.removeById(post._id)).then(post => {
     PostCategory.find({post_id: post._id}).should.have.lengthOf(0);
+    ReadOnlyPostCategory.find({post_id: post._id}).should.have.lengthOf(0);
     Category.findOne({name: 'foo'}).posts.should.have.lengthOf(0);
     Category.findOne({name: 'bar'}).posts.should.have.lengthOf(0);
     Category.findOne({name: 'baz'}).posts.should.have.lengthOf(0);

test/scripts/models/tag.ts

@@ -7,6 +7,7 @@ describe('Tag', () => {
   const Tag = hexo.model('Tag');
   const Post = hexo.model('Post');
   const PostTag = hexo.model('PostTag');
+  const ReadOnlyPostTag = hexo._binaryRelationIndex.post_tag;
 
   before(() => hexo.init());
 
@@ -244,6 +245,7 @@ describe('Tag', () => {
     await Tag.removeById(tag._id!);
 
     PostTag.find({tag_id: tag._id}).should.have.lengthOf(0);
+    ReadOnlyPostTag.find({tag_id: tag._id}).should.have.lengthOf(0);
 
     await Promise.all(posts.map(post => Post.removeById(post._id)));
   });