fix(backend): replaced 'cavans' with '@napi-rs/canvas' already used b…

…y pdfjs-dist
lies-exposed · Jan 17, 2025 · 227d131 · 227d131
1 parent 0175f53
commit 227d131
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 53 deletions.
diff --git a/packages/@liexp/backend/package.json b/packages/@liexp/backend/package.json
@@ -31,7 +31,6 @@
     "@databases/sql": "^3.3.0",
     "@liexp/core": "workspace:*",
     "@liexp/shared": "workspace:*",
-    "canvas": "^3.0.1",
     "date-fns": "^4.1.0",
     "fp-ts": "^2.16.9",
     "io-ts": "^2.2.22",

diff --git a/packages/@liexp/backend/src/flows/media/thumbnails/extractThumbnailFromPDF.flow.ts b/packages/@liexp/backend/src/flows/media/thumbnails/extractThumbnailFromPDF.flow.ts
@@ -3,7 +3,7 @@ import {
   ImageType,
   type PDFType,
 } from "@liexp/shared/lib/io/http/Media/index.js";
-import Canvas from "canvas";
+import { createCanvas } from "@napi-rs/canvas";
 import * as TE from "fp-ts/lib/TaskEither.js";
 import {
   type PDFPageProxy,
@@ -27,7 +27,7 @@ export const extractThumbnailFromPDFPage = (
 
         const outputScale = 1;
 
-        const canvas = Canvas.createCanvas(viewport.width, viewport.height);
+        const canvas = createCanvas(viewport.width, viewport.height);
 
         const context = canvas.getContext("2d");
         canvas.height = viewport.height;
@@ -64,7 +64,9 @@ export const extractThumbnailFromPDF = <
     fp.RTE.flatMapTaskEither(fetchPDF(media.location)),
     fp.RTE.chainTaskEitherK((pdf) =>
       pipe(
-        [1, 2, 3],
+        pdf.numPages,
+        (p) => [1, p / 2, p].map(Math.floor),
+        fp.A.uniq(fp.N.Eq),
         fp.A.traverse(fp.TE.ApplicativePar)((page) =>
           pipe(
             fp.TE.tryCatch(() => pdf.getPage(page), ServerError.fromUnknown),

diff --git a/packages/@liexp/shared/src/providers/pdf/pdf.provider.ts b/packages/@liexp/shared/src/providers/pdf/pdf.provider.ts
@@ -34,6 +34,9 @@ const toError = (e: any): IOError => {
 
 interface PDFProviderContext {
   client: typeof pdf;
+  cMapUrl?: string;
+  cMapPacked?: boolean;
+  standardFontDataUrl?: string;
 }
 
 export interface PDFProvider {
@@ -47,7 +50,12 @@ export const PDFProvider = (ctx: PDFProviderContext): PDFProvider => {
   return {
     getDocument: (data) =>
       TE.tryCatch(() => {
-        return ctx.client.getDocument({ data, useSystemFonts: true }).promise;
+        return ctx.client.getDocument({
+          data,
+          cMapUrl: ctx.cMapUrl,
+          cMapPacked: ctx.cMapPacked,
+          standardFontDataUrl: ctx.standardFontDataUrl,
+        }).promise;
       }, toError),
     getAllTextContents(pdf) {
       return TE.tryCatch(async () => {

diff --git a/packages/@liexp/ui/src/components/admin/media/button/GenerateThumbnailButton.tsx b/packages/@liexp/ui/src/components/admin/media/button/GenerateThumbnailButton.tsx
@@ -51,7 +51,7 @@ const SelectThumbnailModalContent: React.FC<{
             <img
               src={t}
               style={{
-                width: extra.thumbnailWidth === 0 ? 400 : extra.thumbnailWidth,
+                width: extra.thumbnailWidth === 0 ? 200 : extra.thumbnailWidth,
                 height:
                   extra.thumbnailHeight === 0 ? "auto" : extra.thumbnailHeight,
               }}

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/services/api/src/context/context.type.ts b/services/api/src/context/context.type.ts
@@ -16,18 +16,13 @@ import { type SpaceContext } from "@liexp/backend/lib/context/space.context.js";
 import { type WikipediaProvider } from "@liexp/backend/lib/providers/wikipedia/wikipedia.provider.js";
 import { type ServerBlockNoteEditor } from "@liexp/shared/lib/providers/blocknote/ssr.js";
 import { type HTTPProvider } from "@liexp/shared/lib/providers/http/http.provider.js";
-import { type PDFProvider } from "@liexp/shared/lib/providers/pdf/pdf.provider.js";
 import { type AppConfig } from "#app/config.js";
 import { type ENV } from "#io/ENV.js";
 
 export interface ENVContext {
   env: ENV;
 }
 
-interface PDFProviderContext {
-  pdf: PDFProvider;
-}
-
 interface HTTPProviderContext {
   http: HTTPProvider;
 }
@@ -46,7 +41,6 @@ export type ServerContext = ENVContext &
   LoggerContext &
   SpaceContext &
   FSClientContext &
-  PDFProviderContext &
   HTTPProviderContext &
   WikipediaProviderContext &
   NERProviderContext &

diff --git a/services/api/src/context/index.ts b/services/api/src/context/index.ts
@@ -21,15 +21,13 @@ import { fromEndpoints } from "@liexp/shared/lib/providers/EndpointsRESTClient/E
 import { APIRESTClient } from "@liexp/shared/lib/providers/api-rest.provider.js";
 import { editor } from "@liexp/shared/lib/providers/blocknote/ssr.js";
 import { HTTPProvider } from "@liexp/shared/lib/providers/http/http.provider.js";
-import { PDFProvider } from "@liexp/shared/lib/providers/pdf/pdf.provider.js";
 import * as axios from "axios";
 import ffmpeg from "fluent-ffmpeg";
 import { sequenceS } from "fp-ts/lib/Apply.js";
 import { type TaskEither } from "fp-ts/lib/TaskEither.js";
 import { Redis } from "ioredis";
 import MW from "nodemw";
 import metadataParser from "page-metadata-parser";
-import * as pdf from "pdfjs-dist/legacy/build/pdf.mjs";
 import * as puppeteer from "puppeteer-core";
 import { type VanillaPuppeteer } from "puppeteer-extra";
 import WinkFn from "wink-nlp";
@@ -150,7 +148,6 @@ export const makeContext =
             apiKey: env.GEO_CODE_API_KEY,
           }),
         ),
-        pdf: fp.TE.right(PDFProvider({ client: pdf })),
         wp: fp.TE.right(wpProvider),
         rw: fp.TE.right(rationalWikiProvider),
         queue: fp.TE.right(GetQueueProvider(fsClient, config.dirs.temp.queue)),

diff --git a/services/api/src/flows/media/editMedia.flow.ts b/services/api/src/flows/media/editMedia.flow.ts
@@ -77,17 +77,20 @@ export const editMedia = (
                 thumbnail: undefined,
                 id,
               }),
-              fp.RTE.map(() => undefined),
+              fp.RTE.map(() => null),
             ),
           )
         : O.isSome(transferThumbnail) && m.thumbnail
-          ? MediaPubSub.TransferMediaFromExternalProviderPubSub.publish({
-              mediaId: m.id,
-              url: m.thumbnail,
-              fileName: `${m.id}-thumb`,
-              mimeType: m.type,
-            })
-          : fp.RTE.right(0),
+          ? pipe(
+              MediaPubSub.TransferMediaFromExternalProviderPubSub.publish({
+                mediaId: m.id,
+                url: m.thumbnail,
+                fileName: `${m.id}-thumb`,
+                mimeType: m.type,
+              }),
+              fp.RTE.map(() => m.thumbnail),
+            )
+          : fp.RTE.right(fp.O.getOrElse(() => m.thumbnail)(thumbnail)),
     ),
     fp.RTE.bind("location", ({ media }) =>
       O.isSome(transfer)
@@ -102,13 +105,14 @@ export const editMedia = (
           )
         : fp.RTE.right(location),
     ),
-    fp.RTE.chain(({ media, location }) =>
+    fp.RTE.chain(({ media, location, thumbnail }) =>
       pipe(
         MediaRepository.save<ServerContext>([
           {
             ...media,
             ...body,
             location,
+            thumbnail,
             keywords: body.keywords.map((id) => ({ id })),
             links: body.links.map((id) => ({ id })),
             events: body.events.map((id) => ({

diff --git a/services/api/test/AppTest.ts b/services/api/test/AppTest.ts
@@ -25,7 +25,6 @@ import { GetPuppeteerProvider } from "@liexp/backend/lib/providers/puppeteer.pro
 import { MakeSpaceProvider } from "@liexp/backend/lib/providers/space/space.provider.js";
 import { GetLogger, Logger } from "@liexp/core/lib/logger/index.js";
 import { HTTPProvider } from "@liexp/shared/lib/providers/http/http.provider.js";
-import { PDFProvider } from "@liexp/shared/lib/providers/pdf/pdf.provider.js";
 import { throwTE } from "@liexp/shared/lib/utils/task.utils.js";
 import { AxiosInstance } from "axios";
 import { sequenceS, sequenceT } from "fp-ts/lib/Apply.js";
@@ -129,7 +128,6 @@ export const loadAppContext = async (
       openai: {} as any,
       blocknote: {} as any,
       redis: mocks.redis,
-      pdf: PDFProvider({ client: {} as any }),
       geo: GeocodeProvider({
         http: HTTPProvider(mocks.axios as any),
         apiKey: "fake-geo-api-key",

diff --git a/services/worker/src/config.ts b/services/worker/src/config.ts
@@ -1,4 +1,3 @@
-import { mkdirSync } from "fs";
 import path from "path";
 import { type BEConfig } from "@liexp/backend/lib/context/config.context.js";
 import { type FSClientContext } from "@liexp/backend/lib/context/fs.context.js";
@@ -10,7 +9,14 @@ import { type ReaderTaskEither } from "fp-ts/lib/ReaderTaskEither.js";
 import { pipe } from "fp-ts/lib/function.js";
 import { type WorkerError } from "./io/worker.error.js";
 
-export type WorkerConfig = BEConfig;
+export interface WorkerConfig extends BEConfig {
+  dirs: BEConfig["dirs"] & {
+    pdf: {
+      cMapUrl: string;
+      standardFontDataUrl: string;
+    };
+  };
+}
 
 export const Config =
   <C extends FSClientContext & LoggerContext>(
@@ -19,6 +25,14 @@ export const Config =
   (ctx) => {
     const tempRoot = path.resolve(cwd, "temp");
 
+    const pdfJSDistPath = path.resolve(cwd, "../../node_modules/pdfjs-dist/");
+    const pdfFolders = {
+      cMapUrl: path.resolve(pdfJSDistPath, "cmaps").concat(path.sep),
+      standardFontDataUrl: path
+        .resolve(pdfJSDistPath, "standard_fonts")
+        .concat(path.sep),
+    };
+
     const configFolders = {
       nlp: path.resolve(cwd, "config", "nlp"),
     };
@@ -32,14 +46,12 @@ export const Config =
       tg: path.resolve(tempRoot, "tg"),
     };
 
-    [...Object.values(configFolders), ...Object.values(tempFolders)].forEach(
-      (folder) => {
-        mkdirSync(folder, { recursive: true });
-      },
-    );
-
     return pipe(
-      [...Object.values(configFolders), ...Object.values(tempFolders)],
+      [
+        ...Object.values(configFolders),
+        ...Object.values(tempFolders),
+        ...Object.values(pdfFolders),
+      ],
       fp.A.traverse(fp.TE.ApplicativePar)((filePath) =>
         ensureFolderExists(filePath)(ctx),
       ),
@@ -50,6 +62,7 @@ export const Config =
         },
         dirs: {
           cwd,
+          pdf: pdfFolders,
           config: configFolders,
           temp: tempFolders,
         },