Release Daily -> Alpha

.github/dependabot.yml

@@ -1,6 +1,15 @@
 ---
 version: 2
 updates:
+  # ignore all updates for the @liexp/packages
+  - package-ecosystem: "npm"
+    directories:
+      - "/packages/@liexp/*"
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "*"
+
   - package-ecosystem: "npm"
     directories:
       - "/"
@@ -11,6 +20,8 @@ updates:
       - "/packages/@liexp/ui"
       - "/services/api"
       - "/services/web"
+      - "/services/ai-bot"
+      - "/services/worker"
       - "/services/admin-web"
       - "/services/storybook"
     schedule:

.github/workflows/packages-pull-request.yml

@@ -30,10 +30,12 @@ jobs:
           pnpm core lint
           pnpm shared lint
           pnpm test lint
+          pnpm backend lint
           pnpm ui lint
 
       - name: Run tests
         env:
           DEBUG: "-@liexp*"
         run: |
           pnpm shared test
+          pnpm backend test

.zed/settings.json

@@ -0,0 +1,20 @@
+// Folder-specific settings
+//
+// For a full list of overridable settings, and general information on folder-specific settings,
+// see the documentation: https://zed.dev/docs/configuring-zed#settings-files
+{
+  "file_scan_exclusions": [
+    "**/.git",
+    "**/.svn",
+    "**/.hg",
+    "**/.jj",
+    "**/CVS",
+    "**/.DS_Store",
+    "**/Thumbs.db",
+    "**/.classpath",
+    "**/.settings",
+    "**/lib",
+    "**/build",
+    "**/node_modules"
+  ]
+}

compose.yml

@@ -81,6 +81,7 @@ services:
       - api-node-modules:/usr/src/app/node_modules
       - ./packages/@liexp:/usr/src/app/packages/@liexp:ro
       - ./services/api:/usr/src/app/services/api:cached
+      - ./services/worker/config:/usr/src/app/services/api/config:ro
     working_dir: /usr/src/app/services/api
 
   api.liexp.dev:
@@ -124,7 +125,8 @@ services:
       - worker-node-modules:/usr/src/app/node_modules
       - ./packages/@liexp:/usr/src/app/packages/@liexp:ro
       - ./services/worker:/usr/src/app/services/worker:ro
-      - ./services/worker/temp:/usr/src/app/services/worker/temp:rw
+      - ./services/worker/config:/usr/src/app/services/worker/config:cached
+      - ./services/worker/temp:/usr/src/app/services/worker/temp:cached
     mem_limit: 512M
     restart: always
     networks:

deploy/compose.yml

@@ -28,13 +28,15 @@ services:
       timeout: 10s
       retries: 10
       start_period: 30s
+    restart: always
     mem_limit: 1G
     depends_on:
       - redis
 
   be-worker:
     image: ghcr.io/lies-exposed/liexp-worker:alpha-latest
     container_name: liexp-be-worker
+    restart: always
     environment:
       TG_BOT_POLLING: "true"
       REDIS_HOST: "127.0.0.1"

lies.exposed.code-workspace

@@ -63,6 +63,7 @@
       "**/bower_components": true,
       "**/*.code-search": true,
       "build/**": true,
+      "coverage/**": true,
       "lib/**": true,
     },
     "eslint.workingDirectories": [

package.json

@@ -43,17 +43,17 @@
   "devDependencies": {
     "@commitlint/cli": "^19.6.0",
     "@commitlint/config-conventional": "^19.6.0",
-    "@types/node": "^22.10.2",
+    "@types/node": "^22.10.7",
     "@vitest/coverage-v8": "^2.1.8",
     "eslint": "^9.18.0",
     "husky": "^9.1.7",
     "knip": "^5.39.4",
     "lint-staged": "^15.2.11",
     "typescript": "^5.6.3",
-    "typescript-eslint": "^8.18.0",
-    "vitest": "^2.1.8"
+    "typescript-eslint": "^8.20.0",
+    "vitest": "^3.0.3"
   },
-  "packageManager": "[email protected].0+sha512.76e2379760a4328ec4415815bcd6628dee727af3779aaa4c914e3944156c4299921a89f976381ee107d41f12cfa4b66681ca9c718f0668fa0831ed4c6d8ba56c",
+  "packageManager": "[email protected].4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0",
   "engines": {
     "node": ">= 22",
     "npm": "please use pnpm >= 9",

packages/@liexp/backend/package.json

@@ -24,9 +24,9 @@
     "src/**/*.(ts|tsx|js|jsx)": "eslint --fix"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.726.1",
-    "@aws-sdk/lib-storage": "^3.726.1",
-    "@aws-sdk/s3-request-presigner": "^3.726.1",
+    "@aws-sdk/client-s3": "^3.732.0",
+    "@aws-sdk/lib-storage": "^3.732.0",
+    "@aws-sdk/s3-request-presigner": "^3.732.0",
     "@databases/escape-identifier": "^1.0.3",
     "@databases/sql": "^3.3.0",
     "@liexp/core": "workspace:*",
@@ -43,8 +43,8 @@
   },
   "devDependencies": {
     "@liexp/test": "workspace:*",
-    "@aws-sdk/types": "^3.723.0",
-    "@langchain/core": "^0.3.29",
+    "@aws-sdk/types": "^3.731.0",
+    "@langchain/core": "^0.3.30",
     "@langchain/openai": "^0.3.17",
     "@types/fluent-ffmpeg": "^2.1.27",
     "@types/jsdom": "^21.1.7",
@@ -60,7 +60,7 @@
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "typescript": "^5.6.2",
     "vite-tsconfig-paths": "^5.1.4",
-    "vitest": "^2.1.8",
+    "vitest": "^3.0.3",
     "vitest-mock-extended": "^2.0.2"
   },
   "peerDependencies": {
@@ -83,5 +83,5 @@
     "wink-nlp": "^2.3.0",
     "wink-nlp-utils": "^2.1.0"
   },
-  "packageManager": "[email protected].3+sha512.1f79bc245a66eb0b07c5d4d83131240774642caaa86ef7d0434ab47c0d16f66b04e21e0c086eb61e62c77efc4d7f7ec071afad3796af64892fae66509173893a"
+  "packageManager": "[email protected].4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0"
 }

packages/@liexp/backend/src/context/api.context.ts

@@ -1,5 +1,5 @@
 import { type Endpoints } from "@liexp/shared/lib/endpoints/index.js";
-import { type EndpointsRESTClient } from "@liexp/shared/lib/providers/EndpointsRESTClient/types";
+import { type EndpointsRESTClient } from "@liexp/shared/lib/providers/EndpointsRESTClient/types.js";
 
 export interface APIContext {
   api: EndpointsRESTClient<Endpoints>;

packages/@liexp/backend/src/context/index.ts

@@ -1,4 +1,3 @@
-import { type URLMetadataClient } from "../providers/URLMetadata.provider.js";
 import { type FFMPEGProvider } from "../providers/ffmpeg/ffmpeg.provider.js";
 import { type GeocodeProvider } from "../providers/geocode/geocode.provider.js";
 import { type IGProvider } from "../providers/ig/ig.provider.js";
@@ -7,10 +6,6 @@ import { type NERProvider } from "../providers/ner/ner.provider.js";
 import { type TGBotProvider } from "../providers/tg/tg.provider.js";
 import { type WikipediaProvider } from "../providers/wikipedia/wikipedia.provider.js";
 
-export interface URLMetadataContext {
-  urlMetadata: URLMetadataClient;
-}
-
 export interface FFMPEGProviderContext {
   ffmpeg: FFMPEGProvider;
 }

packages/@liexp/backend/src/context/urlMetadata.context.ts

@@ -0,0 +1,5 @@
+import type { URLMetadataClient } from "../providers/URLMetadata.provider.js";
+
+export interface URLMetadataContext {
+  urlMetadata: URLMetadataClient;
+}

packages/@liexp/backend/src/errors/index.ts

@@ -3,4 +3,4 @@ export * from "./NotAuthorizedError.js";
 export * from "./NotFoundError.js";
 export * from "./ServerError.js";
 
-export { IOError } from "ts-io-error/lib/index.js";
+export { IOError } from "ts-io-error";

packages/@liexp/backend/src/flows/admin/nlp/upsertEntities.flow.ts

@@ -18,6 +18,7 @@ import { type DBError } from "../../../providers/orm/database.provider.js";
 import { fetchActors } from "../../../queries/actors/fetchActors.query.js";
 import { fetchGroups } from "../../../queries/groups/fetchGroups.query.js";
 import { fetchKeywords } from "../../../queries/keywords/fetchKeywords.query.js";
+import { ensureFolderExists } from "../../fs/ensureFolderExists.flow.js";
 
 const makePatterns = (s: string, acronym: boolean): string[] => {
   const chunks = s.split(" ");
@@ -130,7 +131,8 @@ export const upsertNLPEntities = <
       ];
 
       return pipe(
-        ctx.fs.writeObject(nplConfig, JSON.stringify(entities, null, 4)),
+        ensureFolderExists(ctx.config.dirs.config.nlp)(ctx),
+        TE.chain(() => ctx.fs.writeObject(nplConfig, JSON.stringify(entities))),
         TE.map(() => entities),
         TE.mapLeft(ServerError.fromUnknown),
       );

packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts

@@ -0,0 +1,132 @@
+import { fp, pipe } from "@liexp/core/lib/fp/index.js";
+import { uuid } from "@liexp/shared/lib/io/http/Common/UUID.js";
+import { SCIENTIFIC_STUDY } from "@liexp/shared/lib/io/http/Events/EventType.js";
+import { HumanReadableStringArb } from "@liexp/shared/lib/tests/arbitrary/HumanReadableString.arbitrary.js";
+import { throwTE } from "@liexp/shared/lib/utils/task.utils.js";
+import { sanitizeURL } from "@liexp/shared/lib/utils/url.utils.js";
+import { fc } from "@liexp/test";
+import { describe, expect, it } from "vitest";
+import { mockDeep } from "vitest-mock-extended";
+import { EventV2Entity } from "../../entities/Event.v2.entity.js";
+import { UserEntity } from "../../entities/User.entity.js";
+import { initContext } from "../../test/index.js";
+import { mockedContext, mockTERightOnce } from "../../test/mocks/mock.utils.js";
+import {
+  createEventFromURL,
+  type CreateEventFromURLContext,
+} from "./createEventFromURL.flow.js";
+
+describe(createEventFromURL.name, () => {
+  const appTest = {
+    ctx: mockedContext<CreateEventFromURLContext>({
+      puppeteer: mockDeep(),
+      logger: mockDeep(),
+      db: mockDeep(),
+      ner: mockDeep(),
+      fs: mockDeep(),
+      urlMetadata: mockDeep(),
+      config: initContext().config,
+    }),
+  };
+
+  it("should create an event from a URL", async () => {
+    const [url] = fc
+      .sample(fc.nat(), 1)
+      .map((id) => `https://www.sciencedirect.com/article/${id}` as any);
+
+    const title = fc.sample(HumanReadableStringArb(), 1)[0];
+    const description = fc.sample(HumanReadableStringArb(), 1)[0];
+
+    // no url in db
+    mockTERightOnce(appTest.ctx.db.execQuery, () => null);
+    // event by url
+    mockTERightOnce(appTest.ctx.puppeteer.execute, () =>
+      fp.O.some({
+        type: SCIENTIFIC_STUDY.value,
+        date: new Date(),
+        payload: {
+          title,
+          url,
+          description,
+        },
+      }),
+    );
+
+    // link by url
+    let savedEvent: any;
+    mockTERightOnce(appTest.ctx.db.save, (_, event) => {
+      savedEvent = event[0];
+      return event;
+    });
+
+    mockTERightOnce(appTest.ctx.db.findOneOrFail, () => savedEvent);
+
+    // mocks.urlMetadata.fetchMetadata.mockResolvedValue({
+    //   title,
+    //   description,
+    //   url: scientificStudyData.url,
+    //   keywords: [],
+    // });
+
+    // mockTERightOnce(appTest.ctx.puppeteer.getBrowser, () => null);
+    // mocks.puppeteer.page.goto.mockResolvedValueOnce(undefined);
+
+    // // evaluate title
+    // mocks.puppeteer.page.$eval.mockResolvedValueOnce(title);
+    // // evaluate dropdown click
+    // mocks.puppeteer.page.click.mockResolvedValueOnce(undefined);
+    // // evaluate date string
+    // mocks.puppeteer.page.$eval.mockResolvedValueOnce([
+    //   "Received 27 July 2020",
+    //   "Accepted 1 August 2020",
+    // ]);
+    // // wait for
+    // mocks.puppeteer.page.waitForSelector.mockResolvedValueOnce(undefined);
+    // mocks.puppeteer.page.$$.mockResolvedValueOnce([
+    //   {
+    //     evaluate: vi.fn().mockResolvedValue(description),
+    //   },
+    // ]);
+
+    // mocks.puppeteer.page.$eval.mockResolvedValueOnce("page content");
+
+    // mocks.ner.winkMethods.learnCustomEntities.mockResolvedValueOnce({} as any);
+    // mocks.ner.doc.out.mockReturnValue([]);
+    // mocks.ner.doc.sentences.mockReturnValue({ each: vi.fn() } as any);
+    // mocks.ner.doc.customEntities.mockReturnValue({
+    //   out: vi.fn().mockReturnValue([]),
+    // } as any);
+    // mocks.ner.doc.tokens.mockReturnValue({ each: vi.fn() } as any);
+
+    // mocks.fs.existsSync.mockReturnValue(false);
+    // mocks.fs.readFileSync.mockReturnValue("[]");
+
+    const user = new UserEntity();
+
+    const event: any = await pipe(
+      createEventFromURL(
+        user,
+        uuid(),
+        url,
+        SCIENTIFIC_STUDY.value,
+      )(appTest.ctx),
+      throwTE,
+    );
+
+    expect(appTest.ctx.db.save).toHaveBeenCalledWith(EventV2Entity, [
+      expect.objectContaining({
+        type: SCIENTIFIC_STUDY.value,
+        payload: {
+          title,
+          description,
+          url: sanitizeURL(url),
+        },
+      }),
+    ]);
+
+    expect(event.type).toBe(SCIENTIFIC_STUDY.value);
+    expect(event.date).toBe(savedEvent.date);
+
+    expect(event.payload).toMatchObject(savedEvent.payload);
+  });
+});