From 7219c323742d91fd9749209db6e0363a3d020533 Mon Sep 17 00:00:00 2001 From: ascariandrea Date: Fri, 17 Jan 2025 19:29:29 +0100 Subject: [PATCH] chore: renamed extract event from url flow --- packages/@liexp/backend/package.json | 7 +- .../event/createEventFromURL.flow.spec.ts | 55 +--- .../flows/event/createEventFromURL.flow.ts | 2 +- ...ec.ts => extractEventFromURL.flow.spec.ts} | 8 +- ...RL.flow.ts => extractEventFromURL.flow.ts} | 164 ++++++------ .../flows/media/createAndUpload.flow.spec.ts | 241 ++++++++++++++++++ .../src/flows/media/createAndUpload.flow.ts | 32 ++- .../createFromTGMessage.flow.spec.ts | 4 +- .../upsertPinnedMessage.flow.spec.ts | 13 +- .../src/flows/tg/upsertPinnedMessage.flow.ts | 12 +- packages/@liexp/backend/src/test/index.ts | 89 ++----- .../backend/src/test/mocks/mock.utils.ts | 18 +- packages/@liexp/backend/tsconfig.build.json | 4 +- packages/@liexp/backend/tsconfig.json | 9 +- ...1785bf42b588eaa482b4a024c964d3096dc6e0.txt | 1 - packages/@liexp/backend/vitest.base-config.js | 43 ++++ pnpm-lock.yaml | 16 +- .../ai-bot/src/flows/ai/embedAndQuestion.ts | 2 +- .../ai-bot/src/flows/ai/summarizeTexFlow.ts | 2 +- services/api/tsconfig.json | 1 + services/data/package.json | 2 +- services/worker/package.json | 5 +- services/worker/src/bin/extract-events.ts | 2 +- vitest.config.mjs | 11 +- 24 files changed, 471 insertions(+), 272 deletions(-) rename packages/@liexp/backend/src/flows/event/{extractFromURL.flow.spec.ts => extractEventFromURL.flow.spec.ts} (94%) rename packages/@liexp/backend/src/flows/event/{extractFromURL.flow.ts => extractEventFromURL.flow.ts} (72%) create mode 100644 packages/@liexp/backend/src/flows/media/createAndUpload.flow.spec.ts delete mode 100644 packages/@liexp/backend/urls/0d1785bf42b588eaa482b4a024c964d3096dc6e0.txt create mode 100644 packages/@liexp/backend/vitest.base-config.js diff --git a/packages/@liexp/backend/package.json b/packages/@liexp/backend/package.json index 8a98bf75f3..b779607d9e 100644 --- a/packages/@liexp/backend/package.json +++ b/packages/@liexp/backend/package.json @@ -8,12 +8,15 @@ "module": "lib/index.js", "exports": { ".": "./lib/index.js", + "./vitest-base-config": "./vitest.base-config.mjs", "./lib/*": "./lib/*" }, "files": [ - "lib" + "lib", + "vitest.base-config.mjs" ], "scripts": { + "typecheck": "tsc", "build": "tsc -b tsconfig.build.json", "clean": "rm -rf lib", "lint": "eslint src", @@ -31,6 +34,7 @@ "@databases/sql": "^3.3.0", "@liexp/core": "workspace:*", "@liexp/shared": "workspace:*", + "@napi-rs/canvas": "^0.1.65", "date-fns": "^4.1.0", "fp-ts": "^2.16.9", "io-ts": "^2.2.22", @@ -58,6 +62,7 @@ "puppeteer-core": "^23.11.1", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", + "supertest": "7.0.0", "typescript": "^5.7.3", "vite-tsconfig-paths": "^5.1.4", "vitest": "^3.0.3", diff --git a/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts b/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts index 07fff8ffe1..15337ae0c3 100644 --- a/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts +++ b/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts @@ -6,11 +6,10 @@ import { throwTE } from "@liexp/shared/lib/utils/task.utils.js"; import { sanitizeURL } from "@liexp/shared/lib/utils/url.utils.js"; import { fc } from "@liexp/test"; import { describe, expect, it } from "vitest"; -import { mockDeep } from "vitest-mock-extended"; import { EventV2Entity } from "../../entities/Event.v2.entity.js"; import { UserEntity } from "../../entities/User.entity.js"; -import { initContext } from "../../test/index.js"; -import { mockedContext, mockTERightOnce } from "../../test/mocks/mock.utils.js"; +import { mockedContext } from "../../test/index.ts"; +import { mockTERightOnce } from "../../test/mocks/mock.utils.js"; import { createEventFromURL, type CreateEventFromURLContext, @@ -18,15 +17,7 @@ import { describe(createEventFromURL.name, () => { const appTest = { - ctx: mockedContext({ - puppeteer: mockDeep(), - logger: mockDeep(), - db: mockDeep(), - ner: mockDeep(), - fs: mockDeep(), - urlMetadata: mockDeep(), - config: initContext().config, - }), + ctx: mockedContext({}), }; it("should create an event from a URL", async () => { @@ -61,46 +52,6 @@ describe(createEventFromURL.name, () => { mockTERightOnce(appTest.ctx.db.findOneOrFail, () => savedEvent); - // mocks.urlMetadata.fetchMetadata.mockResolvedValue({ - // title, - // description, - // url: scientificStudyData.url, - // keywords: [], - // }); - - // mockTERightOnce(appTest.ctx.puppeteer.getBrowser, () => null); - // mocks.puppeteer.page.goto.mockResolvedValueOnce(undefined); - - // // evaluate title - // mocks.puppeteer.page.$eval.mockResolvedValueOnce(title); - // // evaluate dropdown click - // mocks.puppeteer.page.click.mockResolvedValueOnce(undefined); - // // evaluate date string - // mocks.puppeteer.page.$eval.mockResolvedValueOnce([ - // "Received 27 July 2020", - // "Accepted 1 August 2020", - // ]); - // // wait for - // mocks.puppeteer.page.waitForSelector.mockResolvedValueOnce(undefined); - // mocks.puppeteer.page.$$.mockResolvedValueOnce([ - // { - // evaluate: vi.fn().mockResolvedValue(description), - // }, - // ]); - - // mocks.puppeteer.page.$eval.mockResolvedValueOnce("page content"); - - // mocks.ner.winkMethods.learnCustomEntities.mockResolvedValueOnce({} as any); - // mocks.ner.doc.out.mockReturnValue([]); - // mocks.ner.doc.sentences.mockReturnValue({ each: vi.fn() } as any); - // mocks.ner.doc.customEntities.mockReturnValue({ - // out: vi.fn().mockReturnValue([]), - // } as any); - // mocks.ner.doc.tokens.mockReturnValue({ each: vi.fn() } as any); - - // mocks.fs.existsSync.mockReturnValue(false); - // mocks.fs.readFileSync.mockReturnValue("[]"); - const user = new UserEntity(); const event: any = await pipe( diff --git a/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts b/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts index a2e1ee03d8..8e2fd25ef7 100644 --- a/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts +++ b/packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts @@ -16,7 +16,7 @@ import { EventV2Entity } from "../../entities/Event.v2.entity.js"; import { type UserEntity } from "../../entities/User.entity.js"; import { ServerError } from "../../errors/ServerError.js"; import { findByURL } from "../../queries/events/scientificStudy.query.js"; -import { extractEventFromURL } from "./extractFromURL.flow.js"; +import { extractEventFromURL } from "./extractEventFromURL.flow.js"; export type CreateEventFromURLContext = LoggerContext & ConfigContext & diff --git a/packages/@liexp/backend/src/flows/event/extractFromURL.flow.spec.ts b/packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.spec.ts similarity index 94% rename from packages/@liexp/backend/src/flows/event/extractFromURL.flow.spec.ts rename to packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.spec.ts index f29744f692..27ab7c64b5 100644 --- a/packages/@liexp/backend/src/flows/event/extractFromURL.flow.spec.ts +++ b/packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.spec.ts @@ -9,25 +9,23 @@ import { describe, expect, it, vi } from "vitest"; import { mock } from "vitest-mock-extended"; import { LinkEntity } from "../../entities/Link.entity.js"; import { UserEntity } from "../../entities/User.entity.js"; -import { initContext, testConfig } from "../../test/index.js"; -import { mockedContext, mockTERightOnce } from "../../test/mocks/mock.utils.js"; +import { mockedContext } from "../../test/index.js"; +import { mockTERightOnce } from "../../test/mocks/mock.utils.js"; import { mocks } from "../../test/mocks.js"; import { createEventFromURL, type CreateEventFromURLContext, } from "./createEventFromURL.flow.js"; -import { extractEventFromURL } from "./extractFromURL.flow.js"; +import { extractEventFromURL } from "./extractEventFromURL.flow.js"; describe.skip(extractEventFromURL.name, () => { const appTest = { ctx: mockedContext({ puppeteer: mock(), - logger: mock(), db: mock(), ner: mock(), fs: mock(), urlMetadata: mock(), - config: testConfig, }), }; diff --git a/packages/@liexp/backend/src/flows/event/extractFromURL.flow.ts b/packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.ts similarity index 72% rename from packages/@liexp/backend/src/flows/event/extractFromURL.flow.ts rename to packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.ts index 64e81da776..5a11899f7b 100644 --- a/packages/@liexp/backend/src/flows/event/extractFromURL.flow.ts +++ b/packages/@liexp/backend/src/flows/event/extractEventFromURL.flow.ts @@ -9,7 +9,6 @@ import { } from "@liexp/shared/lib/io/http/Events/EventType.js"; import { toInitialValue } from "@liexp/shared/lib/providers/blocknote/utils.js"; import { parse } from "date-fns"; -import { sequenceS } from "fp-ts/lib/Apply.js"; import * as O from "fp-ts/lib/Option.js"; import { type ReaderTaskEither } from "fp-ts/lib/ReaderTaskEither.js"; import * as TE from "fp-ts/lib/TaskEither.js"; @@ -176,97 +175,90 @@ const extractByProvider = (ctx) => { return pipe( TE.Do, - TE.bind("relations", () => - sequenceS(TE.ApplicativeSeq)({ - relations: extractRelationsFromURL(p, l.url)(ctx), - provider: extractPageMetadataFromProviderLink(p, host, l)(ctx), - }), + TE.bind("relations", () => extractRelationsFromURL(p, l.url)(ctx)), + TE.bind("provider", () => + extractPageMetadataFromProviderLink(p, host, l)(ctx), ), - TE.bind( - "suggestions", - ({ - relations: { - relations: { entities }, - provider, - }, - }) => { - if (fp.O.isSome(provider)) { + TE.bind("metadata", ({ provider }) => { + if (fp.O.isSome(provider)) { + return fp.TE.right(provider.value); + } + + return fp.TE.right({ + url: l.url, + title: l.title, + description: l.description ?? l.title, + keywords: [], + image: l.image?.id ?? null, + icon: "", + provider: undefined, + type, + } satisfies Metadata); + }), + TE.bind("suggestions", ({ relations: { entities }, metadata }) => { + return pipe( + TE.Do, + TE.bind("link", () => { return pipe( - TE.Do, - TE.bind("link", () => { - return pipe( - LinkIO.decodeSingle(l), - fp.E.fold(() => fp.O.none, fp.O.some), - fp.TE.right, - ); - }), - TE.bind("relations", () => - pipe( - fp.TE.right( - getRelationIdsFromEventRelations({ - groupsMembers: [], - media: [], - areas: [], - actors: entities.actors as any[], - groups: entities.groups as any[], - keywords: entities.keywords as any[], - links: entities.links as any[], - }), - ), - ), - ), - TE.chain(({ link, relations }) => - TE.tryCatch(() => { - const suggestionMaker = getSuggestions((v) => - Promise.resolve(toInitialValue(v)), - ); + LinkIO.decodeSingle(l), + fp.E.fold(() => fp.O.none, fp.O.some), + fp.TE.right, + ); + }), - return suggestionMaker( - provider.value, - link, - O.none, - relations, - ); - }, ServerError.fromUnknown), + TE.bind("relations", () => + pipe( + fp.TE.right( + getRelationIdsFromEventRelations({ + groupsMembers: [], + media: [], + areas: [], + actors: entities.actors as any[], + groups: entities.groups as any[], + keywords: entities.keywords as any[], + links: entities.links as any[], + }), ), - TE.map((suggestions) => { - return suggestions.find((s) => s.event.type === type); - }), - TE.map(O.fromNullable), - ); - } - return TE.right(O.none); - }, - ), + ), + ), + TE.chain(({ link, relations }) => + TE.tryCatch(() => { + const suggestionMaker = getSuggestions((v) => + Promise.resolve(toInitialValue(v)), + ); - TE.map( - ({ - relations: { - relations: { entities }, - }, - suggestions, - }) => - pipe( - suggestions, - O.map((s) => ({ - ...s.event, - id: uuid(), - excerpt: s.event.excerpt ?? null, - body: s.event.body ?? null, - location: null, - links: [l], - keywords: [], - media: [], - events: [], - socialPosts: [], - actors: entities.actors, - groups: entities.groups, - stories: [], - createdAt: new Date(), - updatedAt: new Date(), - deletedAt: null, - })), + return suggestionMaker(metadata, link, O.none, relations); + }, ServerError.fromUnknown), ), + TE.map((suggestions) => { + return suggestions.find((s) => s.event.type === type); + }), + TE.map(O.fromNullable), + ); + }), + + TE.map(({ relations: { entities }, suggestions }) => + pipe( + suggestions, + O.map((s) => ({ + ...s.event, + id: uuid(), + excerpt: s.event.excerpt ?? null, + body: s.event.body ?? null, + location: null, + links: [l], + keywords: [], + media: [], + events: [], + socialPosts: [], + actors: entities.actors, + groups: entities.groups, + stories: [], + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + })), + ), ), ); }; diff --git a/packages/@liexp/backend/src/flows/media/createAndUpload.flow.spec.ts b/packages/@liexp/backend/src/flows/media/createAndUpload.flow.spec.ts new file mode 100644 index 0000000000..50c26613a8 --- /dev/null +++ b/packages/@liexp/backend/src/flows/media/createAndUpload.flow.spec.ts @@ -0,0 +1,241 @@ +import { fp } from "@liexp/core/lib/fp/index.js"; +import { + IframeVideoType, + ImageType, + MP4Type, +} from "@liexp/shared/lib/io/http/Media/MediaType.js"; +import { MediaArb } from "@liexp/shared/lib/tests/index.js"; +import { throwTE } from "@liexp/shared/lib/utils/task.utils.js"; +import * as tests from "@liexp/test/lib/index.js"; +import { pipe } from "fp-ts/lib/function.js"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { mock, mockClear, mockDeep } from "vitest-mock-extended"; +import { type FSClient } from "../../providers/fs/fs.provider.js"; +import { mockedContext } from "../../test/index.js"; +import { mockTERightOnce } from "../../test/mocks/mock.utils.js"; +import { sharpMock } from "../../test/mocks/sharp.mock.js"; +import { mocks } from "../../test/mocks.js"; +import { + createAndUpload, + type CreateAndUploadFlowContext, +} from "./createAndUpload.flow.js"; + +describe(createAndUpload.name, () => { + const fs = mock(); + const Test = { + ctx: mockedContext({ + env: process.env as any, + fs, + ner: mockDeep(), + http: mockDeep(), + pdf: mockDeep(), + ffmpeg: mockDeep(), + puppeteer: mockDeep(), + imgProc: mockDeep(), + s3: mockDeep(), + queue: mockDeep(), + db: mockDeep(), + }), + mocks, + }; + const addJob = vi.fn().mockImplementationOnce(() => fp.TE.right(undefined)); + + beforeEach(() => { + mockClear(Test.ctx.db); + mockClear(Test.ctx.s3); + + mockTERightOnce(Test.ctx.db.save, (_, m) => m); + }); + + test.todo("Should create a media from PDF location"); + + test("Should create a media from image location", async () => { + const [media] = tests.fc + .sample(MediaArb, 1) + .map(({ createdAt, updatedAt, id, thumbnail, ...m }, i) => ({ + ...m, + id, + label: `label-${id}`, + location: `https://example.com/${id}.jpg`, + type: ImageType.types[0].value, + creator: undefined, + })); + + const mediaUploadLocation = tests.fc.sample(tests.fc.webUrl(), 1)[0]; + mockTERightOnce(Test.ctx.s3.upload, () => ({ + Location: mediaUploadLocation, + })); + + const response = await pipe( + createAndUpload( + { + ...media, + location: media.location, + thumbnail: undefined, + }, + { Body: "image", ContentType: ImageType.types[0].value }, + media.id, + false, + )(Test.ctx), + throwTE, + ); + + expect(response).toMatchObject({ + location: mediaUploadLocation, + }); + }); + + test("Should create a media from MP4 file location", async () => { + const [media] = tests.fc + .sample(MediaArb, 1) + .map(({ createdAt, updatedAt, deletedAt, id, thumbnail, ...m }, i) => ({ + ...m, + id, + label: `label-${id}`, + location: `https://example.com/${id}.mp4`, + type: MP4Type.value, + creator: undefined, + extra: undefined, + })); + + const mediaUploadLocation = tests.fc.sample(tests.fc.webUrl(), 1)[0]; + mockTERightOnce(Test.ctx.s3.upload, () => ({ + Location: mediaUploadLocation, + })); + + const result = await pipe( + createAndUpload( + { + ...media, + location: media.location, + thumbnail: undefined, + }, + { Body: {}, ContentType: MP4Type.value }, + media.id, + false, + )(Test.ctx), + throwTE, + ); + + expect(addJob).not.toHaveBeenCalled(); + + expect(result).toMatchObject({ + ...media, + id: expect.any(String), + description: media.description ?? media.label, + creator: undefined, + location: mediaUploadLocation, + // extra: { + // width: 0, + // height: expect.any(Number), + // thumbnailWidth: 0, + // thumbnailHeight: 0, + // thumbnails: [], + // needRegenerateThumbnail: false, + // }, + extra: undefined, + // socialPosts: [], + // transferable: true, + // createdAt: expect.any(String), + // updatedAt: expect.any(String), + }); + }); + + test("Should create a media from iframe/video location", async () => { + const [media] = tests.fc + .sample(MediaArb, 1) + .map(({ createdAt, updatedAt, deletedAt, thumbnail, id, ...m }, i) => ({ + ...m, + id, + label: `label-${id}`, + location: `https://www.youtube.com/watch?v=${id}`, + type: IframeVideoType.value, + creator: undefined, + })); + + const response = await pipe( + createAndUpload( + { + ...media, + location: media.location, + thumbnail: undefined, + }, + { Body: {}, ContentType: IframeVideoType.value }, + media.id, + false, + )(Test.ctx), + throwTE, + ); + + expect(Test.ctx.s3.upload).toHaveBeenCalledTimes(0); + expect(Test.ctx.db.save).toHaveBeenCalledTimes(1); + + expect(response).toMatchObject({ + ...media, + id: expect.any(String), + // location: `https://www.youtube.com/embed/${media.id}`, + location: `https://www.youtube.com/watch?v=${media.id}`, + description: media.description ?? media.label, + creator: undefined, + // extra: { + // width: 0, + // height: 0, + // thumbnailWidth: 0, + // thumbnailHeight: 0, + // thumbnails: [], + // needRegenerateThumbnail: false, + // }, + // socialPosts: [], + // transferable: true, + // createdAt: expect.any(String), + // updatedAt: expect.any(String), + }); + }); + + test("Should get an error when 'location' in media is duplicated", async () => { + const [media] = tests.fc + .sample(MediaArb, 1) + .map(({ id, createdAt, updatedAt, ...m }) => ({ + ...m, + id, + label: `label-${id}`, + description: `description-${id}`, + events: [], + links: [], + keywords: [], + areas: [], + featuredInStories: [], + socialPosts: [], + location: `https://example.com/${id}.jpg`, + thumbnail: `https://example.com/${id}-thumb.jpg`, + creator: undefined, + extra: undefined, + })); + + Test.ctx.db.findOneOrFail.mockResolvedValueOnce(media as any); + + Test.mocks.axios.get.mockImplementation(() => { + return Promise.resolve({ data: Buffer.from([]) }); + }); + + Test.mocks.redis.publish.mockResolvedValueOnce(1); + + const task = pipe( + createAndUpload( + { + ...media, + location: media.location, + thumbnail: undefined, + }, + { Body: {}, ContentType: IframeVideoType.value }, + media.id, + true, + )(Test.ctx), + ); + + await expect(throwTE(task)).rejects.toThrowError(); + + expect(sharpMock.toFormat).not.toHaveBeenCalled(); + expect(sharpMock.toBuffer).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@liexp/backend/src/flows/media/createAndUpload.flow.ts b/packages/@liexp/backend/src/flows/media/createAndUpload.flow.ts index 8443231d89..0a6fdbd434 100644 --- a/packages/@liexp/backend/src/flows/media/createAndUpload.flow.ts +++ b/packages/@liexp/backend/src/flows/media/createAndUpload.flow.ts @@ -31,22 +31,23 @@ import { type MediaEntity } from "../../entities/Media.entity.js"; import { ServerError } from "../../errors/ServerError.js"; import { upload } from "../../flows/space/upload.flow.js"; import { MediaRepository } from "../../services/entity-repository.service.js"; +import { LoggerService } from "../../services/logger/logger.service.js"; import { createThumbnail } from "./thumbnails/createThumbnail.flow.js"; -export const createAndUpload = < - C extends SpaceContext & - ENVContext & - QueuesProviderContext & - DatabaseContext & - LoggerContext & - ConfigContext & - FSClientContext & - HTTPProviderContext & - PDFProviderContext & - FFMPEGProviderContext & - PuppeteerProviderContext & - ImgProcClientContext, ->( +export type CreateAndUploadFlowContext = SpaceContext & + ENVContext & + QueuesProviderContext & + DatabaseContext & + LoggerContext & + ConfigContext & + FSClientContext & + HTTPProviderContext & + PDFProviderContext & + FFMPEGProviderContext & + PuppeteerProviderContext & + ImgProcClientContext; + +export const createAndUpload = ( createMediaData: Media.CreateMedia, { Body, ContentType }: { Body: any; ContentType?: MediaType }, id: UUID | undefined, @@ -61,6 +62,7 @@ export const createAndUpload = < if (IframeVideoType.is(createMediaData.type)) { return fp.RTE.right(createMediaData.location); } + const mediaKey = getMediaKey( "media", mediaId, @@ -79,6 +81,7 @@ export const createAndUpload = < ); }), // ctx.logger.debug.logInTaskEither("Result %O"), + LoggerService.RTE.info("Result %O"), fp.RTE.bind("thumbnail", ({ mediaId, location }) => pipe( extractThumb @@ -98,6 +101,7 @@ export const createAndUpload = < MediaRepository.save([ { ...createMediaData, + description: createMediaData.description ?? createMediaData.label, events: [], links: [], featuredInStories: [], diff --git a/packages/@liexp/backend/src/flows/tg/__tests__/createFromTGMessage.flow.spec.ts b/packages/@liexp/backend/src/flows/tg/__tests__/createFromTGMessage.flow.spec.ts index 158766fb74..6788c3e77b 100644 --- a/packages/@liexp/backend/src/flows/tg/__tests__/createFromTGMessage.flow.spec.ts +++ b/packages/@liexp/backend/src/flows/tg/__tests__/createFromTGMessage.flow.spec.ts @@ -16,7 +16,7 @@ import { TGMessageArb, TGPhotoArb, } from "../../../test/arbitraries/TGMessage.arb.js"; -import { initContext } from "../../../test/index.js"; +import { mockedContext } from "../../../test/index.js"; import puppeteerMocks from "../../../test/mocks/puppeteer.mock.js"; import { mocks } from "../../../test/mocks.js"; import { type UserTest } from "../../../test/user.utils.js"; @@ -36,7 +36,7 @@ interface MessageTest { describe("Create From TG Message", () => { let admin: UserTest; - const ctx = initContext(); + const ctx = mockedContext(); beforeAll(() => { [admin] = fc.sample(UserArb, 1).map((u) => ({ diff --git a/packages/@liexp/backend/src/flows/tg/__tests__/upsertPinnedMessage.flow.spec.ts b/packages/@liexp/backend/src/flows/tg/__tests__/upsertPinnedMessage.flow.spec.ts index d6f4bc8611..d921ec1118 100644 --- a/packages/@liexp/backend/src/flows/tg/__tests__/upsertPinnedMessage.flow.spec.ts +++ b/packages/@liexp/backend/src/flows/tg/__tests__/upsertPinnedMessage.flow.spec.ts @@ -1,23 +1,30 @@ import { KeywordArb } from "@liexp/shared/lib/tests/arbitrary/Keyword.arbitrary.js"; import { ActorArb, UncategorizedArb } from "@liexp/shared/lib/tests/index.js"; import { throwTE } from "@liexp/shared/lib/utils/task.utils.js"; -import { fc } from "@liexp/test"; +import { fc } from "@liexp/test/lib/index.js"; import * as E from "fp-ts/lib/Either.js"; import { describe, expect, test } from "vitest"; +import { mockDeep } from "vitest-mock-extended"; import { ActorEntity } from "../../../entities/Actor.entity.js"; import { EventV2Entity } from "../../../entities/Event.v2.entity.js"; import { KeywordEntity } from "../../../entities/Keyword.entity.js"; -import { initContext } from "../../../test/index.js"; +import { mockedContext } from "../../../test/index.js"; import { mocks } from "../../../test/mocks.js"; import { toPinnedMessage, upsertPinnedMessage, + type UpsertPinnerMessageFlowContext, } from "../upsertPinnedMessage.flow.js"; describe("Upsert Pinned Message Flow", () => { const Test = { - ctx: initContext(), + ctx: mockedContext({ + db: mockDeep(), + env: process.env as any, + tg: mockDeep(), + }), }; + test.skip("Should upsert the message with 5 keywords", async () => { const keywordCount = 10; const actorCount = 10; diff --git a/packages/@liexp/backend/src/flows/tg/upsertPinnedMessage.flow.ts b/packages/@liexp/backend/src/flows/tg/upsertPinnedMessage.flow.ts index 8fe69044e9..2c1d1c23c7 100644 --- a/packages/@liexp/backend/src/flows/tg/upsertPinnedMessage.flow.ts +++ b/packages/@liexp/backend/src/flows/tg/upsertPinnedMessage.flow.ts @@ -38,13 +38,13 @@ ${keywords.map((k) => `#${k.tag} (${k.eventCount})`).join("\n")} \n `; +export type UpsertPinnerMessageFlowContext = DatabaseContext & + LoggerContext & + ENVContext & + TGBotProviderContext; + export const upsertPinnedMessage = - < - C extends DatabaseContext & - LoggerContext & - ENVContext & - TGBotProviderContext, - >( + ( limit: number, ): ReaderTaskEither => (ctx) => { diff --git a/packages/@liexp/backend/src/test/index.ts b/packages/@liexp/backend/src/test/index.ts index 74f055a372..10c9c9e878 100644 --- a/packages/@liexp/backend/src/test/index.ts +++ b/packages/@liexp/backend/src/test/index.ts @@ -1,8 +1,6 @@ import { GetLogger } from "@liexp/core/lib/index.js"; -import { HTTPProvider } from "@liexp/shared/lib/providers/http/http.provider.js"; -import { PDFProvider } from "@liexp/shared/lib/providers/pdf/pdf.provider.js"; import D from "debug"; -import * as puppeteer from "puppeteer-core"; +import { mockDeep, type DeepMockProxy } from "vitest-mock-extended"; import { type ConfigContext } from "../context/config.context.js"; import { type DatabaseContext } from "../context/db.context.js"; import { type ENVContext } from "../context/env.context.js"; @@ -20,21 +18,7 @@ import { type PuppeteerProviderContext } from "../context/puppeteer.context.js"; import { type QueuesProviderContext } from "../context/queue.context.js"; import { type SpaceContext } from "../context/space.context.js"; import { type URLMetadataContext } from "../context/urlMetadata.context.js"; -import { type BACKEND_ENV } from "../io/ENV.js"; -import { MakeURLMetadata } from "../providers/URLMetadata.provider.js"; -import { GetFFMPEGProvider } from "../providers/ffmpeg/ffmpeg.provider.js"; -import { GetFSClient } from "../providers/fs/fs.provider.js"; -import { MakeImgProcClient } from "../providers/imgproc/imgproc.provider.js"; -import { GetNERProvider } from "../providers/ner/ner.provider.js"; -import { GetDatabaseClient } from "../providers/orm/index.js"; -import { GetPuppeteerProvider } from "../providers/puppeteer.provider.js"; -import { GetQueueProvider } from "../providers/queue.provider.js"; -import { MakeSpaceProvider } from "../providers/space/space.provider.js"; -import { TGBotProvider } from "../providers/tg/tg.provider.js"; import { EventsConfig } from "../queries/config/index.js"; -import { mocks } from "./mocks.js"; - -const pdfContext = PDFProvider({ client: mocks.pdf }); type TestContext = ENVContext & PDFProviderContext & @@ -65,57 +49,32 @@ export const testConfig = { events: EventsConfig, }; -export const initContext = (): TestContext => { - D.enable(process.env.DEBUG ?? "*"); +export type MockedContext> = { + [K in keyof C]: DeepMockProxy; +}; +export const mockedContext = >( + ctx: Partial>, +): Omit & LoggerContext & ConfigContext => { const logger = GetLogger("test"); - const fs = GetFSClient({ client: mocks.fs }); - - const ctx = { - env: process.env as any as BACKEND_ENV, - db: GetDatabaseClient({ - connection: mocks.db.connection, - logger, - }), - fs, - s3: MakeSpaceProvider({ - client: mocks.s3.client as any, - getSignedUrl: mocks.s3.getSignedUrl, - classes: mocks.s3.classes as any, - }), + D.enable(process.env.DEBUG ?? "@liexp:*"); + return { + puppeteer: mockDeep(), + db: mockDeep(), + ner: mockDeep(), + fs: mockDeep(), + urlMetadata: mockDeep(), + env: process.env as any, + pdf: mockDeep(), + http: mockDeep(), + tg: mockDeep(), + s3: mockDeep(), + imgProc: mockDeep(), + queue: mockDeep(), + ffmpeg: mockDeep(), + ...ctx, config: testConfig, - pdf: pdfContext, - puppeteer: GetPuppeteerProvider( - mocks.puppeteer, - {}, - puppeteer.KnownDevices, - ), - ffmpeg: GetFFMPEGProvider(mocks.ffmpeg), - queue: GetQueueProvider(fs, "fake-queue"), - http: HTTPProvider(mocks.axios as any), - ner: GetNERProvider({ - nlp: mocks.ner, - entitiesFile: "fake", - logger, - }), - urlMetadata: MakeURLMetadata({ - client: mocks.urlMetadata.fetchHTML as any, - parser: { - getMetadata: mocks.urlMetadata.fetchMetadata, - }, - }), - tg: TGBotProvider( - { logger: logger, client: () => mocks.tg as any }, - { token: "fake", chat: "fake", polling: false, baseApiUrl: "fake" }, - ), - imgProc: MakeImgProcClient({ - logger: logger.extend("imgproc"), - client: mocks.sharp, - exifR: mocks.exifR, - }), - logger: GetLogger("test"), + logger: ctx?.logger ?? logger, }; - - return ctx; }; diff --git a/packages/@liexp/backend/src/test/mocks/mock.utils.ts b/packages/@liexp/backend/src/test/mocks/mock.utils.ts index d8ad6d8d3a..27fef2ad3d 100644 --- a/packages/@liexp/backend/src/test/mocks/mock.utils.ts +++ b/packages/@liexp/backend/src/test/mocks/mock.utils.ts @@ -1,7 +1,7 @@ import { fp } from "@liexp/core/lib/fp/index.js"; -import { type TaskEither } from "fp-ts/lib/TaskEither"; +import { type ReaderTaskEither } from "fp-ts/lib/ReaderTaskEither.js"; +import { type TaskEither } from "fp-ts/lib/TaskEither.js"; import { type MockInstance } from "vitest"; -import { type DeepMockProxy } from "vitest-mock-extended"; export const mockTERightOnce = ( fn: MockInstance<(...args: any[]) => TaskEither>, @@ -11,10 +11,10 @@ export const mockTERightOnce = ( return fp.TE.right(value(...args)); }); -type MockedContext> = { - [K in keyof C]: DeepMockProxy; -}; - -export const mockedContext = >( - ctx: C, -): MockedContext => ctx; +export const mockRTERightOnce = ( + fn: MockInstance<(...args: any[]) => ReaderTaskEither>, + value: (...args: any[]) => U, +) => + fn.mockImplementationOnce((...args) => { + return fp.RTE.right(value(...args)); + }); diff --git a/packages/@liexp/backend/tsconfig.build.json b/packages/@liexp/backend/tsconfig.build.json index dd2596cc76..5777481e1a 100644 --- a/packages/@liexp/backend/tsconfig.build.json +++ b/packages/@liexp/backend/tsconfig.build.json @@ -6,6 +6,6 @@ "rootDir": "./src", "tsBuildInfoFile": "lib/build.tsbuildinfo" }, - "include": ["src", "typings", "../shared/typings", "src/**/*.json"], - "exclude": ["node_modules", "lib", "src/**/*.spec.ts"] + "include": ["src/**/*.ts", "typings", "../shared/typings", "src/**/*.json"], + "exclude": ["node_modules", "lib", "src/**/*.spec.ts", "src/**/__tests__"] } diff --git a/packages/@liexp/backend/tsconfig.json b/packages/@liexp/backend/tsconfig.json index a30e795b0b..f6eecae02e 100644 --- a/packages/@liexp/backend/tsconfig.json +++ b/packages/@liexp/backend/tsconfig.json @@ -20,7 +20,14 @@ "rootDir": "./src", "tsBuildInfoFile": "lib/.tsbuildinfo" }, - "include": ["./src", "./typings", "../shared/typings", "./src/**/*.json"], + "include": [ + "./src", + "./typings", + "../shared/typings", + "./src/**/*.json", + "vitest.base-config.js", + "vitest.config.ts" + ], "exclude": ["node_modules", "lib"], "references": [ { diff --git a/packages/@liexp/backend/urls/0d1785bf42b588eaa482b4a024c964d3096dc6e0.txt b/packages/@liexp/backend/urls/0d1785bf42b588eaa482b4a024c964d3096dc6e0.txt deleted file mode 100644 index acdfa86028..0000000000 --- a/packages/@liexp/backend/urls/0d1785bf42b588eaa482b4a024c964d3096dc6e0.txt +++ /dev/null @@ -1 +0,0 @@ -"page content" \ No newline at end of file diff --git a/packages/@liexp/backend/vitest.base-config.js b/packages/@liexp/backend/vitest.base-config.js new file mode 100644 index 0000000000..3091b9d489 --- /dev/null +++ b/packages/@liexp/backend/vitest.base-config.js @@ -0,0 +1,43 @@ +import { URL } from "url"; +import viteTsconfigPaths from "vite-tsconfig-paths"; +import { defineConfig } from "vitest/config"; + +export const PathnameAlias = (url) => (mockPath) => { + console.log("mockPath", mockPath, url); + const pathname = new URL(mockPath, url).pathname; + console.log("pathname", pathname); + return pathname; +}; + +export const extendBaseConfig = (root, configFn) => { + const toBackendAlias = PathnameAlias(import.meta.url); + const toAlias = PathnameAlias(root); + const config = configFn(toAlias); + + return defineConfig({ + root: toAlias("./"), + test: { + environment: "node", + watch: false, + alias: { + sharp: toBackendAlias("lib/test/mocks/sharp.mock.js"), + canvas: toBackendAlias("lib/test/mocks/canvas.mock.js"), + "pdfjs-dist/legacy/build/pdf.js": toBackendAlias( + "lib/test/mocks/pdfjs.mock.js", + ), + "@blocknote/core": toBackendAlias( + "lib/test/mocks/blocknote-core.mock.js", + ), + "@blocknote/react/**": toBackendAlias( + "lib/test/mocks/blocknote-react.mock.js", + ), + }, + ...config.test, + }, + plugins: [ + viteTsconfigPaths({ + root: toAlias("./"), + }), + ].concat(config.plugins || []), + }); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f50b2bc88c..911ec2648e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -73,6 +73,9 @@ importers: '@liexp/shared': specifier: workspace:* version: link:../shared + '@napi-rs/canvas': + specifier: ^0.1.65 + version: 0.1.65 axios: specifier: ^1 version: 1.7.9(debug@4.4.0) @@ -188,6 +191,9 @@ importers: puppeteer-extra-plugin-stealth: specifier: ^2.11.2 version: 2.11.2(puppeteer-extra@3.3.6(puppeteer-core@23.11.1)) + supertest: + specifier: 7.0.0 + version: 7.0.0 typescript: specifier: ^5.7.3 version: 5.7.3 @@ -1319,15 +1325,6 @@ importers: typeorm: specifier: ^0.3.20 version: 0.3.20(ioredis@5.4.2)(pg@8.13.1) - wink-eng-lite-web-model: - specifier: ^1.8.0 - version: 1.8.1 - wink-nlp: - specifier: ^2.3.0 - version: 2.3.2 - wink-nlp-utils: - specifier: ^2.1.0 - version: 2.1.0 devDependencies: '@types/node-cron': specifier: ^3.0.11 @@ -12478,7 +12475,6 @@ snapshots: '@napi-rs/canvas-linux-x64-gnu': 0.1.65 '@napi-rs/canvas-linux-x64-musl': 0.1.65 '@napi-rs/canvas-win32-x64-msvc': 0.1.65 - optional: true '@nodelib/fs.scandir@2.1.5': dependencies: diff --git a/services/ai-bot/src/flows/ai/embedAndQuestion.ts b/services/ai-bot/src/flows/ai/embedAndQuestion.ts index 91237d0941..0889c7a766 100644 --- a/services/ai-bot/src/flows/ai/embedAndQuestion.ts +++ b/services/ai-bot/src/flows/ai/embedAndQuestion.ts @@ -11,7 +11,7 @@ export const embedAndQuestionFlow: JobProcessRTE = (job) => (ctx) => { return pipe( loadDocs(job)(ctx), fp.TE.chain((docs) => - fp.TE.tryCatch(async () => { + fp.TE.tryCatch(() => { return ctx.langchain.queryDocument( docs, job.data.question ?? defaultQuestion, diff --git a/services/ai-bot/src/flows/ai/summarizeTexFlow.ts b/services/ai-bot/src/flows/ai/summarizeTexFlow.ts index e325411ee3..a0219c2c6d 100644 --- a/services/ai-bot/src/flows/ai/summarizeTexFlow.ts +++ b/services/ai-bot/src/flows/ai/summarizeTexFlow.ts @@ -9,7 +9,7 @@ export const summarizeTextFlow: JobProcessRTE = (job) => (ctx) => { return pipe( loadDocs(job)(ctx), fp.TE.chain((docs) => - fp.TE.tryCatch(async () => { + fp.TE.tryCatch(() => { return ctx.langchain.summarizeText(docs, { model: ctx.config.config.localAi.models ?.summarization as AvailableModels, diff --git a/services/api/tsconfig.json b/services/api/tsconfig.json index cb62e40361..ca3b06eaf4 100644 --- a/services/api/tsconfig.json +++ b/services/api/tsconfig.json @@ -43,6 +43,7 @@ ], "exclude": ["**/build", "**/lib"], "references": [ + { "path": "../../packages/@liexp/core" }, { "path": "../../packages/@liexp/shared" }, { "path": "../../packages/@liexp/backend" } ] diff --git a/services/data/package.json b/services/data/package.json index 85154ad836..41583eb496 100644 --- a/services/data/package.json +++ b/services/data/package.json @@ -35,7 +35,7 @@ "@types/node": "^22.9.3", "dotenv": "^16.4.7", "prettier": "^2.5.1", - "supertest": "^6.2.2", + "supertest": "^7", "typescript": "^4.6.2" } } diff --git a/services/worker/package.json b/services/worker/package.json index 31376f0f70..8a7b16627c 100644 --- a/services/worker/package.json +++ b/services/worker/package.json @@ -67,10 +67,7 @@ "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "sharp": "^0.33.5", - "typeorm": "^0.3.20", - "wink-eng-lite-web-model": "^1.8.0", - "wink-nlp": "^2.3.0", - "wink-nlp-utils": "^2.1.0" + "typeorm": "^0.3.20" }, "devDependencies": { "@types/node-cron": "^3.0.11", diff --git a/services/worker/src/bin/extract-events.ts b/services/worker/src/bin/extract-events.ts index 0e91ed0e24..bfea5f45ba 100644 --- a/services/worker/src/bin/extract-events.ts +++ b/services/worker/src/bin/extract-events.ts @@ -2,7 +2,7 @@ import { EventV2Entity } from "@liexp/backend/lib/entities/Event.v2.entity.js"; import { type DataPayloadLink, extractEventFromURL, -} from "@liexp/backend/lib/flows/event/extractFromURL.flow.js"; +} from "@liexp/backend/lib/flows/event/extractEventFromURL.flow.js"; import { getOneAdminOrFail } from "@liexp/backend/lib/flows/user/getOneUserOrFail.flow.js"; import { findByURL } from "@liexp/backend/lib/queries/events/scientificStudy.query.js"; import { throwTE } from "@liexp/shared/lib/utils/task.utils.js"; diff --git a/vitest.config.mjs b/vitest.config.mjs index 53e0cc1ec0..982f1ee3bc 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -1,7 +1,6 @@ -import { coverageConfigDefaults, defineConfig } from 'vitest/config'; +import { coverageConfigDefaults, defineConfig } from "vitest/config"; export default defineConfig({ - root: __dirname, test: { globals: true, coverage: { @@ -33,7 +32,7 @@ export default defineConfig({ lines: 80, statements: 80, branches: 80, - } - } - } -}) \ No newline at end of file + }, + }, + }, +});