Skip to content

Commit

Permalink
chore: renamed extract event from url flow
Browse files Browse the repository at this point in the history
  • Loading branch information
@ascariandrea
ascariandrea committed Jan 23, 2025
1 parent 90b2c1f commit 7219c32
Show file tree
Hide file tree
Showing 24 changed files with 471 additions and 272 deletions.
7 changes: 6 additions & 1 deletion packages/@liexp/backend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@
"module": "lib/index.js",
"exports": {
".": "./lib/index.js",
"./vitest-base-config": "./vitest.base-config.mjs",
"./lib/*": "./lib/*"
},
"files": [
"lib"
"lib",
"vitest.base-config.mjs"
],
"scripts": {
"typecheck": "tsc",
"build": "tsc -b tsconfig.build.json",
"clean": "rm -rf lib",
"lint": "eslint src",
Expand All @@ -31,6 +34,7 @@
"@databases/sql": "^3.3.0",
"@liexp/core": "workspace:*",
"@liexp/shared": "workspace:*",
"@napi-rs/canvas": "^0.1.65",
"date-fns": "^4.1.0",
"fp-ts": "^2.16.9",
"io-ts": "^2.2.22",
Expand Down Expand Up @@ -58,6 +62,7 @@
"puppeteer-core": "^23.11.1",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"supertest": "7.0.0",
"typescript": "^5.7.3",
"vite-tsconfig-paths": "^5.1.4",
"vitest": "^3.0.3",
Expand Down
55 changes: 3 additions & 52 deletions packages/@liexp/backend/src/flows/event/createEventFromURL.flow.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,18 @@ import { throwTE } from "@liexp/shared/lib/utils/task.utils.js";
import { sanitizeURL } from "@liexp/shared/lib/utils/url.utils.js";
import { fc } from "@liexp/test";
import { describe, expect, it } from "vitest";
import { mockDeep } from "vitest-mock-extended";
import { EventV2Entity } from "../../entities/Event.v2.entity.js";
import { UserEntity } from "../../entities/User.entity.js";
import { initContext } from "../../test/index.js";
import { mockedContext, mockTERightOnce } from "../../test/mocks/mock.utils.js";
import { mockedContext } from "../../test/index.ts";
import { mockTERightOnce } from "../../test/mocks/mock.utils.js";
import {
createEventFromURL,
type CreateEventFromURLContext,
} from "./createEventFromURL.flow.js";

describe(createEventFromURL.name, () => {
const appTest = {
ctx: mockedContext<CreateEventFromURLContext>({
puppeteer: mockDeep(),
logger: mockDeep(),
db: mockDeep(),
ner: mockDeep(),
fs: mockDeep(),
urlMetadata: mockDeep(),
config: initContext().config,
}),
ctx: mockedContext<CreateEventFromURLContext>({}),
};

it("should create an event from a URL", async () => {
Expand Down Expand Up @@ -61,46 +52,6 @@ describe(createEventFromURL.name, () => {

mockTERightOnce(appTest.ctx.db.findOneOrFail, () => savedEvent);

// mocks.urlMetadata.fetchMetadata.mockResolvedValue({
// title,
// description,
// url: scientificStudyData.url,
// keywords: [],
// });

// mockTERightOnce(appTest.ctx.puppeteer.getBrowser, () => null);
// mocks.puppeteer.page.goto.mockResolvedValueOnce(undefined);

// // evaluate title
// mocks.puppeteer.page.$eval.mockResolvedValueOnce(title);
// // evaluate dropdown click
// mocks.puppeteer.page.click.mockResolvedValueOnce(undefined);
// // evaluate date string
// mocks.puppeteer.page.$eval.mockResolvedValueOnce([
// "Received 27 July 2020",
// "Accepted 1 August 2020",
// ]);
// // wait for
// mocks.puppeteer.page.waitForSelector.mockResolvedValueOnce(undefined);
// mocks.puppeteer.page.$$.mockResolvedValueOnce([
// {
// evaluate: vi.fn().mockResolvedValue(description),
// },
// ]);

// mocks.puppeteer.page.$eval.mockResolvedValueOnce("page content");

// mocks.ner.winkMethods.learnCustomEntities.mockResolvedValueOnce({} as any);
// mocks.ner.doc.out.mockReturnValue([]);
// mocks.ner.doc.sentences.mockReturnValue({ each: vi.fn() } as any);
// mocks.ner.doc.customEntities.mockReturnValue({
// out: vi.fn().mockReturnValue([]),
// } as any);
// mocks.ner.doc.tokens.mockReturnValue({ each: vi.fn() } as any);

// mocks.fs.existsSync.mockReturnValue(false);
// mocks.fs.readFileSync.mockReturnValue("[]");

const user = new UserEntity();

const event: any = await pipe(
Expand Down
2 changes: 1 addition & 1 deletion packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { EventV2Entity } from "../../entities/Event.v2.entity.js";
import { type UserEntity } from "../../entities/User.entity.js";
import { ServerError } from "../../errors/ServerError.js";
import { findByURL } from "../../queries/events/scientificStudy.query.js";
import { extractEventFromURL } from "./extractFromURL.flow.js";
import { extractEventFromURL } from "./extractEventFromURL.flow.js";

export type CreateEventFromURLContext = LoggerContext &
ConfigContext &
Expand Down
8 changes: 3 additions & 5 deletions ...c/flows/event/extractFromURL.flow.spec.ts → ...ws/event/extractEventFromURL.flow.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,23 @@ import { describe, expect, it, vi } from "vitest";
import { mock } from "vitest-mock-extended";
import { LinkEntity } from "../../entities/Link.entity.js";
import { UserEntity } from "../../entities/User.entity.js";
import { initContext, testConfig } from "../../test/index.js";
import { mockedContext, mockTERightOnce } from "../../test/mocks/mock.utils.js";
import { mockedContext } from "../../test/index.js";
import { mockTERightOnce } from "../../test/mocks/mock.utils.js";
import { mocks } from "../../test/mocks.js";
import {
createEventFromURL,
type CreateEventFromURLContext,
} from "./createEventFromURL.flow.js";
import { extractEventFromURL } from "./extractFromURL.flow.js";
import { extractEventFromURL } from "./extractEventFromURL.flow.js";

describe.skip(extractEventFromURL.name, () => {
const appTest = {
ctx: mockedContext<CreateEventFromURLContext>({
puppeteer: mock(),
logger: mock(),
db: mock(),
ner: mock(),
fs: mock(),
urlMetadata: mock(),
config: testConfig,
}),
};

Expand Down
164 changes: 78 additions & 86 deletions ...nd/src/flows/event/extractFromURL.flow.ts → ...c/flows/event/extractEventFromURL.flow.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import {
} from "@liexp/shared/lib/io/http/Events/EventType.js";
import { toInitialValue } from "@liexp/shared/lib/providers/blocknote/utils.js";
import { parse } from "date-fns";
import { sequenceS } from "fp-ts/lib/Apply.js";
import * as O from "fp-ts/lib/Option.js";
import { type ReaderTaskEither } from "fp-ts/lib/ReaderTaskEither.js";
import * as TE from "fp-ts/lib/TaskEither.js";
Expand Down Expand Up @@ -176,97 +175,90 @@ const extractByProvider =
(ctx) => {
return pipe(
TE.Do,
TE.bind("relations", () =>
sequenceS(TE.ApplicativeSeq)({
relations: extractRelationsFromURL(p, l.url)(ctx),
provider: extractPageMetadataFromProviderLink(p, host, l)(ctx),
}),
TE.bind("relations", () => extractRelationsFromURL(p, l.url)(ctx)),
TE.bind("provider", () =>
extractPageMetadataFromProviderLink(p, host, l)(ctx),
),
TE.bind(
"suggestions",
({
relations: {
relations: { entities },
provider,
},
}) => {
if (fp.O.isSome(provider)) {
TE.bind("metadata", ({ provider }) => {
if (fp.O.isSome(provider)) {
return fp.TE.right(provider.value);
}

return fp.TE.right({
url: l.url,
title: l.title,
description: l.description ?? l.title,
keywords: [],
image: l.image?.id ?? null,
icon: "",
provider: undefined,
type,
} satisfies Metadata);
}),
TE.bind("suggestions", ({ relations: { entities }, metadata }) => {
return pipe(
TE.Do,
TE.bind("link", () => {
return pipe(
TE.Do,
TE.bind("link", () => {
return pipe(
LinkIO.decodeSingle(l),
fp.E.fold(() => fp.O.none, fp.O.some),
fp.TE.right,
);
}),
TE.bind("relations", () =>
pipe(
fp.TE.right(
getRelationIdsFromEventRelations({
groupsMembers: [],
media: [],
areas: [],
actors: entities.actors as any[],
groups: entities.groups as any[],
keywords: entities.keywords as any[],
links: entities.links as any[],
}),
),
),
),
TE.chain(({ link, relations }) =>
TE.tryCatch(() => {
const suggestionMaker = getSuggestions((v) =>
Promise.resolve(toInitialValue(v)),
);
LinkIO.decodeSingle(l),
fp.E.fold(() => fp.O.none, fp.O.some),
fp.TE.right,
);
}),

return suggestionMaker(
provider.value,
link,
O.none,
relations,
);
}, ServerError.fromUnknown),
TE.bind("relations", () =>
pipe(
fp.TE.right(
getRelationIdsFromEventRelations({
groupsMembers: [],
media: [],
areas: [],
actors: entities.actors as any[],
groups: entities.groups as any[],
keywords: entities.keywords as any[],
links: entities.links as any[],
}),
),
TE.map((suggestions) => {
return suggestions.find((s) => s.event.type === type);
}),
TE.map(O.fromNullable),
);
}
return TE.right(O.none);
},
),
),
),
TE.chain(({ link, relations }) =>
TE.tryCatch(() => {
const suggestionMaker = getSuggestions((v) =>
Promise.resolve(toInitialValue(v)),
);

TE.map(
({
relations: {
relations: { entities },
},
suggestions,
}) =>
pipe(
suggestions,
O.map((s) => ({
...s.event,
id: uuid(),
excerpt: s.event.excerpt ?? null,
body: s.event.body ?? null,
location: null,
links: [l],
keywords: [],
media: [],
events: [],
socialPosts: [],
actors: entities.actors,
groups: entities.groups,
stories: [],
createdAt: new Date(),
updatedAt: new Date(),
deletedAt: null,
})),
return suggestionMaker(metadata, link, O.none, relations);
}, ServerError.fromUnknown),
),
TE.map((suggestions) => {
return suggestions.find((s) => s.event.type === type);
}),
TE.map(O.fromNullable),
);
}),

TE.map(({ relations: { entities }, suggestions }) =>
pipe(
suggestions,
O.map((s) => ({
...s.event,
id: uuid(),
excerpt: s.event.excerpt ?? null,
body: s.event.body ?? null,
location: null,
links: [l],
keywords: [],
media: [],
events: [],
socialPosts: [],
actors: entities.actors,
groups: entities.groups,
stories: [],
createdAt: new Date(),
updatedAt: new Date(),
deletedAt: null,
})),
),
),
);
};
Expand Down
Loading

0 comments on commit 7219c32

Please sign in to comment.