fix(backend): create event from url subscriber

compose.yml

@@ -125,8 +125,8 @@ services:
       - worker-node-modules:/usr/src/app/node_modules
       - ./packages/@liexp:/usr/src/app/packages/@liexp:ro
       - ./services/worker:/usr/src/app/services/worker:ro
-      - ./services/worker/config:/usr/src/app/services/worker/config:rw
-      - ./services/worker/temp:/usr/src/app/services/worker/temp:rw
+      - ./services/worker/config:/usr/src/app/services/worker/config:cached
+      - ./services/worker/temp:/usr/src/app/services/worker/temp:cached
     mem_limit: 512M
     restart: always
     networks:

lies.exposed.code-workspace

@@ -63,6 +63,7 @@
       "**/bower_components": true,
       "**/*.code-search": true,
       "build/**": true,
+      "coverage/**": true,
       "lib/**": true,
     },
     "eslint.workingDirectories": [

packages/@liexp/backend/src/flows/event/scientific-study/createFromURL.flow.ts → packages/@liexp/backend/src/flows/event/createEventFromURL.flow.ts

@@ -1,24 +1,24 @@
 import { fp, pipe } from "@liexp/core/lib/fp/index.js";
-import { type URL } from "@liexp/shared/lib/io/http/Common/index.js";
-import { SCIENTIFIC_STUDY } from "@liexp/shared/lib/io/http/Events/EventType.js";
+import { type UUID, type URL } from "@liexp/shared/lib/io/http/Common/index.js";
+import { type EventType } from "@liexp/shared/lib/io/http/Events/EventType.js";
 import * as O from "fp-ts/lib/Option.js";
 import { type ReaderTaskEither } from "fp-ts/lib/ReaderTaskEither.js";
 import * as TE from "fp-ts/lib/TaskEither.js";
 import { Equal } from "typeorm";
-import { type ConfigContext } from "../../../context/config.context.js";
-import { type DatabaseContext } from "../../../context/db.context.js";
-import { type FSClientContext } from "../../../context/fs.context.js";
+import { type ConfigContext } from "../../context/config.context.js";
+import { type DatabaseContext } from "../../context/db.context.js";
+import { type FSClientContext } from "../../context/fs.context.js";
 import {
   type NERProviderContext,
   type URLMetadataContext,
-} from "../../../context/index.js";
-import { type LoggerContext } from "../../../context/logger.context.js";
-import { type PuppeteerProviderContext } from "../../../context/puppeteer.context.js";
-import { EventV2Entity } from "../../../entities/Event.v2.entity.js";
-import { type UserEntity } from "../../../entities/User.entity.js";
-import { ServerError } from "../../../errors/ServerError.js";
-import { findByURL } from "../../../queries/events/scientificStudy.query.js";
-import { extractEventFromURL } from "../extractFromURL.flow.js";
+} from "../../context/index.js";
+import { type LoggerContext } from "../../context/logger.context.js";
+import { type PuppeteerProviderContext } from "../../context/puppeteer.context.js";
+import { EventV2Entity } from "../../entities/Event.v2.entity.js";
+import { type UserEntity } from "../../entities/User.entity.js";
+import { ServerError } from "../../errors/ServerError.js";
+import { findByURL } from "../../queries/events/scientificStudy.query.js";
+import { extractEventFromURL } from "./extractFromURL.flow.js";
 
 export const createEventFromURL = <
   C extends LoggerContext &
@@ -30,7 +30,9 @@ export const createEventFromURL = <
     PuppeteerProviderContext,
 >(
   user: UserEntity,
+  eventId: UUID,
   url: URL,
+  type: EventType,
 ): ReaderTaskEither<C, ServerError, EventV2Entity> => {
   return pipe(
     findByURL<C>(url),
@@ -43,30 +45,32 @@ export const createEventFromURL = <
         fp.RTE.ask<C>(),
         fp.RTE.chainTaskEitherK((ctx) =>
           pipe(
-            ctx.puppeteer.getBrowser({}),
-            TE.mapLeft(ServerError.fromUnknown),
-            TE.chain((b) =>
-              pipe(
+            ctx.puppeteer.execute({}, (b) => {
+              return pipe(
                 TE.tryCatch(
                   () => b.pages().then((p) => p[0]),
                   ServerError.fromUnknown,
                 ),
                 TE.chain((p) =>
                   extractEventFromURL(p, user, {
-                    type: SCIENTIFIC_STUDY.value,
+                    type,
                     url,
                   })(ctx),
                 ),
-                TE.chainFirst(() => {
-                  return TE.tryCatch(() => b.close(), ServerError.fromUnknown);
-                }),
-              ),
-            ),
+              );
+            }),
             TE.chain((meta) => {
               if (O.isSome(meta)) {
-                return ctx.db.save(EventV2Entity, [meta.value]);
+                return ctx.db.save(EventV2Entity, [
+                  { ...meta.value, id: eventId },
+                ]);
               }
-              return TE.left(ServerError.of());
+
+              return TE.left(
+                ServerError.fromUnknown(
+                  new Error(`No event can be extracted from url #{url} `),
+                ),
+              );
             }),
             TE.chain(([result]) =>
               ctx.db.findOneOrFail(EventV2Entity, {

packages/@liexp/backend/src/flows/event/extractFromURL.flow.ts

@@ -28,7 +28,7 @@ import { ServerError } from "../../errors/ServerError.js";
 import { extractRelationsFromURL } from "../admin/nlp/extractRelationsFromURL.flow.js";
 import { fetchAndSave } from "../links/link.flow.js";
 
-const extractEventFromProviderLink =
+const extractPageMetadataFromProviderLink =
   <C extends LoggerContext>(
     p: puppeteer.Page,
     host: string,
@@ -170,14 +170,15 @@ const extractByProvider =
     p: puppeteer.Page,
     host: string,
     l: LinkEntity,
+    type: EventType,
   ): ReaderTaskEither<C, ServerError, O.Option<EventV2Entity>> =>
   (ctx) => {
     return pipe(
       TE.Do,
       TE.bind("relations", () =>
         sequenceS(TE.ApplicativePar)({
           relations: extractRelationsFromURL(p, l.url)(ctx),
-          provider: extractEventFromProviderLink(p, host, l)(ctx),
+          provider: extractPageMetadataFromProviderLink(p, host, l)(ctx),
         }),
       ),
       TE.bind(
@@ -240,7 +241,7 @@ const extractByProvider =
                 );
               }, ServerError.fromUnknown),
               TE.map((suggestions) =>
-                suggestions.find((s) => s.event.type === "ScientificStudy"),
+                suggestions.find((s) => s.event.type === type),
               ),
               TE.map(O.fromNullable),
             );
@@ -305,7 +306,10 @@ export const extractEventFromURL =
     ctx.logger.debug.log("Extracting event from host %s (%s)", host, l.url);
 
     return pipe(
-      fetchAndSave(user, l.url)(ctx),
-      TE.chain((le) => extractByProvider(p, host, le)(ctx)),
-    );
+      fp.RTE.Do,
+      fp.RTE.bind("event", () => fetchAndSave<C>(user, l.url)),
+      fp.RTE.chain(({ event }) => {
+        return pipe(extractByProvider<C>(p, host, event, l.type));
+      }),
+    )(ctx);
   };

packages/@liexp/backend/src/pubsub/events/createEventFromURL.pubSub.ts

@@ -0,0 +1,15 @@
+import { URL } from "@liexp/shared/lib/io/http/Common/URL.js";
+import { UUID } from "@liexp/shared/lib/io/http/Common/UUID.js";
+import { EventType } from "@liexp/shared/lib/io/http/Events/EventType.js";
+import * as t from "io-ts";
+import { RedisPubSub } from "../../providers/redis/redis.provider.js";
+
+export const CreateEventFromURLPubSub = RedisPubSub(
+  "event:create-from-url",
+  t.strict({
+    url: URL,
+    type: EventType,
+    eventId: UUID,
+    userId: UUID,
+  }),
+);

packages/@liexp/backend/src/queries/events/createEvent.query.ts

@@ -14,7 +14,7 @@ import { fetchRelationIds } from "./fetchEventRelations.query.js";
 export const createEventQuery = <
   C extends DatabaseContext & URLMetadataContext & LoggerContext,
 >(
-  input: http.Events.CreateEventBody,
+  input: http.Events.CreateEventPlainBody,
 ): ReaderTaskEither<C, DBError, DeepPartial<EventV2Entity>> => {
   return pipe(
     fetchRelationIds({

packages/@liexp/shared/src/endpoints/events/scientificStudy.endpoint.ts

@@ -75,7 +75,7 @@ export const Edit = Endpoint({
   Input: {
     Params: t.type({ id: UUID }),
     Body: t.strict(
-      propsOmit(CreateScientificStudyBody.types[0], ["type"]),
+      propsOmit(CreateScientificStudyBody, ["type"]),
       "CreateScientificStudyBody",
     ),
   },

packages/@liexp/shared/src/io/http/Events/ScientificStudy.ts

@@ -1,6 +1,5 @@
 import * as t from "io-ts";
 import { UUID } from "io-ts-types/lib/UUID.js";
-import { URL } from "../Common/URL.js";
 import {
   CreateEventCommon,
   EditEventCommon,
@@ -21,20 +20,12 @@ export const ScientificStudyPayload = t.strict(
 
 export type ScientificStudyPayload = t.TypeOf<typeof ScientificStudyPayload>;
 
-export const CreateScientificStudyPlainBody = t.strict(
+export const CreateScientificStudyBody = t.strict(
   {
     ...CreateEventCommon.type.props,
     type: SCIENTIFIC_STUDY,
     payload: ScientificStudyPayload,
   },
-  "CreateScientificStudyPlainBody",
-);
-export type CreateScientificStudyPlainBody = t.TypeOf<
-  typeof CreateScientificStudyPlainBody
->;
-
-export const CreateScientificStudyBody = t.union(
-  [CreateScientificStudyPlainBody, t.strict({ url: URL })],
   "CreateScientificStudyBody",
 );
 

packages/@liexp/shared/src/io/http/Events/index.ts

@@ -1,7 +1,7 @@
 import * as t from "io-ts";
 import { type Actor } from "../Actor.js";
 import { type Area } from "../Area.js";
-import { type UUID } from "../Common/index.js";
+import { URL, type UUID } from "../Common/index.js";
 import { type Group } from "../Group.js";
 import { type GroupMember } from "../GroupMember.js";
 import { type Keyword } from "../Keyword.js";
@@ -43,17 +43,33 @@ export const EventMap: { [key in Event["type"]]: t.Mixed } = {
   Quote: Quote.Quote,
 };
 
-export const CreateEventBody = t.union(
+export const EventFromURLBody = t.strict(
+  {
+    url: URL,
+    type: EventType,
+  },
+  "EventFromURLBody",
+);
+
+export type EventFromURLBody = t.TypeOf<typeof EventFromURLBody>;
+
+export const CreateEventPlainBody = t.union(
   [
     Book.CreateBookBody,
     Death.CreateDeathBody,
     Patent.CreatePatentBody,
-    ScientificStudy.CreateScientificStudyBody.types[0],
+    ScientificStudy.CreateScientificStudyBody,
     Uncategorized.CreateEventBody,
     Documentary.CreateDocumentaryBody,
     Transaction.CreateTransactionBody,
     Quote.CreateQuoteBody,
   ],
+  "CreateEventPlainBody",
+);
+export type CreateEventPlainBody = t.TypeOf<typeof CreateEventPlainBody>;
+
+export const CreateEventBody = t.union(
+  [CreateEventPlainBody, EventFromURLBody],
   "CreateEventBody",
 );
 

packages/@liexp/shared/src/tests/arbitrary/Event.arbitrary.ts

@@ -13,7 +13,7 @@ interface CreateEventBodyArbOpts {
   keywordIds?: boolean;
 }
 
-const createEventProps = propsOmit(http.Events.CreateEventBody.types[4], [
+const createEventProps = propsOmit(http.Events.CreateEventPlainBody.types[4], [
   "excerpt",
   "body",
   "date",
@@ -27,7 +27,7 @@ export const CreateEventBodyArb = ({
   linksIds = false,
   mediaIds = false,
   keywordIds = false,
-}: CreateEventBodyArbOpts = {}): fc.Arbitrary<http.Events.CreateEventBody> =>
+}: CreateEventBodyArbOpts = {}): fc.Arbitrary<http.Events.CreateEventPlainBody> =>
   getArbitrary(t.strict(createEventProps)).map((b) => ({
     ...b,
     excerpt: undefined,

packages/@liexp/shared/src/tests/arbitrary/events/ScientificStudy.arbitrary.ts

@@ -7,7 +7,7 @@ import { URLArb } from "../URL.arbitrary.js";
 import { UUIDArb } from "../common/UUID.arbitrary.js";
 
 const createScientificStudyProps = propsOmit(
-  http.Events.ScientificStudy.CreateScientificStudyBody.types[0],
+  http.Events.ScientificStudy.CreateScientificStudyBody,
   ["excerpt", "body", "date", "draft", "payload", "media", "links", "keywords"],
 );
 

packages/@liexp/ui/src/components/admin/events/CreateEventFromLinkButton.tsx

@@ -1,18 +1,13 @@
 import { flow, fp } from "@liexp/core/lib/fp/index.js";
-import { getRelationIdsFromEventRelations } from "@liexp/shared/lib/helpers/event/getEventRelationIds.js";
-import { getSuggestions } from "@liexp/shared/lib/helpers/event-suggestion.js";
 import { type Link } from "@liexp/shared/lib/io/http/Link.js";
-import { type Media } from "@liexp/shared/lib/io/http/Media/Media.js";
 import { type EventSuggestion } from "@liexp/shared/lib/io/http/index.js";
 import * as io from "@liexp/shared/lib/io/index.js";
 import { type Either } from "fp-ts/lib/Either.js";
-import * as O from "fp-ts/lib/Option.js";
 import { useRecordContext } from "ra-core";
 import * as React from "react";
 import { Button } from "react-admin";
 import { useNavigate } from "react-router";
 import { useDataProvider } from "../../../hooks/useDataProvider.js";
-import { toBNDocument } from "../../Common/BlockNote/utils/utils.js";
 import { ErrorBox } from "../../Common/ErrorBox.js";
 import { Box, MenuItem, Select } from "../../mui/index.js";
 import EventPreview from "../previews/EventPreview.js";
@@ -40,24 +35,8 @@ export const CreateEventFromLinkButton: React.FC = () => {
         return Promise.resolve(fp.E.right(suggestion));
       }
 
-      const result = await apiProvider
-        .get("open-graph/metadata", { url: link.url, type: "Link" })
-        .then(async ({ data: { metadata: m, relations } }: any) => {
-          const suggestions = await getSuggestions(toBNDocument)(
-            m,
-            O.some(link),
-            O.fromNullable(link.image as Media),
-            getRelationIdsFromEventRelations(relations.entities),
-          );
-
-          const suggestEvent = suggestions.find((t) => t.event.type === type);
-          if (suggestEvent) {
-            return fp.E.right(suggestEvent);
-          }
-          return fp.E.left(new Error("No suggestion found"));
-        });
-
-      return result;
+      const result = await apiProvider.post("events", { url: link.url, type });
+      return result.data;
     },
     [record, type],
   );

services/api/src/flows/events/scientific-studies/createFromPlainObject.flow.ts

@@ -1,18 +1,19 @@
 import { EventV2Entity } from "@liexp/backend/lib/entities/Event.v2.entity.js";
 import { type KeywordEntity } from "@liexp/backend/lib/entities/Keyword.entity.js";
 import { type MediaEntity } from "@liexp/backend/lib/entities/Media.entity.js";
+import { type UserEntity } from "@liexp/backend/lib/entities/User.entity.js";
 import { pipe } from "@liexp/core/lib/fp/index.js";
 import { UUID } from "@liexp/shared/lib/io/http/Common/index.js";
-import { type CreateScientificStudyPlainBody } from "@liexp/shared/lib/io/http/Events/ScientificStudy.js";
+import { type CreateScientificStudyBody } from "@liexp/shared/lib/io/http/Events/ScientificStudy.js";
 import * as TE from "fp-ts/lib/TaskEither.js";
 import { type DeepPartial, Equal } from "typeorm";
 import { type TEReader } from "#flows/flow.types.js";
 
 export const createScientificStudyFromPlainObject =
-  ({
-    payload,
-    ...body
-  }: CreateScientificStudyPlainBody): TEReader<EventV2Entity> =>
+  (
+    { payload, ...body }: CreateScientificStudyBody,
+    user: UserEntity,
+  ): TEReader<EventV2Entity> =>
   (ctx) => {
     const scientificStudyData = {
       ...body,
@@ -53,6 +54,7 @@ export const createScientificStudyFromPlainObject =
       ctx.db.save(EventV2Entity, [
         {
           ...scientificStudyData,
+          user,
         },
       ]),
       TE.chain(([result]) =>