Skip to content

Commit

Permalink
feat: lbac 2220: ajout du rome aux computed job partners (#1636)
Browse files Browse the repository at this point in the history
* feat: ajout du rome aux computed job partners

* fix: update fillFieldsForPartnersFactory documentation

* fix: typing

* fix: tests

* fix: review Kevin

* fix: tests

* fix: talisman
  • Loading branch information
remy-auricoste authored Nov 27, 2024
1 parent 13f2290 commit 7269665
Show file tree
Hide file tree
Showing 17 changed files with 304 additions and 144 deletions.
20 changes: 8 additions & 12 deletions .talismanrc
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
fileignoreconfig:
- filename: .infra/vault/vault.yml
checksum: 40717e8b277d9c9812c89d62fd3e09d0d70cdd0bb76a460f09486a31b6dd9cc7
- filename: .infra/files/configs/mongodb/seed.gpg
checksum: 183a82c08eca1f53d2d258b71abc36e5b352286f46912fcb022859b52f9fb0cc
- filename: .infra/files/scripts/seed.sh
checksum: b1361c237cee243b74fc9e66fd59af80c538d6884b8f94c3659d25cdc2b873c2
- filename: .infra/local/mongod.conf
checksum: bb2ce0c27102259a5fa39da1fb4460af9ad6ad58adc715312e53dcd69c8e6be7
- filename: .infra/vault/vault.yml
checksum: 40717e8b277d9c9812c89d62fd3e09d0d70cdd0bb76a460f09486a31b6dd9cc7
- filename: cypress/e2e/check-admin-algo-company.cy.ts
checksum: 6a898ee25c84ce2c41da5666aa4f84e6e4aaf0a614dbc0a00153abe99e3617c1
- filename: server/src/jobs/brevoContacts/sendContactsToBrevo.ts
checksum: cdb19cf5444adabf94cdac26c3f458415c633d35fdc523608ee0e1c0eab7feb1
- filename: server/src/services/formulaire.service.test.ts
checksum: a263b92a95e69de7efb856d73122b8f05ef98140c96dd0212213f26dce85056e
- filename: shared/routes/v3/jobs/jobs.routes.v3.model.ts
checksum: 94f2736d7791571d7888a50f53e36e2af246f22c1c832b85477c34cb5328af0d
- filename: cypress/e2e/manual/create-many-applications.cy.ts
checksum: 752e2ab70ab367400bc0e831904b09e3bb131fdfaafb8c38aff89393414f62c5
- filename: cypress/pages/FlowAdminPage.ts
Expand Down Expand Up @@ -88,7 +84,7 @@ fileignoreconfig:
- filename: server/src/services/emails.service.test.ts
checksum: b06fe36c0000e1ab1e45617b6cd402267d0492eb7867fbf8f4b817161fe671d9
- filename: server/src/services/formulaire.service.test.ts
checksum: 16a7d096bc389e6e8a0811a22c18dd6dab3c2875397fb327a85a95fc4a05d044
checksum: a263b92a95e69de7efb856d73122b8f05ef98140c96dd0212213f26dce85056e
- filename: server/src/services/formulaire.service.ts
checksum: e93ff7ce146d35e70eedcbe9b66097750e7754650468a5ada6b0ed72f0fdbc49
- filename: server/src/services/jobs/jobOpportunity/jobOpportunity.service.test.ts
Expand Down Expand Up @@ -174,7 +170,7 @@ fileignoreconfig:
- filename: shared/routes/v1Jobs.routes.ts
checksum: aa0fb2458520f24921a48af03ad05c3f4a92052374182851f24a3afa7421a5b8
- filename: shared/routes/v3/jobs/jobs.routes.v3.model.test.ts
checksum: aba2650704ad4267506ab7e1eccfdff9faa7ef1d7cc482a104eddf36c07fbd58
checksum: f4f3e429e6a7871a218f19eead273c0477636ed3c19b9e4dfd9e55bde1bc7dc8
- filename: shared/utils/objectUtils.ts
checksum: cfd0e48e8762b43c6431dad873727f6ac091ca6f5c7555e37a8ee3fba03184d1
- filename: ui/common/hooks/useAuth.ts
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export const franceTravailRomeoFixture = {
"Software Engineer": [
{
contexte: " ",
identifiant: "1",
identifiant: "0",
intitule: "Software Engineer",
metiersRome: [
{
Expand Down Expand Up @@ -48,7 +48,6 @@ export const franceTravailRomeoFixture = {
scorePrediction: 0.744,
},
],
uuidInference: "180d530a-474a-496b-8d3b-f3d91928c663",
},
],
} as const satisfies Record<string, IRomeoAPIResponse>
Expand Down
50 changes: 9 additions & 41 deletions server/src/jobs/franceTravail/pocRomeo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,52 +3,20 @@ import { logger } from "@/common/logger"
import { IRomeoPayload, getRomeoPredictions } from "../../common/apis/franceTravail/franceTravail.client"
import { getDbCollection } from "../../common/utils/mongodbUtils"

type MainObject = {
contexte: string
identifiant: string
intitule: string
uuidInference: string
metiersRome: MetierRome[] // Original metiersRome array
[key: string]: any // Allows adding dynamic keys
}

type MetierRome = {
codeAppellation: string
codeRome: string
libelleAppellation: string
libelleRome: string
scorePrediction: number
}

function transformMetiersRomeArray(objects: MainObject[]): MainObject[] {
return objects.map((object) => {
const { metiersRome, ...rest } = object

metiersRome.forEach((metier, index) => {
const suffix = index + 1 // Start from 1
for (const [key, value] of Object.entries(metier)) {
rest[`${key}${suffix}`] = value
}
})

return rest as MainObject
})
}

export const pocRomeo = async () => {
const result = (await getDbCollection("raw_hellowork")
const rawHelloworkDocuments = await getDbCollection("raw_hellowork")
.find({}, { projection: { "job.title": 1, "job.company_sector": 1 } })
.limit(3)
.toArray()) as any
const dataset: IRomeoPayload[] = result.map(
({ _id, job }): IRomeoPayload => ({
.toArray()
const dataset: IRomeoPayload[] = rawHelloworkDocuments.map((document) => {
const { _id } = document
const job = document.job as any
return {
intitule: job.title,
identifiant: _id.toString(),
contexte: job.company_sector,
})
)
}
})
const romeoResult = await getRomeoPredictions(dataset)
if (!romeoResult) return null
const formated = transformMetiersRomeArray(romeoResult)
logger.info(formated)
logger.info(romeoResult)
}
2 changes: 2 additions & 0 deletions server/src/jobs/offrePartenaire/fillComputedJobsPartners.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { fillOpcoInfosForPartners } from "./fillOpcoInfosForPartners"
import { fillRomeForPartners } from "./fillRomeForPartners"
import { fillSiretInfosForPartners } from "./fillSiretInfosForPartners"
import { validateComputedJobPartners } from "./validateComputedJobPartners"

export const fillComputedJobsPartners = async () => {
await fillOpcoInfosForPartners()
await fillSiretInfosForPartners()
await fillRomeForPartners()
await validateComputedJobPartners()
}
25 changes: 11 additions & 14 deletions server/src/jobs/offrePartenaire/fillFieldsForPartnersFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import { streamGroupByCount } from "@/common/utils/streamUtils"
* @param sourceFields: champs nécessaires à la récupération des données
* @param filledFields: champs potentiellement modifiés par l'enrichissement
* @param groupSize: taille du packet de documents (utile pour optimiser les appels API et BDD)
* @param getData: fonction récupérant les nouvelles données. Les champs retournés seront modifiés et écraseront les anciennes données
* @param getData: fonction récupérant les nouvelles données.
* La fonction doit retourner un tableau d'objet contenant l'_id du document à mettre à jour et les nouvelles valeurs à mettre à jour.
* Les valeurs retournées seront modifiées et écraseront les anciennes données.
*/
export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJobsPartnersOfferPrivate, FilledFields extends keyof IJobsPartnersOfferPrivate | "business_error">({
job,
Expand All @@ -28,7 +30,7 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
job: COMPUTED_ERROR_SOURCE
sourceFields: readonly SourceFields[]
filledFields: readonly FilledFields[]
getData: (sourceFields: Pick<IComputedJobsPartners, SourceFields | FilledFields>[]) => Promise<Array<Pick<IComputedJobsPartners, FilledFields> | undefined>>
getData: (sourceFields: Pick<IComputedJobsPartners, SourceFields | FilledFields | "_id">[]) => Promise<Array<Pick<IComputedJobsPartners, FilledFields | "_id">>>
groupSize: number
}) => {
const logger = globalLogger.child({
Expand Down Expand Up @@ -68,19 +70,14 @@ export const fillFieldsForPartnersFactory = async <SourceFields extends keyof IJ
try {
const responses = await getData(documents)

const dataToWrite = documents.flatMap((document, index) => {
const newFields = responses[index]
if (!newFields) {
return []
}
return [
{
updateOne: {
filter: { _id: document._id },
update: { $set: newFields },
},
const dataToWrite = responses.map((document) => {
const { _id, ...newFields } = document
return {
updateOne: {
filter: { _id },
update: { $set: newFields },
},
]
}
})
if (dataToWrite.length) {
await getDbCollection("computed_jobs_partners").bulkWrite(dataToWrite, {
Expand Down
4 changes: 3 additions & 1 deletion server/src/jobs/offrePartenaire/fillOpcoInfosForPartners.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ export const fillOpcoInfosForPartners = async () => {
const opcoData = opcosData.find((data) => data.siret === document.workplace_siret)
if (!opcoData) {
return {
_id: document._id,
workplace_idcc: document.workplace_idcc ?? null,
workplace_opco: document.workplace_opco ?? OPCOS_LABEL.UNKNOWN_OPCO,
}
}

const result: Pick<IComputedJobsPartners, (typeof filledFields)[number]> = {
const result: Pick<IComputedJobsPartners, (typeof filledFields)[number] | "_id"> = {
_id: document._id,
workplace_idcc: document.workplace_idcc ?? opcoData.idcc,
workplace_opco: document.workplace_opco ?? opcoData.opco,
}
Expand Down
116 changes: 116 additions & 0 deletions server/src/jobs/offrePartenaire/fillRomeForPartners.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
import { givenSomeComputedJobPartners } from "@tests/fixture/givenSomeComputedJobPartners"
import { useMongo } from "@tests/utils/mongo.test.utils"
import nock from "nock"
import { IRomeoAPIResponse } from "shared/models/cacheRomeo.model"
import { beforeEach, describe, expect, it, vi } from "vitest"

import { nockFranceTravailRomeo } from "@/common/apis/franceTravail/franceTravail.client.fixture"
import { getDbCollection } from "@/common/utils/mongodbUtils"

import { cacheRomeFixture, cacheRomeResultFixture } from "../../../../shared/fixtures/cacheRome.fixture"
import { nockFranceTravailTokenAccessRomeo } from "../../common/apis/franceTravail/franceTravail.client.fixture"

import { fillRomeForPartners } from "./fillRomeForPartners"

const now = new Date("2024-07-21T04:49:06.000+02:00")

describe("fillRomeForPartners", () => {
useMongo()

beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(now)

nock("https://api.francetravail.io").post(/.*/).reply(404)

return async () => {
vi.useRealTimers()
nock.cleanAll()
await getDbCollection("computed_jobs_partners").deleteMany({})
await getDbCollection("cache_romeo").deleteMany({})
}
})

const title = "Chef de partie, second de cuisine H/F"
const nafLabel = "Commerce de détail d'habillement en magasin spécialisé"

it("should enrich with cache", async () => {
// given
const romeCode = "K1601"
await givenSomeComputedJobPartners([
{
offer_title: title,
workplace_naf_label: nafLabel,
offer_rome_codes: null,
},
])
const cacheRomeo = cacheRomeFixture({
intitule: title,
contexte: nafLabel,
metiersRome: [
cacheRomeResultFixture({
codeRome: romeCode,
}),
],
})
await getDbCollection("cache_romeo").insertOne(cacheRomeo)
// when
await fillRomeForPartners()
// then
const jobs = await getDbCollection("computed_jobs_partners").find({}).toArray()
expect.soft(jobs.length).toBe(1)
const [job] = jobs
const { offer_rome_codes } = job
expect.soft(job.errors).toEqual([])
expect.soft(offer_rome_codes).toEqual([romeCode])
})
it("should enrich with api", async () => {
// given
const romeCode = "J1501"
await givenSomeComputedJobPartners([
{
offer_title: title,
workplace_naf_label: nafLabel,
offer_rome_codes: null,
},
])
nock.cleanAll()
nockFranceTravailTokenAccessRomeo()
const apiResponse: IRomeoAPIResponse = [
{
intitule: title,
identifiant: "0",
contexte: nafLabel,
metiersRome: [
cacheRomeResultFixture({
codeRome: romeCode,
scorePrediction: 0.8,
}),
cacheRomeResultFixture({
codeRome: "ignored",
scorePrediction: 0.7,
}),
],
},
]
nockFranceTravailRomeo(
[
{
intitule: title,
identifiant: "0",
contexte: nafLabel,
},
],
apiResponse
)
// when
await fillRomeForPartners()
// then
const jobs = await getDbCollection("computed_jobs_partners").find({}).toArray()
expect.soft(jobs.length).toBe(1)
const [job] = jobs
const { offer_rome_codes } = job
expect.soft(job.errors).toEqual([])
expect.soft(offer_rome_codes).toEqual([romeCode])
})
})
33 changes: 33 additions & 0 deletions server/src/jobs/offrePartenaire/fillRomeForPartners.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { COMPUTED_ERROR_SOURCE, IComputedJobsPartners } from "shared/models/jobsPartnersComputed.model"

import { getRomesInfos } from "../../services/cacheRomeo.service"

import { fillFieldsForPartnersFactory } from "./fillFieldsForPartnersFactory"

export const fillRomeForPartners = async () => {
const filledFields = ["offer_rome_codes"] as const satisfies (keyof IComputedJobsPartners)[]
return fillFieldsForPartnersFactory({
job: COMPUTED_ERROR_SOURCE.API_ROMEO,
sourceFields: ["offer_title", "workplace_naf_label"],
filledFields,
groupSize: 50,
getData: async (documents) => {
const validDocuments = documents.flatMap((document) => (document.offer_title ? [document] : []))
const queries = validDocuments.map(({ offer_title, workplace_naf_label }) => ({ intitule: offer_title!, contexte: workplace_naf_label }))
const allRomeInfos = await getRomesInfos(queries)

return validDocuments.flatMap((document, index) => {
const romeInfos = allRomeInfos[index]
if (!romeInfos) {
return []
}

const result: Pick<IComputedJobsPartners, (typeof filledFields)[number] | "_id"> = {
_id: document._id,
offer_rome_codes: [romeInfos],
}
return [result]
})
},
})
}
5 changes: 3 additions & 2 deletions server/src/jobs/offrePartenaire/fillSiretInfosForPartners.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ export const fillSiretInfosForPartners = async () => {

const business_error = data?.etat_administratif === "F" ? JOB_PARTNER_BUSINESS_ERROR.CLOSED_COMPANY : null

const result: Pick<IComputedJobsPartners, (typeof filledFields)[number]> = {
const result: Pick<IComputedJobsPartners, (typeof filledFields)[number] | "_id"> = {
_id: document._id,
workplace_size: document.workplace_size ?? establishment_size,
workplace_legal_name: document.workplace_legal_name ?? establishment_raison_sociale,
workplace_brand: document.workplace_brand ?? establishment_enseigne,
Expand All @@ -52,7 +53,7 @@ export const fillSiretInfosForPartners = async () => {
workplace_naf_code: document.workplace_naf_code ?? naf_code,
workplace_naf_label: document.workplace_naf_label ?? naf_label,
workplace_geopoint: document.workplace_geopoint ?? (geo_coordinates ? convertStringCoordinatesToGeoPoint(geo_coordinates) : null),
business_error,
business_error: document.business_error ?? business_error,
}

return [result]
Expand Down
3 changes: 2 additions & 1 deletion server/src/jobs/offrePartenaire/importRHAlternance.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { useMongo } from "@tests/utils/mongo.test.utils"
import omit from "lodash-es/omit"
import { beforeEach, describe, expect, it, vi } from "vitest"

import { getDbCollection } from "@/common/utils/mongodbUtils"
Expand Down Expand Up @@ -27,7 +28,7 @@ describe("import RH Alternance", () => {
it("should test that the mapper works", async () => {
const mapped = rawRhAlternanceToComputedMapper(now)(generateRawRHAlternanceJobFixture())
expect.soft(mapped.business_error).toBeFalsy()
expect.soft(mapped).toMatchSnapshot()
expect.soft(omit(mapped, "_id")).toMatchSnapshot()
})
it("should detect a business error", async () => {
const mapped = rawRhAlternanceToComputedMapper(now)(
Expand Down
1 change: 1 addition & 0 deletions server/src/jobs/offrePartenaire/importRHAlternance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ export const rawRhAlternanceToComputedMapper =
const offer_creation = jobSubmitDateTime ? dayjs.tz(jobSubmitDateTime).toDate() : now
const isValid: boolean = jobType === "Alternance"
const computedJob: IComputedJobsPartners = {
_id: new ObjectId(),
partner_job_id: jobCode,
partner_label: JOBPARTNERS_LABEL.RH_ALTERNANCE,
contract_type: jobType === "Alternance" ? [TRAINING_CONTRACT_TYPE.APPRENTISSAGE, TRAINING_CONTRACT_TYPE.PROFESSIONNALISATION] : [],
Expand Down
Loading

0 comments on commit 7269665

Please sign in to comment.