codegouvfr · guillermau · Mar 3, 2025 · Mar 3, 2025 · Mar 3, 2025 · Mar 4, 2025
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -36,5 +36,17 @@
                 "update"
             ],
         }
+        ,
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Fullcheck",
+            "runtimeExecutable": "yarn",
+            "cwd": "${workspaceFolder}",
+            "runtimeArgs": [
+                "run",
+                "fullcheck"
+            ],
+        }
     ]
 }
diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts
@@ -7,7 +7,10 @@ import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialec
 import { CompiledData } from "../src/core/ports/CompileData";
 import { Db } from "../src/core/ports/DbApi";
 import { ExternalDataOrigin } from "../src/core/ports/GetSoftwareExternalData";
-import SoftwareRow = Db.SoftwareRow;
+import { getOrPopulateFromIds } from "../src/core/usecases/createSoftwareFromForm";
+import { wikidataAdapter } from "../src/core/adapters/wikidata";
+import { getDbApiAndInitializeCache } from "../src/core/bootstrap";
+import { DbApiV2 } from "../src/core/ports/DbApiV2";
 
 export type Params = {
     pgConfig: { dbUrl: string };
@@ -18,14 +21,21 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => {
     const { dbApi: gitDbApi } = createGitDbApi(gitDbConfig);
     if (!pgConfig.dbUrl) throw new Error("Missing PG database url, please set the DATABASE_URL environnement variable");
     const pgDb = new Kysely<Database>({ dialect: createPgDialect(pgConfig.dbUrl) });
+    const { dbApi } = getDbApiAndInitializeCache({ dbKind: "kysely", kyselyDb: pgDb });
 
-    const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb();
+    const {
+        composedSoftwareRows: softwareRows,
+        agentRows,
+        softwareReferentRows,
+        softwareUserRows,
+        instanceRows
+    } = await gitDbApi.fetchDb();
 
     await insertAgents(agentRows, pgDb);
 
     const agentIdByEmail = await makeGetAgentIdByEmail(pgDb);
 
-    await insertSoftwares(softwareRows, agentIdByEmail, pgDb);
+    await insertSoftwares(softwareRows, agentIdByEmail, pgDb, dbApi);
     await insertSoftwareReferents({
         softwareReferentRows: softwareReferentRows,
         agentIdByEmail,
@@ -47,9 +57,10 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => {
 };
 
 const insertSoftwares = async (
-    softwareRows: SoftwareRow[],
+    softwareRows: Db.ComposedSoftwareRow[],
     agentIdByEmail: Record<string, number>,
-    db: Kysely<Database>
+    db: Kysely<Database>,
+    dbApi: DbApiV2
 ) => {
     console.info("Deleting than Inserting softwares");
     console.info("Number of softwares to insert : ", softwareRows.length);
@@ -59,7 +70,7 @@ const insertSoftwares = async (
         await trx
             .insertInto("softwares")
             .values(
-                softwareRows.map(({ similarSoftwareExternalDataIds: _, addedByAgentEmail, ...row }) => ({
+                softwareRows.map(({ similarSoftwareIds: _, addedByAgentEmail, ...row }) => ({
                     ...row,
                     addedByAgentId: agentIdByEmail[addedByAgentEmail],
                     dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null,
@@ -72,17 +83,43 @@ const insertSoftwares = async (
             .executeTakeFirst();
         await sql`SELECT setval('softwares_id_seq', (SELECT MAX(id) FROM softwares))`.execute(trx);
 
-        await trx
-            .insertInto("softwares__similar_software_external_datas")
-            .values(
-                softwareRows.flatMap(row =>
-                    Array.from(new Set(row.similarSoftwareExternalDataIds)).map(externalId => ({
-                        softwareId: row.id,
-                        similarExternalId: externalId
-                    }))
-                )
-            )
-            .execute();
+        // Array<{ softwareId: number; similarSoftwareIds: Array<number> } | undefined>
+        const similarSoftware = await Promise.all(
+            softwareRows.map(async software => {
+                const softwareDbRow = await dbApi.software.getByName(software.name);
+
+                if (softwareDbRow) {
+                    const fetchedSimilarSoftwareIds = await getOrPopulateFromIds({
+                        dbApi,
+                        externalDataService: wikidataAdapter,
+                        similarSoftwareExternalDataIds: software.similarSoftwareExternalDataIds,
+                        agentId: agentIdByEmail[software.addedByAgentEmail]
+                    });
+                    return {
+                        softwareId: softwareDbRow.softwareId,
+                        similarSoftwareIds: fetchedSimilarSoftwareIds
+                    };
+                } else {
+                    console.error("Importation faiiled or record cannot be found.");
+                    return undefined;
+                }
+            })
+        );
+
+        const dbSimilarSoftware = similarSoftware
+            .filter(software => software !== undefined)
+            .map(similarCouple => {
+                return similarCouple.similarSoftwareIds.map(similarSoftwareId => {
+                    return {
+                        softwareId: similarCouple.softwareId,
+                        similarSoftwareId: similarSoftwareId
+                    };
+                });
+            })
+            .flat();
+
+        // TODO Insert software
+        await trx.insertInto("softwares__similar_software_external_datas").values(dbSimilarSoftware).execute();
     });
 };
 

diff --git a/api/src/core/adapters/dbApi/createGitDbApi.ts b/api/src/core/adapters/dbApi/createGitDbApi.ts
@@ -73,7 +73,8 @@ export function createGitDbApi(params: GitDbApiParams): { dbApi: DbApi; initiali
                         ).then(buffers => buffers.map(buffer => JSON.parse(buffer.toString("utf8"))));
 
                     dOut.resolve({
-                        softwareRows,
+                        softwareRows: [], // Doesn't exist in GitRepo
+                        composedSoftwareRows: softwareRows,
                         agentRows,
                         softwareReferentRows,
                         softwareUserRows,
@@ -95,7 +96,7 @@ export function createGitDbApi(params: GitDbApiParams): { dbApi: DbApi; initiali
                     await Promise.all(
                         (
                             [
-                                [softwareJsonRelativeFilePath, newDb.softwareRows],
+                                [softwareJsonRelativeFilePath, newDb.composedSoftwareRows],
                                 [agentJsonRelativeFilePath, newDb.agentRows],
                                 [softwareReferentJsonRelativeFilePath, newDb.softwareReferentRows],
                                 [softwareUserJsonRelativeFilePath, newDb.softwareUserRows],

diff --git a/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts b/api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts
@@ -23,16 +23,9 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
         .leftJoin("instances", "s.id", "instances.mainSoftwareSillId")
         .leftJoin("software_external_datas as ext", "ext.externalId", "s.externalId")
         .leftJoin("software_external_datas as parentExt", "parentExt.externalId", "s.parentSoftwareWikidataId")
-        .leftJoin(
-            "softwares__similar_software_external_datas",
-            "softwares__similar_software_external_datas.softwareId",
-            "s.id"
-        )
-        .leftJoin(
-            "software_external_datas as similarExt",
-            "softwares__similar_software_external_datas.similarExternalId",
-            "similarExt.externalId"
-        )
+        .leftJoin("softwares__similar_software_external_datas as sse", "sse.softwareId", "s.id")
+        .leftJoin("softwares as similarSoft", "sse.similarSoftwareId", "similarSoft.id")
+        .leftJoin("software_external_datas as similarExt", "similarSoft.externalId", "similarExt.externalId")
         .groupBy(["s.id", "csft.softwareId", "parentExt.externalId", "ext.externalId"])
         .select([
             "s.id",
@@ -97,6 +90,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                     )
                     .end()
                     .as("softwareExternalData"),
+            ({ fn }) => fn.jsonAgg("similarSoft").distinct().as("similarSoftwares"),
             ({ fn }) => fn.jsonAgg("similarExt").distinct().as("similarExternalSoftwares"),
             ({ fn }) => fn.jsonAgg("users").distinct().as("users"),
             ({ fn }) => fn.jsonAgg("referents").distinct().as("referents"),
@@ -115,6 +109,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                     latestVersion,
                     parentWikidataSoftware,
                     serviceProviders,
+                    similarSoftwares,
                     similarExternalSoftwares,
                     dereferencing,
                     doRespectRgaa,
@@ -130,7 +125,8 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                         ...stripNullOrUndefinedValues(software),
                         addedByAgentEmail: agentById[addedByAgentId].email,
                         updateTime: new Date(+updateTime).getTime(),
-                        referencedSinceTime: new Date(+referencedSinceTime).getTime(),
+                        referencedSinceTime: referencedSinceTime ? new Date(referencedSinceTime).getTime() : undefined,
+                        isReferenced: referencedSinceTime ? true : false,
                         doRespectRgaa,
                         softwareExternalData: softwareExternalData ?? undefined,
                         annuaireCnllServiceProviders: annuaireCnllServiceProviders ?? undefined,
@@ -139,6 +135,7 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
                         parentWikidataSoftware: parentWikidataSoftware ?? undefined,
                         dereferencing: dereferencing ?? undefined,
                         serviceProviders: serviceProviders ?? [],
+                        similarSoftwares: similarSoftwares?.map(software => software.id ?? 3), // TODOO
                         similarExternalSoftwares: (similarExternalSoftwares ?? [])
                             .filter(isNotNull)
                             .map(similar => ({

diff --git a/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts b/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts
@@ -8,6 +8,7 @@ import { createPgSoftwareExternalDataRepository } from "./createPgSoftwareExtern
 import { createPgSoftwareRepository } from "./createPgSoftwareRepository";
 import { createPgReferentRepository, createPgUserRepository } from "./createPgUserAndReferentRepository";
 import { Database } from "./kysely.database";
+import { createPgSimilarSoftwareRepository } from "./createPgSimilarSoftwareRepository";
 
 export const createKyselyPgDbApi = (db: Kysely<Database>): DbApiV2 => {
     return {
@@ -18,6 +19,7 @@ export const createKyselyPgDbApi = (db: Kysely<Database>): DbApiV2 => {
         agent: createPgAgentRepository(db),
         softwareReferent: createPgReferentRepository(db),
         softwareUser: createPgUserRepository(db),
+        similarSoftware: createPgSimilarSoftwareRepository(db),
         getCompiledDataPrivate: createGetCompiledData(db)
     };
 };
diff --git a/api/src/core/adapters/dbApi/kysely/createPgSimilarSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSimilarSoftwareRepository.ts
@@ -0,0 +1,32 @@
+import { Kysely } from "kysely";
+import { SimilarSoftwareRepository } from "../../../ports/DbApiV2";
+import { Database } from "./kysely.database";
+
+export const createPgSimilarSoftwareRepository = (db: Kysely<Database>): SimilarSoftwareRepository => ({
+    create: (
+        similars: {
+            softwareId: number;
+            similarSoftwareId: number;
+        }[],
+        ignore?: boolean
+    ) => {
+        return db.transaction().execute(async trx => {
+            let reqest = trx.insertInto("softwares__similar_software_external_datas");
+            if (ignore) {
+                reqest = reqest.ignore();
+            }
+
+            await reqest.values(similars).execute();
+
+            return Promise.resolve();
+        });
+    },
+    getBySoftwareId: async (id: number) => {
+        const result = await db
+            .selectFrom("softwares__similar_software_external_datas")
+            .selectAll()
+            .where("softwareId", "=", id)
+            .execute();
+        return result.map(row => row.similarSoftwareId);
+    }
+});