Skip to content

Commit 5cb2f40

Browse files
guillermauguillermau
committed
#306 : Refactor HAL connector and fixes
1 parent 97c4902 commit 5cb2f40

File tree

11 files changed

+94
-34
lines changed

11 files changed

+94
-34
lines changed

‎api/src/connectors/HAL/getHalSoftware.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,23 +53,33 @@ export async function fetchHalSoftwareById(halDocid: string): Promise<HAL.API.So
5353

5454
return json.response.docs[0];
5555
}
56+
type Props = {
57+
queryString?: string;
58+
SWHFilter?: boolean;
59+
};
5660

57-
export async function fetchHalSoftwares(queryString: string = ""): Promise<Array<HAL.API.Software>> {
58-
// Filter only software who have an swhidId to filter clean data on https://hal.science, TODO remove and set it as an option to be generic
59-
let url = `https://api.archives-ouvertes.fr/search/?q=docType_s:SOFTWARE&rows=10000&fl=${halSoftwareFieldsToReturnAsString}&fq=swhidId_s:["" TO *]`;
61+
export async function fetchHalSoftwares(params: Props): Promise<Array<HAL.API.Software>> {
62+
const { queryString, SWHFilter } = params;
63+
64+
let url = `https://api.archives-ouvertes.fr/search/?fq=docType_s:SOFTWARE&rows=10000&fl=${halSoftwareFieldsToReturnAsString}`;
6065

6166
if (queryString) {
6267
url = url + `&q=${encodeURIComponent(queryString)}`;
6368
}
6469

70+
// Filter only software who have an swhidId to filter clean data on https://hal.science, TODO remove and set it as an option to be generic
71+
if (SWHFilter) {
72+
url = url + `&fq=swhidId_s:["" TO *]`;
73+
}
74+
6575
const res = await fetch(url).catch(err => {
6676
console.error(err);
6777
throw new HAL.API.FetchError(undefined);
6878
});
6979

7080
if (res.status === 429) {
7181
await new Promise(resolve => setTimeout(resolve, 100));
72-
return fetchHalSoftwares();
82+
return fetchHalSoftwares(params);
7383
}
7484

7585
if (res.status === 404) {

‎api/src/connectors/HAL/types/HAL.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ export namespace HAL {
6767
Object.setPrototypeOf(this, new.target.prototype);
6868
}
6969
}
70-
70+
7171
export type Software = {
7272
// the following fields are the ones that we use
7373
docid: string;
@@ -92,9 +92,9 @@ export namespace HAL {
9292
softProgrammingLanguage_s: string[];
9393
softVersion_s: string[];
9494
licence_s: string[];
95-
95+
9696
// The following is the complete list of fields that could be returned by the HAL API
97-
97+
9898
// label_s: string;
9999
// citationRef_s: string;
100100
// citationFull_s: string;
@@ -219,7 +219,7 @@ export namespace HAL {
219219
// softDevelopmentStatus_s: string[];
220220
// softRuntimePlatform_s: string[];
221221
};
222-
222+
223223
export type Domain = {
224224
docid: number;
225225
haveNext_bool: boolean;
@@ -231,7 +231,7 @@ export namespace HAL {
231231
dateLastIndexed_tdate: string; // ISO date
232232
level_i: number;
233233
};
234-
234+
235235
export type Structure = {
236236
acronym_s: string[];
237237
acronym_sci: string[];
@@ -272,7 +272,7 @@ export namespace HAL {
272272
url_s: string;
273273
valid_s: string;
274274
};
275-
275+
276276
export type Author = {
277277
accountAssociated_bool: boolean;
278278
affPref_i: number;

‎api/src/core/adapters/hal/getHalSoftwareOptions.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export const rawHalSoftwareToExternalOption =
3333
// HAL documentation is here : https://api.archives-ouvertes.fr/docs/search
3434

3535
export const getHalSoftwareOptions: GetSoftwareExternalDataOptions = async ({ queryString, language }) => {
36-
const rawHalSoftwares = await halAPIGateway.software.getAll(queryString);
36+
const rawHalSoftwares = await halAPIGateway.software.getAll({ queryString });
3737

3838
return rawHalSoftwares.map(rawHalSoftwareToExternalOption(language));
3939
};

‎api/src/core/adapters/hal/getSoftwareForm.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import memoize from "memoizee";
12
import { SoftwareFormData, SoftwareType } from "../../usecases/readWriteSillData";
23
import { halAPIGateway } from "../../../connectors/HAL";
34
import { HAL } from "../../../connectors/HAL/types/HAL";
@@ -52,3 +53,17 @@ export const halRawSoftwareToSoftwareForm = async (halSoftware: HAL.API.Software
5253

5354
return formData;
5455
};
56+
57+
export const getHalSoftwareForm = memoize(async (halDocId: string): Promise<SoftwareFormData | undefined> => {
58+
const halRawSoftware = await halAPIGateway.software.getById(halDocId).catch(error => {
59+
if (!(error instanceof HAL.API.FetchError)) throw error;
60+
if (error.status === 404 || error.status === undefined) return;
61+
throw error;
62+
});
63+
64+
if (!halRawSoftware) {
65+
throw Error();
66+
}
67+
68+
return halRawSoftwareToSoftwareForm(halRawSoftware);
69+
});

‎api/src/core/adapters/hal/index.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
import { getHalSoftwareOptions } from "./getHalSoftwareOptions";
22
import { getHalSoftwareExternalData } from "./getHalSoftwareExternalData";
3+
import { getHalSoftwareForm } from "./getSoftwareForm";
4+
import { Adapter } from "../type";
35

4-
export const halAdapter = {
6+
export const halAdapter: Adapter = {
7+
sourceType: "HAL",
58
softwareExternalData: {
6-
getByHalId: getHalSoftwareExternalData
9+
getById: getHalSoftwareExternalData
710
},
811
softwareOptions: {
9-
getByHalId: getHalSoftwareOptions
12+
getById: getHalSoftwareOptions
13+
},
14+
softwareForm: {
15+
getById: getHalSoftwareForm
1016
}
1117
};

‎api/src/core/adapters/type.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { ExternalDataOrigin, GetSoftwareExternalData } from "../ports/GetSoftwareExternalData";
2+
import { GetSoftwareExternalDataOptions } from "../ports/GetSoftwareExternalDataOptions";
3+
import { SoftwareFormData } from "../usecases/readWriteSillData";
4+
5+
export type Adapter = {
6+
sourceType: ExternalDataOrigin;
7+
softwareExternalData: {
8+
getById: GetSoftwareExternalData;
9+
};
10+
softwareOptions: {
11+
getById: GetSoftwareExternalDataOptions;
12+
};
13+
softwareForm: {
14+
getById: (externalId: string) => Promise<SoftwareFormData | undefined>;
15+
};
16+
};

‎api/src/core/adapters/wikidata/index.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { Adapter } from "../type";
2+
import { getWikidataForm } from "./getSoftwareForm";
3+
import { getWikidataSoftware } from "./getWikidataSoftware";
4+
import { getWikidataSoftwareOptions } from "./getWikidataSoftwareOptions";
5+
6+
export const wikidataAdapter: Adapter = {
7+
sourceType: "wikidata",
8+
softwareExternalData: {
9+
getById: getWikidataSoftware
10+
},
11+
softwareOptions: {
12+
getById: getWikidataSoftwareOptions
13+
},
14+
softwareForm: {
15+
getById: getWikidataForm
16+
}
17+
};

‎api/src/core/bootstrap.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import { Database } from "./adapters/dbApi/kysely/kysely.database";
55
import { makeFetchAndSaveExternalDataForAllSoftwares } from "./adapters/fetchExternalData";
66
import { getCnllPrestatairesSill } from "./adapters/getCnllPrestatairesSill";
77
import { getServiceProviders } from "./adapters/getServiceProviders";
8-
import { getWikidataSoftware } from "./adapters/wikidata/getWikidataSoftware";
9-
import { getWikidataSoftwareOptions } from "./adapters/wikidata/getWikidataSoftwareOptions";
8+
import { wikidataAdapter } from "./adapters/wikidata";
109
import { halAdapter } from "./adapters/hal";
1110
import type { ComptoirDuLibreApi } from "./ports/ComptoirDuLibreApi";
1211
import { DbApiV2 } from "./ports/DbApiV2";
@@ -83,13 +82,13 @@ function getSoftwareExternalDataFunctions(externalSoftwareDataOrigin: ExternalDa
8382
switch (externalSoftwareDataOrigin) {
8483
case "wikidata":
8584
return {
86-
"getSoftwareExternalDataOptions": getWikidataSoftwareOptions,
87-
"getSoftwareExternalData": getWikidataSoftware
85+
"getSoftwareExternalDataOptions": wikidataAdapter.softwareOptions.getById,
86+
"getSoftwareExternalData": wikidataAdapter.softwareExternalData.getById
8887
};
8988
case "HAL":
9089
return {
91-
"getSoftwareExternalDataOptions": halAdapter.softwareOptions.getByHalId,
92-
"getSoftwareExternalData": halAdapter.softwareExternalData.getByHalId
90+
"getSoftwareExternalDataOptions": halAdapter.softwareOptions.getById,
91+
"getSoftwareExternalData": halAdapter.softwareExternalData.getById
9392
};
9493
default:
9594
const unreachableCase: never = externalSoftwareDataOrigin;

‎api/src/core/updateTools.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ import { Database } from "./adapters/dbApi/kysely/kysely.database";
55
import { makeFetchAndSaveExternalDataForAllSoftwares } from "./adapters/fetchExternalData";
66
import { getCnllPrestatairesSill } from "./adapters/getCnllPrestatairesSill";
77
import { getServiceProviders } from "./adapters/getServiceProviders";
8-
import { getWikidataSoftware } from "./adapters/wikidata/getWikidataSoftware";
9-
import { getWikidataSoftwareOptions } from "./adapters/wikidata/getWikidataSoftwareOptions";
8+
import { wikidataAdapter } from "./adapters/wikidata";
109
import { halAdapter } from "./adapters/hal";
1110
import { DbApiV2 } from "./ports/DbApiV2";
1211
import type { ExternalDataOrigin, GetSoftwareExternalData } from "./ports/GetSoftwareExternalData";
@@ -61,13 +60,13 @@ function getSoftwareExternalDataFunctions(externalSoftwareDataOrigin: ExternalDa
6160
switch (externalSoftwareDataOrigin) {
6261
case "wikidata":
6362
return {
64-
"getSoftwareExternalDataOptions": getWikidataSoftwareOptions,
65-
"getSoftwareExternalData": getWikidataSoftware
63+
"getSoftwareExternalDataOptions": wikidataAdapter.softwareOptions.getById,
64+
"getSoftwareExternalData": wikidataAdapter.softwareExternalData.getById
6665
};
6766
case "HAL":
6867
return {
69-
"getSoftwareExternalDataOptions": halAdapter.softwareOptions.getByHalId,
70-
"getSoftwareExternalData": halAdapter.softwareExternalData.getByHalId
68+
"getSoftwareExternalDataOptions": halAdapter.softwareOptions.getById,
69+
"getSoftwareExternalData": halAdapter.softwareExternalData.getById
7170
};
7271
default:
7372
const unreachableCase: never = externalSoftwareDataOrigin;

‎api/src/core/usecases/importFromSource.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export const importFromHALSource: (dbApi: DbApiV2) => (agentEmail: string) => Pr
1717
about: "This is a bot user created to import data."
1818
});
1919

20-
const softwares = await halAPIGateway.software.getAll();
20+
const softwares = await halAPIGateway.software.getAll({ SWHFilter: true });
2121
const dbSoftwares = await dbApi.software.getAll();
2222
const dbSoftwaresNames = dbSoftwares.map(software => {
2323
return software.softwareName;

0 commit comments

Comments
 (0)