From 7f8f4e715150ea804004cf840df8299068a17515 Mon Sep 17 00:00:00 2001 From: Hamed Mohamed Date: Thu, 29 Jan 2026 22:12:49 +0300 Subject: [PATCH 1/5] feat: add stats tracking on pacote.extract through extractAndResolve --- .changeset/stats-tracking-pacote-extract.md | 8 +++ .../scanner/src/class/DateProvider.class.ts | 9 +++ .../scanner/src/class/StatsCollector.class.ts | 62 +++++++++++++++++++ workspaces/scanner/src/depWalker.ts | 30 +++++++-- workspaces/scanner/src/types.ts | 16 +++-- workspaces/tarball/src/tarball.ts | 11 +++- 6 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 .changeset/stats-tracking-pacote-extract.md create mode 100644 workspaces/scanner/src/class/DateProvider.class.ts create mode 100644 workspaces/scanner/src/class/StatsCollector.class.ts diff --git a/.changeset/stats-tracking-pacote-extract.md b/.changeset/stats-tracking-pacote-extract.md new file mode 100644 index 00000000..d5209f64 --- /dev/null +++ b/.changeset/stats-tracking-pacote-extract.md @@ -0,0 +1,8 @@ +--- +"@nodesecure/tarball": minor +"@nodesecure/scanner": minor +--- + +feat: add stats tracking on pacote.extract through extractAndResolve + +Add support for dependency injection of extractFn in extractAndResolve to enable tracking of pacote.extract calls using StatsCollector. This allows measuring extraction time for each package during scanning. diff --git a/workspaces/scanner/src/class/DateProvider.class.ts b/workspaces/scanner/src/class/DateProvider.class.ts new file mode 100644 index 00000000..94ed7278 --- /dev/null +++ b/workspaces/scanner/src/class/DateProvider.class.ts @@ -0,0 +1,9 @@ +export interface DateProvider { + now(): number; +} + +export class SystemDateProvider implements DateProvider { + now(): number { + return Date.now(); + } +} diff --git a/workspaces/scanner/src/class/StatsCollector.class.ts b/workspaces/scanner/src/class/StatsCollector.class.ts new file mode 100644 index 00000000..cfa8a2c7 --- /dev/null +++ b/workspaces/scanner/src/class/StatsCollector.class.ts @@ -0,0 +1,62 @@ +// Import Internal Dependencies +import { SystemDateProvider, type DateProvider } from "./DateProvider.class.ts"; + +export interface ApiStats { + name: string; + startedAt: number; + executionTime: number; +} + +export interface Stats { + startedAt: number; + executionTime: number; + apiCalls: ApiStats[]; + apiCallsCount: number; +} + +export class StatsCollector { + #apiCalls: ApiStats[] = []; + #dateProvider: DateProvider; + #startedAt: number; + + constructor(dateProvider: DateProvider = new SystemDateProvider()) { + this.#dateProvider = dateProvider; + this.#startedAt = this.#dateProvider.now(); + } + + track any>(name: string, fn: T): ReturnType { + const startedAt = this.#dateProvider.now(); + try { + const result = fn(); + if (result instanceof Promise) { + return result.finally(() => this.#addApiStat(name, startedAt) + ) as ReturnType; + } + + this.#addApiStat(name, startedAt); + + return result; + } + catch (err) { + this.#addApiStat(name, startedAt); + throw err; + } + } + + #addApiStat(name: string, startedAt: number) { + this.#apiCalls.push({ + name, + startedAt, + executionTime: this.#dateProvider.now() - startedAt + }); + } + + getStats(): Stats { + return { + startedAt: this.#startedAt, + executionTime: this.#dateProvider.now() - this.#startedAt, + apiCalls: this.#apiCalls, + apiCallsCount: this.#apiCalls.length + }; + } +} diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index d32ace95..3db2a85e 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -8,6 +8,7 @@ import { extractAndResolve, scanDirOrArchive } from "@nodesecure/tarball"; +import pacote from "pacote"; import * as Vulnera from "@nodesecure/vulnera"; import { npm } from "@nodesecure/tree-walker"; import { parseAuthor } from "@nodesecure/utils"; @@ -30,6 +31,7 @@ import { NpmRegistryProvider } from "./registry/NpmRegistryProvider.ts"; import { RegistryTokenStore } from "./registry/RegistryTokenStore.ts"; import { TempDirectory } from "./class/TempDirectory.class.ts"; import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts"; +import { StatsCollector } from "./class/StatsCollector.class.ts"; import type { Dependency, DependencyVersion, @@ -116,6 +118,7 @@ export async function depWalker( const startedAt = Date.now(); const isRemoteScanning = typeof location === "undefined"; const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token); + const statsCollector = new StatsCollector(); await using tempDir = await TempDirectory.create(); @@ -239,11 +242,23 @@ export async function depWalker( } } + async function trackedExtract( + spec: string, + dest: string, + opts: pacote.Options + ): Promise { + await statsCollector.track( + `pacote.extract[${spec}]`, + () => pacote.extract(spec, dest, opts) + ); + } + const scanDirOptions = { ref: dependency.versions[version] as any, location, isRootNode: scanRootNode && name === manifest.name, - registry + registry, + extractFn: trackedExtract }; operationsQueue.push( scanDirOrArchiveEx(name, version, locker, tempDir, logger, scanDirOptions) @@ -350,7 +365,11 @@ export async function depWalker( packages: [...highlightedPackages] }; payload.dependencies = Object.fromEntries(dependencies); - payload.metadata.executionTime = Date.now() - startedAt; + payload.metadata = { + ...payload.metadata, + executionTime: Date.now() - startedAt, + stats: statsCollector.getStats() + }; return payload as Payload; } @@ -371,6 +390,7 @@ async function scanDirOrArchiveEx( isRootNode: boolean; location: string | undefined; ref: any; + extractFn?: (spec: string, dest: string, opts: pacote.Options) => Promise; } ) { using _ = await locker.acquire(); @@ -380,14 +400,16 @@ async function scanDirOrArchiveEx( registry, location = process.cwd(), isRootNode, - ref + ref, + extractFn } = options; const mama = await (isRootNode ? ManifestManager.fromPackageJSON(location) : extractAndResolve(tempDir.location, { spec: `${name}@${version}`, - registry + registry, + extractFn }) ); diff --git a/workspaces/scanner/src/types.ts b/workspaces/scanner/src/types.ts index 86b1018e..81acbfa6 100644 --- a/workspaces/scanner/src/types.ts +++ b/workspaces/scanner/src/types.ts @@ -208,14 +208,18 @@ export interface Payload { vulnerabilityStrategy: Vulnera.Kind; metadata: { - /** - * UNIX Timestamp when the scan started - */ startedAt: number; - /** - * Execution time in milliseconds - */ executionTime: number; + stats?: { + startedAt: number; + executionTime: number; + apiCalls: Array<{ + name: string; + startedAt: number; + executionTime: number; + }>; + apiCallsCount: number; + }; }; } diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts index be2eb3fe..973399d7 100644 --- a/workspaces/tarball/src/tarball.ts +++ b/workspaces/tarball/src/tarball.ts @@ -200,19 +200,26 @@ export async function scanPackage( }; } +type ExtractFunction = ( + spec: string, + destination: string, + options: pacote.Options +) => Promise; + export interface TarballResolutionOptions { spec: string; registry?: string; + extractFn?: ExtractFunction; } export async function extractAndResolve( location: string, options: TarballResolutionOptions ): Promise { - const { spec, registry } = options; + const { spec, registry, extractFn = pacote.extract } = options; const tarballLocation = path.join(location, spec.replaceAll("/", "_")); - await pacote.extract( + await extractFn( spec, tarballLocation, { From ef5a08747842ba6ea5ea667f18525f3a95054ace Mon Sep 17 00:00:00 2001 From: Hamed Mohamed Date: Fri, 30 Jan 2026 10:59:32 +0300 Subject: [PATCH 2/5] refactor: rename extractFn to pacoteProvider and use interface --- workspaces/scanner/src/depWalker.ts | 26 +++++++++++--------------- workspaces/tarball/src/tarball.ts | 18 ++++++++++-------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index bba533a7..10eaa870 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -260,24 +260,20 @@ export async function depWalker( } } - async function trackedExtract( - spec: string, - dest: string, - opts: pacote.Options - ): Promise { - await statsCollector.track( - `pacote.extract[${spec}]`, - () => pacote.extract(spec, dest, opts) - ); - } - const scanDirOptions = { ref: dependency.versions[version] as any, location, isRootNode: scanRootNode && name === manifest.name, registry, statsCollector, - extractFn: trackedExtract + pacoteProvider: { + extract: async(spec: string, dest: string, opts: pacote.Options) => { + await statsCollector.track( + `pacote.extract[${spec}]`, + () => pacote.extract(spec, dest, opts) + ); + } + } }; operationsQueue.push( scanDirOrArchiveEx(name, version, locker, tempDir, logger, scanDirOptions) @@ -406,7 +402,7 @@ async function scanDirOrArchiveEx( location: string | undefined; ref: any; statsCollector: StatsCollector; - extractFn?: (spec: string, dest: string, opts: pacote.Options) => Promise; + pacoteProvider?: import("@nodesecure/tarball").PacoteProvider; } ) { using _ = await locker.acquire(); @@ -420,7 +416,7 @@ async function scanDirOrArchiveEx( isRootNode, ref, statsCollector, - extractFn + pacoteProvider } = options; const mama = await (isRootNode ? @@ -428,7 +424,7 @@ async function scanDirOrArchiveEx( extractAndResolve(tempDir.location, { spec, registry, - extractFn + pacoteProvider }) ); diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts index 973399d7..0d8fad37 100644 --- a/workspaces/tarball/src/tarball.ts +++ b/workspaces/tarball/src/tarball.ts @@ -200,26 +200,28 @@ export async function scanPackage( }; } -type ExtractFunction = ( - spec: string, - destination: string, - options: pacote.Options -) => Promise; +export interface PacoteProvider { + extract( + spec: string, + destination: string, + options: pacote.Options + ): Promise; +} export interface TarballResolutionOptions { spec: string; registry?: string; - extractFn?: ExtractFunction; + pacoteProvider?: PacoteProvider; } export async function extractAndResolve( location: string, options: TarballResolutionOptions ): Promise { - const { spec, registry, extractFn = pacote.extract } = options; + const { spec, registry, pacoteProvider = pacote } = options; const tarballLocation = path.join(location, spec.replaceAll("/", "_")); - await extractFn( + await pacoteProvider.extract( spec, tarballLocation, { From 3483f283cfd98c6aad57eab9078f6af9b520b214 Mon Sep 17 00:00:00 2001 From: Hamed Mohamed Date: Fri, 30 Jan 2026 15:57:40 +0300 Subject: [PATCH 3/5] refactor: reuse pacoteProvider and update naming convention --- workspaces/scanner/src/depWalker.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index 10eaa870..e8c744db 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -8,7 +8,8 @@ import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk"; import { Mutex, MutexRelease } from "@openally/mutex"; import { extractAndResolve, - scanDirOrArchive + scanDirOrArchive, + type PacoteProvider } from "@nodesecure/tarball"; import * as Vulnera from "@nodesecure/vulnera"; import { npm } from "@nodesecure/tree-walker"; @@ -116,6 +117,16 @@ export async function depWalker( } = options; const statsCollector = new StatsCollector(); + + const pacoteProvider: PacoteProvider = { + extract: async(spec, dest, opts) => { + await statsCollector.track( + `pacote.extract ${spec}`, + () => pacote.extract(spec, dest, opts) + ); + } + }; + const isRemoteScanning = typeof location === "undefined"; const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token); @@ -266,14 +277,7 @@ export async function depWalker( isRootNode: scanRootNode && name === manifest.name, registry, statsCollector, - pacoteProvider: { - extract: async(spec: string, dest: string, opts: pacote.Options) => { - await statsCollector.track( - `pacote.extract[${spec}]`, - () => pacote.extract(spec, dest, opts) - ); - } - } + pacoteProvider }; operationsQueue.push( scanDirOrArchiveEx(name, version, locker, tempDir, logger, scanDirOptions) From fe4ec50783aac3cf74c322418ea366c4302292d4 Mon Sep 17 00:00:00 2001 From: Hamed Mohamed Date: Fri, 30 Jan 2026 16:04:02 +0300 Subject: [PATCH 4/5] refactor: use top-level imported PacoteProvider type --- workspaces/scanner/src/depWalker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index e8c744db..ae42ba7a 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -406,7 +406,7 @@ async function scanDirOrArchiveEx( location: string | undefined; ref: any; statsCollector: StatsCollector; - pacoteProvider?: import("@nodesecure/tarball").PacoteProvider; + pacoteProvider?: PacoteProvider; } ) { using _ = await locker.acquire(); From e985c9fdc37811a1c569346625e38a9e3ecda791 Mon Sep 17 00:00:00 2001 From: Hamed Mohamed Date: Fri, 30 Jan 2026 16:23:24 +0300 Subject: [PATCH 5/5] refactor: remove async/await from pacoteProvider as requested --- workspaces/scanner/src/depWalker.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index ae42ba7a..511aaba5 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -119,12 +119,10 @@ export async function depWalker( const statsCollector = new StatsCollector(); const pacoteProvider: PacoteProvider = { - extract: async(spec, dest, opts) => { - await statsCollector.track( - `pacote.extract ${spec}`, - () => pacote.extract(spec, dest, opts) - ); - } + extract: (spec, dest, opts) => statsCollector.track( + `pacote.extract ${spec}`, + () => pacote.extract(spec, dest, opts) + ).then(() => {}) }; const isRemoteScanning = typeof location === "undefined";