From dd4fab895563a7d738905dba4619aae428191cdd Mon Sep 17 00:00:00 2001 From: xuga Date: Wed, 3 Dec 2025 10:52:48 -0800 Subject: [PATCH 1/2] Track strings to calculate codeSurvivalRate --- .../node/codeMapper/codeMapperService.ts | 24 ++++++++++++++++--- .../tools/node/abstractReplaceStringTool.tsx | 19 ++++++++++++--- src/extension/tools/node/applyPatchTool.tsx | 19 ++++++++++++--- .../common/editSurvivalReporter.ts | 11 ++++++++- .../common/editSurvivalTracker.ts | 13 +++++++++- 5 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/extension/prompts/node/codeMapper/codeMapperService.ts b/src/extension/prompts/node/codeMapper/codeMapperService.ts index e66eaa1dce..4732417f0e 100644 --- a/src/extension/prompts/node/codeMapper/codeMapperService.ts +++ b/src/extension/prompts/node/codeMapper/codeMapperService.ts @@ -305,10 +305,22 @@ function reportEditSurvivalEvent(res: EditSurvivalResult, { requestId, speculati "survivalRateFourGram": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The rate between 0 and 1 of how much of the AI edit is still present in the document." }, "survivalRateNoRevert": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The rate between 0 and 1 of how much of the ranges the AI touched ended up being reverted." }, "didBranchChange": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Indicates if the branch changed in the meantime. If the branch changed (value is 1), this event should probably be ignored." }, - "timeDelayMs": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The time delay between the user accepting the edit and measuring the survival rate." } + "timeDelayMs": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The time delay between the user accepting the edit and measuring the survival rate." }, + "textBeforeAiEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings before AI edits were applied for each edit region." }, + "textAfterAiEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings after AI edits were applied for each edit region." }, + "textAfterUserEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings after user modifications for each edit region." } } */ - res.telemetryService.sendMSFTTelemetryEvent('codeMapper.trackEditSurvival', { requestId, speculationRequestId, requestSource, chatRequestModel, mapper }, { + res.telemetryService.sendMSFTTelemetryEvent('codeMapper.trackEditSurvival', { + requestId, + speculationRequestId, + requestSource, + chatRequestModel, + mapper, + textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined, + textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined, + textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined, + }, { survivalRateFourGram: res.fourGram, survivalRateNoRevert: res.noRevert, timeDelayMs: res.timeDelayMs, @@ -321,6 +333,9 @@ function reportEditSurvivalEvent(res: EditSurvivalResult, { requestId, speculati chatRequestModel, mapper, currentFileContent: res.currentFileContent, + textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined, + textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined, + textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined, }, { survivalRateFourGram: res.fourGram, survivalRateNoRevert: res.noRevert, @@ -332,7 +347,10 @@ function reportEditSurvivalEvent(res: EditSurvivalResult, { requestId, speculati headerRequestId: speculationRequestId, completionTextJson: res.currentFileContent, chatRequestModel, - requestSource + requestSource, + textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined, + textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined, + textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined, }, { timeDelayMs: res.timeDelayMs, survivalRateFourGram: res.fourGram, diff --git a/src/extension/tools/node/abstractReplaceStringTool.tsx b/src/extension/tools/node/abstractReplaceStringTool.tsx index 79e36ed853..68b7cbcfc8 100644 --- a/src/extension/tools/node/abstractReplaceStringTool.tsx +++ b/src/extension/tools/node/abstractReplaceStringTool.tsx @@ -307,10 +307,20 @@ export abstract class AbstractReplaceStringTool { "survivalRateFourGram": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The rate between 0 and 1 of how much of the AI edit is still present in the document." }, "survivalRateNoRevert": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The rate between 0 and 1 of how much of the ranges the AI touched ended up being reverted." }, "didBranchChange": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Indicates if the branch changed in the meantime. If the branch changed (value is 1), this event should probably be ignored." }, - "timeDelayMs": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The time delay between the user accepting the edit and measuring the survival rate." } + "timeDelayMs": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The time delay between the user accepting the edit and measuring the survival rate.", + "textBeforeAiEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings before AI edits were applied for each edit region." }, + "textAfterAiEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings after AI edits were applied for each edit region." }, + "textAfterUserEdits": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Array of text strings after user modifications for each edit region." }} } */ - res.telemetryService.sendMSFTTelemetryEvent('applyPatch.trackEditSurvival', { requestId: this._promptContext?.requestId, requestSource: 'agent', mapper: 'applyPatchTool' }, { + res.telemetryService.sendMSFTTelemetryEvent('applyPatch.trackEditSurvival', { + requestId: this._promptContext?.requestId, + requestSource: 'agent', + mapper: 'applyPatchTool', + textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined, + textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined, + textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined, + }, { survivalRateFourGram: res.fourGram, survivalRateNoRevert: res.noRevert, timeDelayMs: res.timeDelayMs, @@ -402,7 +412,10 @@ export class ApplyPatchTool implements ICopilotTool { res.telemetryService.sendGHTelemetryEvent('applyPatch/trackEditSurvival', { headerRequestId: this._promptContext?.requestId, requestSource: 'agent', - mapper: 'applyPatchTool' + mapper: 'applyPatchTool', + textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined, + textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined, + textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined, }, { survivalRateFourGram: res.fourGram, survivalRateNoRevert: res.noRevert, diff --git a/src/platform/editSurvivalTracking/common/editSurvivalReporter.ts b/src/platform/editSurvivalTracking/common/editSurvivalReporter.ts index d4e970e482..a9af5cbb8c 100644 --- a/src/platform/editSurvivalTracking/common/editSurvivalReporter.ts +++ b/src/platform/editSurvivalTracking/common/editSurvivalReporter.ts @@ -27,6 +27,13 @@ export interface EditSurvivalResult { * See ArcTracker. */ readonly arc?: number; + + /** + * Text states for each edit region + */ + readonly textBeforeAiEdits?: string[]; + readonly textAfterAiEdits?: string[]; + readonly textAfterUserEdits?: string[]; } export class EditSurvivalReporter { @@ -104,7 +111,6 @@ export class EditSurvivalReporter { const currentBranch = this._getCurrentBranchName(); const didBranchChange = currentBranch !== this._initialBranchName; - this._sendTelemetryEvent({ telemetryService: this._telemetryService, fourGram: survivalRate.fourGram, @@ -113,6 +119,9 @@ export class EditSurvivalReporter { didBranchChange, currentFileContent: this._document.getText(), arc: this._arcTracker?.getAcceptedRestrainedCharactersCount(), + textBeforeAiEdits: survivalRate.textBeforeAiEdits, + textAfterAiEdits: survivalRate.textAfterAiEdits, + textAfterUserEdits: survivalRate.textAfterUserEdits, }); } diff --git a/src/platform/editSurvivalTracking/common/editSurvivalTracker.ts b/src/platform/editSurvivalTracking/common/editSurvivalTracker.ts index d3d2ed1ded..38a88a17c4 100644 --- a/src/platform/editSurvivalTracking/common/editSurvivalTracker.ts +++ b/src/platform/editSurvivalTracking/common/editSurvivalTracker.ts @@ -36,13 +36,17 @@ export class EditSurvivalTracker { * fourGram: Number between 0 (no edits survived) and 1 (all edits survived). * noRevert: Number between 0 (the text after user edits equals the text before the AI edits) and 1 (the text after user edits does not revert any text to the initial state) */ - computeTrackedEditsSurvivalScore(): { fourGram: number; noRevert: number } { + computeTrackedEditsSurvivalScore(): { fourGram: number; noRevert: number; textBeforeAiEdits: string[]; textAfterAiEdits: string[]; textAfterUserEdits: string[] } { let similarityScoreSumFourGram = 0; let similarityScoreSumMax = 0; let noRevertSum = 0; let noRevertSumMax = 0; + const allTextBefore: string[] = []; + const allTextAfter: string[] = []; + const allTextCurrent: string[] = []; + const ranges = this._originalEdits.getNewRanges(); const updatedRanges = applyEditsToRanges(ranges, this._combinedEditsSinceStart); @@ -54,6 +58,10 @@ export class EditSurvivalTracker { const newRange = updatedRanges[i]; const textAfterUserEdits = this._text.substring(newRange.start, newRange.endExclusive); + allTextBefore.push(textBeforeAiEdits); + allTextAfter.push(textAfterAiEdits); + allTextCurrent.push(textAfterUserEdits); + const similarity = compute4GramTextSimilarity(textAfterUserEdits, textAfterAiEdits); const aiEditSimilarity = compute4GramTextSimilarity(textAfterAiEdits, textBeforeAiEdits); @@ -75,6 +83,9 @@ export class EditSurvivalTracker { return { fourGram: similarityScoreSumMax === 0 ? 1 : (similarityScoreSumFourGram / similarityScoreSumMax), noRevert: noRevertSumMax === 0 ? 1 : (noRevertSum / noRevertSumMax), + textBeforeAiEdits: allTextBefore, + textAfterAiEdits: allTextAfter, + textAfterUserEdits: allTextCurrent, }; } } From 2fde3c101001074e8c7c6416978add96bed59632 Mon Sep 17 00:00:00 2001 From: meigaoms Date: Wed, 3 Dec 2025 13:53:19 -0800 Subject: [PATCH 2/2] Update test for the new telemetry adding --- src/extension/test/node/utils.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/extension/test/node/utils.spec.ts b/src/extension/test/node/utils.spec.ts index 16f1136d7d..14a0e005e2 100644 --- a/src/extension/test/node/utils.spec.ts +++ b/src/extension/test/node/utils.spec.ts @@ -294,6 +294,15 @@ suite('EditSurvivalTracker', () => { { "fourGram": 0.75, "noRevert": 1, + textAfterAiEdits: [ + '\r\n\r\n\t\tif (!args) {\r\n\t\t\tthrow new Error("invalid json document!");\r\n\t\t}' + ], + textAfterUserEdits: [ + '\r\n\r\n\t\tif (!args) {\r\n\t\t\tthrow new Error("");\r\n\t\t}' + ], + textBeforeAiEdits: [ + '\r\n' + ] } ); });