Skip to content

Commit ccf66de

Browse files
tagboolaJamie Rothfederjrothfedergemini-code-assist[bot]
authored
Add app testing agent tools and prompts (#9409)
* creation of feature branch * New MCP tool for running mobile tests (via app distribution). (#9250) * Scaffolding for new appdistribution MCP tool. * Refactor business logic out of the appdistribution CLI so that it can be used by an MCP tool. * Wire new appdistribution tool up to the business logic. * Fix linting errors. * Update src/appdistribution/distribution.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --------- Co-authored-by: Jamie Rothfeder <rothbutter@google.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Rename appdistribution directory to apptesting (#9268) * Rename appdistribution directory to apptesting * Make variables consistent with directory rename. --------- Co-authored-by: Jamie Rothfeder <rothbutter@google.com> * Use a datastructure to represent test devices rather than a string. (#9280) * Use a datastructure to represent test devices rather than a string. * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Pretty --------- Co-authored-by: Jamie Rothfeder <rothbutter@google.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Add run_test prompt (#9292) * Add initial MCP prompt for running automated tests * Fix typos * MCP tool `apptesting_run_test` can create and run a on-off test. (#9321) * Create a on-off test and execute. * Can now create a on-off test. * Update src/appdistribution/client.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * PR feedback * Separate test check in to a different tool so that gemini can orchestrate running and checking for completion. * Set the devices field to optional --------- Co-authored-by: Jamie Rothfeder <rothbutter@google.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Use the same default device that's used in the Console (#9320) * Update prompt to support generating a test case when there is no test description (#9322) * Use the same default device that's used in the Console * Update prompt to support generating a test case when there is no test description passed * Add custom auto-enablement for app testing (#9373) * Add custom auto-enablement for app testing * Address gemini code assist comments * Fix intersection bug * Fix issues with test * Add get devices tool (#9387) * Display link to results in the Firebase Console (#9406) * Place app testing tools behind an experiment * Address GCA comments * Explicitly set default devices * Address PR comments * Fix the status URL. (#9438) Co-authored-by: Jamie Rothfeder <rothbutter@google.com> --------- Co-authored-by: Jamie Rothfeder <rothbutter@google.com> Co-authored-by: Jamie Rothfeder <jamie.rothfeder@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent 9634532 commit ccf66de

File tree

17 files changed

+675
-197
lines changed

17 files changed

+675
-197
lines changed

src/appdistribution/client.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { appDistributionOrigin } from "../api";
99

1010
import {
1111
AabInfo,
12+
AIInstruction,
1213
BatchRemoveTestersResponse,
1314
BatchUpdateTestCasesRequest,
1415
BatchUpdateTestCasesResponse,
@@ -70,7 +71,7 @@ export class AppDistributionClient {
7071
});
7172
}
7273

73-
async updateReleaseNotes(releaseName: string, releaseNotes: string): Promise<void> {
74+
async updateReleaseNotes(releaseName: string, releaseNotes?: string): Promise<void> {
7475
if (!releaseNotes) {
7576
utils.logWarning("no release notes specified, skipping");
7677
return;
@@ -275,6 +276,7 @@ export class AppDistributionClient {
275276
async createReleaseTest(
276277
releaseName: string,
277278
devices: TestDevice[],
279+
aiInstruction?: AIInstruction,
278280
loginCredential?: LoginCredential,
279281
testCaseName?: string,
280282
): Promise<ReleaseTest> {
@@ -286,6 +288,7 @@ export class AppDistributionClient {
286288
deviceExecutions: devices.map((device) => ({ device })),
287289
loginCredential,
288290
testCase: testCaseName,
291+
aiInstructions: aiInstruction,
289292
},
290293
});
291294
return response.body;

src/appdistribution/distribution.ts

Lines changed: 122 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,75 @@
11
import * as fs from "fs-extra";
2-
import { FirebaseError, getErrMsg } from "../error";
32
import { logger } from "../logger";
43
import * as pathUtil from "path";
4+
import * as utils from "../utils";
5+
import { UploadReleaseResult, TestDevice, ReleaseTest } from "../appdistribution/types";
6+
import { AppDistributionClient } from "./client";
7+
import { FirebaseError, getErrMsg, getErrStatus } from "../error";
8+
9+
const TEST_MAX_POLLING_RETRIES = 40;
10+
const TEST_POLLING_INTERVAL_MILLIS = 30_000;
511

612
export enum DistributionFileType {
713
IPA = "ipa",
814
APK = "apk",
915
AAB = "aab",
1016
}
1117

18+
/** Upload a distribution */
19+
export async function upload(
20+
requests: AppDistributionClient,
21+
appName: string,
22+
distribution: Distribution,
23+
): Promise<string> {
24+
utils.logBullet("uploading binary...");
25+
try {
26+
const operationName = await requests.uploadRelease(appName, distribution);
27+
28+
// The upload process is asynchronous, so poll to figure out when the upload has finished successfully
29+
const uploadResponse = await requests.pollUploadStatus(operationName);
30+
31+
const release = uploadResponse.release;
32+
switch (uploadResponse.result) {
33+
case UploadReleaseResult.RELEASE_CREATED:
34+
utils.logSuccess(
35+
`uploaded new release ${release.displayVersion} (${release.buildVersion}) successfully!`,
36+
);
37+
break;
38+
case UploadReleaseResult.RELEASE_UPDATED:
39+
utils.logSuccess(
40+
`uploaded update to existing release ${release.displayVersion} (${release.buildVersion}) successfully!`,
41+
);
42+
break;
43+
case UploadReleaseResult.RELEASE_UNMODIFIED:
44+
utils.logSuccess(
45+
`re-uploaded already existing release ${release.displayVersion} (${release.buildVersion}) successfully!`,
46+
);
47+
break;
48+
default:
49+
utils.logSuccess(
50+
`uploaded release ${release.displayVersion} (${release.buildVersion}) successfully!`,
51+
);
52+
}
53+
utils.logSuccess(`View this release in the Firebase console: ${release.firebaseConsoleUri}`);
54+
utils.logSuccess(`Share this release with testers who have access: ${release.testingUri}`);
55+
utils.logSuccess(
56+
`Download the release binary (link expires in 1 hour): ${release.binaryDownloadUri}`,
57+
);
58+
return uploadResponse.release.name;
59+
} catch (err: unknown) {
60+
if (getErrStatus(err) === 404) {
61+
throw new FirebaseError(
62+
`App Distribution could not find your app ${appName}. ` +
63+
`Make sure to onboard your app by pressing the "Get started" ` +
64+
"button on the App Distribution page in the Firebase console: " +
65+
"https://console.firebase.google.com/project/_/appdistribution",
66+
{ exit: 1 },
67+
);
68+
}
69+
throw new FirebaseError(`Failed to upload release. ${getErrMsg(err)}`, { exit: 1 });
70+
}
71+
}
72+
1273
/**
1374
* Object representing an APK, AAB or IPA file. Used for uploading app distributions.
1475
*/
@@ -58,3 +119,63 @@ export class Distribution {
58119
return this.fileName;
59120
}
60121
}
122+
123+
/** Wait for release tests to complete */
124+
export async function awaitTestResults(
125+
releaseTests: ReleaseTest[],
126+
requests: AppDistributionClient,
127+
): Promise<void> {
128+
const releaseTestNames = new Set(
129+
releaseTests.map((rt) => rt.name).filter((n): n is string => !!n),
130+
);
131+
for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) {
132+
utils.logBullet(`${releaseTestNames.size} automated test results are pending...`);
133+
await delay(TEST_POLLING_INTERVAL_MILLIS);
134+
for (const releaseTestName of releaseTestNames) {
135+
const releaseTest = await requests.getReleaseTest(releaseTestName);
136+
if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) {
137+
releaseTestNames.delete(releaseTestName);
138+
if (releaseTestNames.size === 0) {
139+
utils.logSuccess("Automated test(s) passed!");
140+
return;
141+
} else {
142+
continue;
143+
}
144+
}
145+
for (const execution of releaseTest.deviceExecutions) {
146+
const device = deviceToString(execution.device);
147+
switch (execution.state) {
148+
case "PASSED":
149+
case "IN_PROGRESS":
150+
continue;
151+
case "FAILED":
152+
throw new FirebaseError(
153+
`Automated test failed for ${device}: ${execution.failedReason}`,
154+
{ exit: 1 },
155+
);
156+
case "INCONCLUSIVE":
157+
throw new FirebaseError(
158+
`Automated test inconclusive for ${device}: ${execution.inconclusiveReason}`,
159+
{ exit: 1 },
160+
);
161+
default:
162+
throw new FirebaseError(
163+
`Unsupported automated test state for ${device}: ${execution.state}`,
164+
{ exit: 1 },
165+
);
166+
}
167+
}
168+
}
169+
}
170+
throw new FirebaseError("It took longer than expected to run your test(s), please try again.", {
171+
exit: 1,
172+
});
173+
}
174+
175+
function delay(ms: number): Promise<number> {
176+
return new Promise((resolve) => setTimeout(resolve, ms));
177+
}
178+
179+
function deviceToString(device: TestDevice): string {
180+
return `${device.model} (${device.version}/${device.orientation}/${device.locale})`;
181+
}

src/appdistribution/options-parser-util.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { FieldHints, LoginCredential, TestDevice } from "./types";
88
* file and converts the input into an string[].
99
* Value takes precedent over file.
1010
*/
11-
export function parseIntoStringArray(value: string, file: string): string[] {
11+
export function parseIntoStringArray(value: string, file = ""): string[] {
1212
// If there is no value then the file gets parsed into a string to be split
1313
if (!value && file) {
1414
ensureFileExists(file);
@@ -61,7 +61,10 @@ export function getAppName(options: any): string {
6161
if (!options.app) {
6262
throw new FirebaseError("set the --app option to a valid Firebase app id and try again");
6363
}
64-
const appId = options.app;
64+
return toAppName(options.app);
65+
}
66+
67+
export function toAppName(appId: string) {
6568
return `projects/${appId.split(":")[1]}/apps/${appId}`;
6669
}
6770

@@ -70,7 +73,7 @@ export function getAppName(options: any): string {
7073
* and converts the input into a string[] of test device strings.
7174
* Value takes precedent over file.
7275
*/
73-
export function parseTestDevices(value: string, file: string): TestDevice[] {
76+
export function parseTestDevices(value: string, file = ""): TestDevice[] {
7477
// If there is no value then the file gets parsed into a string to be split
7578
if (!value && file) {
7679
ensureFileExists(file);

src/appdistribution/types.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,17 @@ export interface ReleaseTest {
116116
deviceExecutions: DeviceExecution[];
117117
loginCredential?: LoginCredential;
118118
testCase?: string;
119+
aiInstructions?: AIInstruction;
120+
}
121+
122+
export interface AIInstruction {
123+
steps: AIStep[];
124+
}
125+
126+
export interface AIStep {
127+
goal: string;
128+
hint?: string;
129+
successCriteria?: string;
119130
}
120131

121132
export interface AiStep {

0 commit comments

Comments
 (0)