🌐 AI搜索 & 代理 主页
Skip to content

Commit 822362e

Browse files
timmywiltimmywil
authored
Tests: add --hard-retries option to test runner
- Add the ability to retry by restarting the worker and getting a different browser instance, after all normal retries have been exhausted. This can sometimes be successful when a refresh is not. Close gh-5438
1 parent ae67ace commit 822362e

File tree

6 files changed

+94
-38
lines changed

6 files changed

+94
-38
lines changed

‎.github/workflows/browserstack.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,4 @@ jobs:
6262
run: npm run pretest
6363

6464
- name: Run tests
65-
run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --run-id ${{ github.run_id }} --isolate --retries 3
65+
run: npm run test:unit -- -v --browserstack "${{ matrix.BROWSER }}" --run-id ${{ github.run_id }} --isolate --retries 3 --hard-retries 1

‎test/runner/browserstack/browsers.js‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,21 +66,33 @@ async function waitForAck( worker, { fullBrowser, verbose } ) {
6666
} );
6767
}
6868

69-
async function ensureAcknowledged( worker, restarts ) {
69+
async function restartWorker( worker ) {
70+
await cleanupWorker( worker, worker.options );
71+
await createBrowserWorker(
72+
worker.url,
73+
worker.browser,
74+
worker.options,
75+
worker.restarts + 1
76+
);
77+
}
78+
79+
export async function restartBrowser( browser ) {
80+
const fullBrowser = getBrowserString( browser );
81+
const worker = workers[ fullBrowser ];
82+
if ( worker ) {
83+
await restartWorker( worker );
84+
}
85+
}
86+
87+
async function ensureAcknowledged( worker ) {
7088
const fullBrowser = getBrowserString( worker.browser );
7189
const verbose = worker.options.verbose;
7290
try {
7391
await waitForAck( worker, { fullBrowser, verbose } );
7492
return worker;
7593
} catch ( error ) {
7694
console.error( error.message );
77-
await cleanupWorker( worker, { verbose } );
78-
await createBrowserWorker(
79-
worker.url,
80-
worker.browser,
81-
worker.options,
82-
restarts + 1
83-
);
95+
await restartWorker( worker.browser );
8496
}
8597
}
8698

@@ -132,7 +144,7 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
132144

133145
// Wait for the worker to show up in the list
134146
// before returning it.
135-
return ensureAcknowledged( worker, restarts );
147+
return ensureAcknowledged( worker );
136148
}
137149

138150
export async function setBrowserWorkerUrl( browser, url ) {
@@ -159,13 +171,7 @@ export async function checkLastTouches() {
159171
}min.`
160172
);
161173
}
162-
await cleanupWorker( worker, options );
163-
await createBrowserWorker(
164-
worker.url,
165-
worker.browser,
166-
options,
167-
worker.restarts
168-
);
174+
await restartWorker( worker );
169175
}
170176
}
171177
}

‎test/runner/browserstack/queue.js‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import chalk from "chalk";
22
import { getBrowserString } from "../lib/getBrowserString.js";
3-
import { checkLastTouches, createBrowserWorker, setBrowserWorkerUrl } from "./browsers.js";
3+
import {
4+
checkLastTouches,
5+
createBrowserWorker,
6+
restartBrowser,
7+
setBrowserWorkerUrl
8+
} from "./browsers.js";
49

510
const TEST_POLL_TIMEOUT = 1000;
611

@@ -32,6 +37,9 @@ export function getNextBrowserTest( reportId ) {
3237
}
3338

3439
export function retryTest( reportId, maxRetries ) {
40+
if ( !maxRetries ) {
41+
return;
42+
}
3543
const test = queue.find( ( test ) => test.id === reportId );
3644
if ( test ) {
3745
test.retries++;
@@ -46,10 +54,31 @@ export function retryTest( reportId, maxRetries ) {
4654
}
4755
}
4856

57+
export async function hardRetryTest( reportId, maxHardRetries ) {
58+
if ( !maxHardRetries ) {
59+
return false;
60+
}
61+
const test = queue.find( ( test ) => test.id === reportId );
62+
if ( test ) {
63+
test.hardRetries++;
64+
if ( test.hardRetries <= maxHardRetries ) {
65+
console.log(
66+
`Hard retrying test ${ reportId } for ${ chalk.yellow(
67+
test.options.modules.join( ", " )
68+
) }...${ test.hardRetries }`
69+
);
70+
await restartBrowser( test.browser );
71+
return true;
72+
}
73+
}
74+
return false;
75+
}
76+
4977
export function addBrowserStackRun( url, browser, options ) {
5078
queue.push( {
5179
browser,
5280
fullBrowser: getBrowserString( browser ),
81+
hardRetries: 0,
5382
id: options.reportId,
5483
url,
5584
options,
@@ -59,7 +88,7 @@ export function addBrowserStackRun( url, browser, options ) {
5988
}
6089

6190
export async function runAllBrowserStack() {
62-
return new Promise( async( resolve, reject )=> {
91+
return new Promise( async( resolve, reject ) => {
6392
while ( queue.length ) {
6493
try {
6594
await checkLastTouches();

‎test/runner/command.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,6 @@ const argv = yargs( process.argv.slice( 2 ) )
6464
type: "boolean",
6565
description: "Log additional information."
6666
} )
67-
.option( "retries", {
68-
alias: "r",
69-
type: "number",
70-
description: "Number of times to retry failed tests in BrowserStack.",
71-
implies: [ "browserstack" ]
72-
} )
7367
.option( "run-id", {
7468
type: "string",
7569
description: "A unique identifier for this run."
@@ -90,6 +84,20 @@ const argv = yargs( process.argv.slice( 2 ) )
9084
"Otherwise, the --browser option will be used, " +
9185
"with the latest version/device for that browser, on a matching OS."
9286
} )
87+
.option( "retries", {
88+
alias: "r",
89+
type: "number",
90+
description: "Number of times to retry failed tests in BrowserStack.",
91+
implies: [ "browserstack" ]
92+
} )
93+
.option( "hard-retries", {
94+
type: "number",
95+
description:
96+
"Number of times to retry failed tests in BrowserStack " +
97+
"by restarting the worker. This is in addition to the normal retries " +
98+
"and are only used when the normal retries are exhausted.",
99+
implies: [ "browserstack" ]
100+
} )
93101
.option( "list-browsers", {
94102
type: "string",
95103
description:

‎test/runner/createTestServer.js‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,29 @@ export async function createTestServer( report ) {
2626

2727
// Add a script tag to the index.html to load the QUnit listeners
2828
app.use( /\/test(?:\/index.html)?\//, ( _req, res ) => {
29-
res.send( indexHTML.replace(
30-
"</head>",
31-
"<script src=\"/test/runner/listeners.js\"></script></head>"
32-
) );
29+
res.send(
30+
indexHTML.replace(
31+
"</head>",
32+
"<script src=\"/test/runner/listeners.js\"></script></head>"
33+
)
34+
);
3335
} );
3436

3537
// Bind the reporter
36-
app.post( "/api/report", bodyParser.json( { limit: "50mb" } ), ( req, res ) => {
37-
if ( report ) {
38-
const response = report( req.body );
39-
if ( response ) {
40-
res.json( response );
41-
return;
38+
app.post(
39+
"/api/report",
40+
bodyParser.json( { limit: "50mb" } ),
41+
async( req, res ) => {
42+
if ( report ) {
43+
const response = await report( req.body );
44+
if ( response ) {
45+
res.json( response );
46+
return;
47+
}
4248
}
49+
res.sendStatus( 204 );
4350
}
44-
res.sendStatus( 204 );
45-
} );
51+
);
4652

4753
// Handle errors from the body parser
4854
app.use( bodyParserErrorHandler() );

‎test/runner/run.js‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js";
1414
import {
1515
addBrowserStackRun,
1616
getNextBrowserTest,
17+
hardRetryTest,
1718
retryTest,
1819
runAllBrowserStack
1920
} from "./browserstack/queue.js";
@@ -30,6 +31,7 @@ export async function run( {
3031
concurrency,
3132
debug,
3233
esm,
34+
hardRetries,
3335
headless,
3436
isolate,
3537
modules = [],
@@ -72,7 +74,7 @@ export async function run( {
7274
// Create the test app and
7375
// hook it up to the reporter
7476
const reports = Object.create( null );
75-
const app = await createTestServer( ( message ) => {
77+
const app = await createTestServer( async( message ) => {
7678
switch ( message.type ) {
7779
case "testEnd": {
7880
const reportId = message.id;
@@ -120,6 +122,11 @@ export async function run( {
120122
if ( retry ) {
121123
return retry;
122124
}
125+
126+
// Return early if hardRetryTest returns true
127+
if ( await hardRetryTest( reportId, hardRetries ) ) {
128+
return;
129+
}
123130
errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
124131
}
125132

0 commit comments

Comments
 (0)