Tests: fix flakey message logs; ignore delete worker failures

timmywil · web-flow · commit 02d23478289e · 2024-03-05T15:50:51.000-05:00
Close gh-5432
diff --git a/test/runner/browserstack/browsers.js b/test/runner/browserstack/browsers.js
@@ -187,13 +187,19 @@ export async function cleanupAllBrowsers( { verbose } ) {
 	const workersRemaining = Object.values( workers );
 	const numRemaining = workersRemaining.length;
 	if ( numRemaining ) {
-		await Promise.all(
-			workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
-		);
-		if ( verbose ) {
-			console.log(
-				`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+		try {
+			await Promise.all(
+				workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
 			);
+			if ( verbose ) {
+				console.log(
+					`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
+				);
+			}
+		} catch ( error ) {
+
+			// Log the error, but do not consider the test run failed
+			console.error( error );
 		}
 	}
 }
diff --git a/test/runner/browserstack/queue.js b/test/runner/browserstack/queue.js
@@ -39,7 +39,7 @@ export function retryTest( reportId, maxRetries ) {
 			console.log(
 				`Retrying test ${ reportId } for ${ chalk.yellow(
 					test.options.modules.join( ", " )
-				) }...`
+				) }...${ test.retries }`
 			);
 			return test;
 		}
diff --git a/test/runner/run.js b/test/runner/run.js
@@ -83,7 +83,19 @@ export async function run( {
 				if ( errors ) {
 					pendingErrors[ reportId ][ message.data.name ] = errors;
 				} else {
-					delete pendingErrors[ reportId ][ message.data.name ];
+					const existing = pendingErrors[ reportId ][ message.data.name ];
+
+					// Show a message for flakey tests
+					if ( existing ) {
+						console.log();
+						console.warn(
+							chalk.italic(
+								chalk.gray( existing.replace( "Test failed", "Test flakey" ) )
+							)
+						);
+						console.log();
+						delete pendingErrors[ reportId ][ message.data.name ];
+					}
 				}
 				break;
 			}
@@ -103,24 +115,15 @@ export async function run( {
 				// Handle failure
 				if ( failed ) {
 					const retry = retryTest( reportId, retries );
+
+					// Retry if retryTest returns a test
 					if ( retry ) {
 						return retry;
 					}
 					errorMessages.push( ...Object.values( pendingErrors[ reportId ] ) );
-					return getNextBrowserTest( reportId );
 				}
 
-				// Handle success
-				if (
-					pendingErrors[ reportId ] &&
-					Object.keys( pendingErrors[ reportId ] ).length
-				) {
-					console.warn( "Detected flaky tests:" );
-					for ( const [ , error ] in Object.entries( pendingErrors[ reportId ] ) ) {
-						console.warn( chalk.italic( chalk.gray( error ) ) );
-					}
-					delete pendingErrors[ reportId ];
-				}
+				// Run the next test
 				return getNextBrowserTest( reportId );
 			}
 			case "ack": {

Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ export function retryTest( reportId, maxRetries ) {`
`39`	`39`	`console.log(`
`40`	`40`	`Retrying test ${ reportId } for ${ chalk.yellow(
`41`	`41`	`test.options.modules.join( ", " )`
`42`		- ) }...`
	`42`	+ ) }...${ test.retries }`
`43`	`43`	`);`
`44`	`44`	`return test;`
`45`	`45`	`}`