fix(test runner): do not write missing snapshot until the last retry (#9246)

This prevents future retries from passing because of the actual
snapshot being written.

In theory, we can avoid running the retry since it should fail anyway.
However, this brings problems, for example in the `describe.serial` mode
where running a test also has some side effects and so it should not be
skipped. Since running a test without a snapshot is rare, it should be
fine to retry it.
This commit is contained in:
Dmitry Gozman 2021-09-30 16:44:52 -07:00 committed by GitHub
parent fcb7d2b15a
commit 5e3ad63b42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 1 deletions

View File

@ -46,7 +46,7 @@ export function toMatchSnapshot(this: ReturnType<Expect['getState']>, received:
options.name, options.name,
testInfo.snapshotPath, testInfo.snapshotPath,
testInfo.outputPath, testInfo.outputPath,
testInfo.config.updateSnapshots, testInfo.retry < testInfo.project.retries ? 'none' : testInfo.config.updateSnapshots,
withNegateComparison, withNegateComparison,
options options
); );

View File

@ -562,3 +562,24 @@ test('should attach expected/actual and no diff', async ({ runInlineTest }, test
]); ]);
}); });
test('should fail with missing expectations and retries', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...files,
'playwright.config.ts': `
module.exports = { retries: 1 };
`,
'a.spec.js': `
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('snapshot.txt');
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.txt');
expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
const data = fs.readFileSync(snapshotOutputPath);
expect(data.toString()).toBe('Hello world');
});