test: add REPORT_ONLY mode for test collection (#3225)

This commit is contained in:
Dmitry Gozman 2020-07-31 11:02:11 -07:00 committed by GitHub
parent 9b3c90e771
commit 3edfb2a9d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 37 additions and 16 deletions

View File

@ -20,10 +20,11 @@ const os = require('os');
const path = require('path'); const path = require('path');
const fs = require('fs'); const fs = require('fs');
const debug = require('debug'); const debug = require('debug');
const platform = os.platform(); const platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
const GoldenUtils = require('../../utils/testrunner/GoldenUtils'); const GoldenUtils = require('../../utils/testrunner/GoldenUtils');
const {installCoverageHooks} = require('./coverage'); const {installCoverageHooks} = require('./coverage');
const browserName = process.env.BROWSER || 'chromium'; const browserName = process.env.BROWSER || 'chromium';
const reportOnly = !!process.env.REPORT_ONLY_PLATFORM;
class PlaywrightEnvironment extends NodeEnvironment { class PlaywrightEnvironment extends NodeEnvironment {
constructor(config, context) { constructor(config, context) {
@ -109,21 +110,32 @@ class PlaywrightEnvironment extends NodeEnvironment {
return args[0] ? describeSkip : this.global.describe; return args[0] ? describeSkip : this.global.describe;
return describeSkip(...args); return describeSkip(...args);
}; };
this.global.describe.fail = this.global.describe.skip;
function addSlow(f) {
f.slow = () => {
return (...args) => f(...args, 90000);
};
return f;
}
const itSkip = this.global.it.skip; const itSkip = this.global.it.skip;
itSkip.slow = () => itSkip; addSlow(itSkip);
addSlow(this.global.it);
this.global.it.skip = (...args) => { this.global.it.skip = (...args) => {
if (args.length === 1) if (args.length === 1)
return args[0] ? itSkip : this.global.it; return args[0] ? itSkip : this.global.it;
return itSkip(...args); return itSkip(...args);
}; };
this.global.it.fail = this.global.it.skip; if (reportOnly) {
this.global.it.slow = () => { this.global.it.fail = condition => {
return (name, fn) => { return addSlow((...inner) => {
return this.global.it(name, fn, 90000); inner[1].__fail = !!condition;
return this.global.it(...inner);
});
}; };
}; } else {
this.global.it.fail = this.global.it.skip;
}
const testOptions = this.global.testOptions; const testOptions = this.global.testOptions;
function toBeGolden(received, goldenName) { function toBeGolden(received, goldenName) {
@ -159,6 +171,11 @@ class PlaywrightEnvironment extends NodeEnvironment {
if (event.name === 'test_start') { if (event.name === 'test_start') {
const fn = event.test.fn; const fn = event.test.fn;
event.test.fn = async () => { event.test.fn = async () => {
if (reportOnly) {
if (fn.__fail)
throw new Error('fail');
return;
}
debug('pw:test')(`start "${testOrSuiteName(event.test)}"`); debug('pw:test')(`start "${testOrSuiteName(event.test)}"`);
try { try {
await this.fixturePool.resolveParametersAndRun(fn); await this.fixturePool.resolveParametersAndRun(fn);

View File

@ -14,10 +14,13 @@
* limitations under the License. * limitations under the License.
*/ */
const fs = require("fs") const fs = require('fs');
const os = require('os');
module.exports = function Reporter() { module.exports = function Reporter() {
this.onRunComplete = (test, runResults) => { this.onRunComplete = (test, runResults) => {
runResults.platform = process.env.REPORT_ONLY_PLATFORM || os.platform();
runResults.browserName = process.env.BROWSER || 'chromium';
fs.writeFileSync('jest-report.json', JSON.stringify(runResults, undefined, 2)); fs.writeFileSync('jest-report.json', JSON.stringify(runResults, undefined, 2));
} };
} }

View File

@ -113,7 +113,8 @@ describe('Async stacks', () => {
}); });
}); });
describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() { const CRASH_FAIL = (FFOX && WIN) || USES_HOOKS;
describe('Page.Events.Crash', function() {
// Firefox Win: it just doesn't crash sometimes. // Firefox Win: it just doesn't crash sometimes.
function crash(pageImpl) { function crash(pageImpl) {
@ -125,12 +126,12 @@ describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
pageImpl._delegate._session.send('Page.crash', {}).catch(e => {}); pageImpl._delegate._session.send('Page.crash', {}).catch(e => {});
} }
it('should emit crash event when page crashes', async({page, toImpl}) => { it.fail(CRASH_FAIL)('should emit crash event when page crashes', async({page, toImpl}) => {
await page.setContent(`<div>This page should crash</div>`); await page.setContent(`<div>This page should crash</div>`);
crash(toImpl(page)); crash(toImpl(page));
await new Promise(f => page.on('crash', f)); await new Promise(f => page.on('crash', f));
}); });
it('should throw on any action after page crashes', async({page, toImpl}) => { it.fail(CRASH_FAIL)('should throw on any action after page crashes', async({page, toImpl}) => {
await page.setContent(`<div>This page should crash</div>`); await page.setContent(`<div>This page should crash</div>`);
crash(toImpl(page)); crash(toImpl(page));
await page.waitForEvent('crash'); await page.waitForEvent('crash');
@ -138,14 +139,14 @@ describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
expect(err).toBeTruthy(); expect(err).toBeTruthy();
expect(err.message).toContain('crash'); expect(err.message).toContain('crash');
}); });
it('should cancel waitForEvent when page crashes', async({page, toImpl}) => { it.fail(CRASH_FAIL)('should cancel waitForEvent when page crashes', async({page, toImpl}) => {
await page.setContent(`<div>This page should crash</div>`); await page.setContent(`<div>This page should crash</div>`);
const promise = page.waitForEvent('response').catch(e => e); const promise = page.waitForEvent('response').catch(e => e);
crash(toImpl(page)); crash(toImpl(page));
const error = await promise; const error = await promise;
expect(error.message).toContain('Page crashed'); expect(error.message).toContain('Page crashed');
}); });
it('should cancel navigation when page crashes', async({page, toImpl, server}) => { it.fail(CRASH_FAIL)('should cancel navigation when page crashes', async({page, toImpl, server}) => {
await page.setContent(`<div>This page should crash</div>`); await page.setContent(`<div>This page should crash</div>`);
server.setRoute('/one-style.css', () => {}); server.setRoute('/one-style.css', () => {});
const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e); const promise = page.goto(server.PREFIX + '/one-style.html').catch(e => e);
@ -154,7 +155,7 @@ describe.fail(FFOX && WIN || USES_HOOKS)('Page.Events.Crash', function() {
const error = await promise; const error = await promise;
expect(error.message).toContain('Navigation failed because page crashed'); expect(error.message).toContain('Navigation failed because page crashed');
}); });
it('should be able to close context when page crashes', async({page, toImpl}) => { it.fail(CRASH_FAIL)('should be able to close context when page crashes', async({page, toImpl}) => {
await page.setContent(`<div>This page should crash</div>`); await page.setContent(`<div>This page should crash</div>`);
crash(toImpl(page)); crash(toImpl(page));
await page.waitForEvent('crash'); await page.waitForEvent('crash');