diff --git a/.gitignore b/.gitignore index 0c5e6d130e..b202fa9c6d 100644 --- a/.gitignore +++ b/.gitignore @@ -43,5 +43,6 @@ test/shared/**/helper.d.ts test/shared/**/screenshot.d.ts test/shared/**/*.tsbuildinfo test/screenshots/**/*.js +!test/screenshots/**/interactive.js test/screenshots/**/*.tsbuildinfo test/screenshots/.generated diff --git a/package.json b/package.json index e3a25b7355..34265db3fc 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "e2etest": "python scripts/test/run_test_suite.py --test-suite=e2e", "generate-jsconfig": "node scripts/generate_jsconfig.js", "install-deps": "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true python scripts/deps/manage_node_deps.py", + "interactive-screenshottest": "INTERACTIVE=1 python scripts/test/run_test_suite.py --test-suite=screenshots", "optimize-svg-images": "python scripts/optimize_svg_images.py", "preinstall": "if [ -z ${PUPPETEER_SKIP_CHROMIUM_DOWNLOAD+x} ]; then echo 'Run npm run install-deps instead' && exit 1; fi", "screenshottest": "python scripts/test/run_test_suite.py --test-suite=screenshots", @@ -44,4 +45,4 @@ "test-local": "node scripts/npm_test.js --layout-tests-directory=test/webtests/", "unittest": "NO_TEXT_COVERAGE=1 python scripts/test/run_unittests.py" } -} \ No newline at end of file +} diff --git a/test/screenshots/goldens/hello-world.png b/test/screenshots/goldens/hello-world.png index 67e885530d..fed7023882 100644 Binary files a/test/screenshots/goldens/hello-world.png and b/test/screenshots/goldens/hello-world.png differ diff --git a/test/screenshots/hello-world/hello-world.ts b/test/screenshots/hello-world/hello-world.ts index 42ce8537a3..c063fb2c25 100644 --- a/test/screenshots/hello-world/hello-world.ts +++ b/test/screenshots/hello-world/hello-world.ts @@ -13,7 +13,7 @@ describe('hello world', () => { it('takes a screenshot', async () => { const {target, frontend} = getBrowserAndPages(); - await target.goto(`${resourcesPath}/console/big-int.html`); + await target.goto(`${resourcesPath}/console/built-ins.html`); await click('#tab-console'); await frontend.waitForSelector('.console-group-messages'); diff --git a/test/screenshots/image_diff/mac/image_diff b/test/screenshots/image_diff/mac/image_diff new file mode 100755 index 0000000000..407b65ab98 Binary files /dev/null and b/test/screenshots/image_diff/mac/image_diff differ diff --git a/test/screenshots/image_diff/mac/libbase.dylib b/test/screenshots/image_diff/mac/libbase.dylib new file mode 100755 index 0000000000..b761216e49 Binary files /dev/null and b/test/screenshots/image_diff/mac/libbase.dylib differ diff --git a/test/screenshots/image_diff/mac/libc++.dylib b/test/screenshots/image_diff/mac/libc++.dylib new file mode 100755 index 0000000000..6d45625a79 Binary files /dev/null and b/test/screenshots/image_diff/mac/libc++.dylib differ diff --git a/test/screenshots/image_diff/mac/libchrome_zlib.dylib b/test/screenshots/image_diff/mac/libchrome_zlib.dylib new file mode 100755 index 0000000000..bfef73d713 Binary files /dev/null and b/test/screenshots/image_diff/mac/libchrome_zlib.dylib differ diff --git a/test/screenshots/interactive/index.html b/test/screenshots/interactive/index.html new file mode 100644 index 0000000000..7efd53d2c8 --- /dev/null +++ b/test/screenshots/interactive/index.html @@ -0,0 +1,47 @@ + + + + + + Screenshot browser + + + +
+
+

File: ...

+

Percentage difference: ...

+
+
+
+
+

Shortcuts:

+ +
+
+
+
+
+

Golden

+
+
+ +
+

Test output

+
+
+
+
+ +
+
+ + + diff --git a/test/screenshots/interactive/interactive.css b/test/screenshots/interactive/interactive.css new file mode 100644 index 0000000000..97e55d661a --- /dev/null +++ b/test/screenshots/interactive/interactive.css @@ -0,0 +1,210 @@ +/** + * Copyright 2020 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +* { + box-sizing: border-box; +} + +html, body { + --max-width: 1600px; + width: 100%; + height: 100%; + margin: 0; + padding: 0; + font-family: Arial, Helvetica, sans-serif; +} + +body { + display: flex; + flex-direction: column; +} + +header { + height: 85px; + background: #0083C3; + color: #FFF; + box-shadow: 0 2px 4px 0 rgba(0,0,0,0.50); + display: flex; + align-items: center; + justify-content: center; + position: relative; + z-index: 3; +} + +header h1, +header h2 { + font-size: 16px; + font-weight: bold; + margin: 0; + padding: 2px 0; +} + +header h1 .file, +header h2 .percentage { + font-weight: normal; +} + +.header-container, +.shortcut-container { + width: 100%; + max-width: var(--max-width); + padding: 0 20px; + display: flex; + flex-direction: column; + justify-content: center; +} + +.shortcuts { + background: #003B59; + box-shadow: 0 2px 4px 0 rgba(0,0,0,0.50); + height: 48px; + position: relative; + z-index: 2; +} + +.diff { + top: 0; + left: 0; + right: 0; + bottom: 0; + position: fixed; + z-index: 10; + pointer-events: none; + background: rgba(0,0,0,0.70); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + opacity: 0; + transition: opacity 0.2s cubic-bezier(0, 0, 0.4, 1); +} + +.diff.visible { + opacity: 1; + pointer-events: auto; +} + +main { + margin: 0 auto; + width: 100%; + max-width: var(--max-width); + padding: 0 20px; + flex: 1; +} + +.main-container { + padding: 40px 0; + width: 100%; + height: 100%; + grid-template-columns: 1fr 1fr; + grid-gap: 40px; + display: grid; +} + +main img, +.diff img { + max-width: 100%; + border: 1px solid #D0D0D0; + box-shadow: 0 6px 7px 0 rgba(0,0,0,0.11); + background: #FFF; + cursor: pointer; +} + +main img.selected { + outline: 6px solid rgba(255, 0, 120, 0.6); +} + +main h3 { + font-size: 14px; + color: #303942; +} + +footer { + height: 48px; + background: #333; + padding: 12px; + display: flex; + flex-direction: row; + align-items: center; + justify-content: flex-end; +} + +footer button { + margin-left: 10px; + padding: 4px 8px; + border-radius: 2px; + background: none; + border: 1px solid #FFF; + color: #FFF; +} + +footer button[disabled] { + opacity: 0.4;; +} + +.status { + width: 100%; + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + position: fixed; + top: 0; + left: 0; +} + +.shortcut-container { + display: flex; + flex-direction: row; + align-items: center; + justify-content: left; + color: #FFF; + margin: 0 auto; + width: 100%; + max-width: var(--max-width); +} + +.shortcut-container h3 { + font-size: 10px; + padding-right: 30px; +} + +.shortcut-container ul { + list-style: none; + display: flex; + padding: 0; +} + +.shortcut-container li { + padding-right: 18px; + font-size: 11px; + font-weight: bold; +} + +.shortcut-container li::before { + content: ''; + padding: 3px 8px; + border-radius: 3px; + border: 1px solid rgba(255,255,255,0.7); + margin-right: 8px; + font-size: 12px; +} +.shortcut-container li.shortcut-golden::before { + content: '←'; +} + +.shortcut-container li.shortcut-generated::before { + content: '→'; +} + +.shortcut-container li.shortcut-confirm::before { + content: 'Enter'; +} + +.shortcut-container li.shortcut-skip::before { + content: 's'; +} diff --git a/test/screenshots/interactive/interactive.js b/test/screenshots/interactive/interactive.js new file mode 100644 index 0000000000..384d44ec72 --- /dev/null +++ b/test/screenshots/interactive/interactive.js @@ -0,0 +1,137 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +const main = document.querySelector('main'); +const mainContainer = document.querySelector('.main-container'); +const goldenImageContainer = mainContainer.querySelector('.golden .image-container'); +const generatedImageContainer = mainContainer.querySelector('.generated .image-container'); +const status = document.querySelector('.status'); +const file = document.querySelector('.file'); +const percentage = document.querySelector('.percentage'); +const diffImageContainer = document.querySelector('.diff'); +const testSkip = document.querySelector('.test-skip'); +const testConfirm = document.querySelector('.test-confirm'); + +const choiceMarker = document.createElement('div'); +choiceMarker.classList.add('choice'); + +const leftImage = new Image(); +const rightImage = new Image(); +const diffImage = new Image(); + +let listening = false; +function setState({type, msg, left, right, diff, rawMisMatchPercentage, fileName}) { + switch (type) { + case 'status': + listening = false; + document.body.appendChild(status); + status.textContent = msg; + mainContainer.remove(); + disableButtons(); + break; + + case 'outcome': + main.appendChild(mainContainer); + status.remove(); + choiceMarker.remove(); + + file.textContent = fileName; + percentage.textContent = `${rawMisMatchPercentage}%`; + + // Golden. + leftImage.src = left; + goldenImageContainer.appendChild(leftImage); + + // Generated. + rightImage.src = right; + generatedImageContainer.appendChild(rightImage); + + // Diff. + diffImage.src = diff; + diffImageContainer.appendChild(diffImage); + diffImage.onerror = () => { + diffImage.src = './no-diff-available.png'; + }; + + listening = true; + enableSkipButton(); + break; + } +} + +function makeChoice(which) { + choiceMarker.dataset.choice = which; + leftImage.classList.toggle('selected', which === 'golden'); + rightImage.classList.toggle('selected', which === 'generated'); + enableConfirmButton(); +} + +function commitChoice() { + document.body.appendChild(choiceMarker); + setWaiting(); +} + +function setWaiting() { + setState({type: 'status', msg: 'Waiting...'}); +} + +function enableSkipButton() { + testSkip.disabled = false; +} + +function enableConfirmButton() { + testConfirm.disabled = false; +} + +function disableButtons() { + testSkip.disabled = true; + testConfirm.disabled = true; +} + +function commitSkip() { + makeChoice('skip'); + commitChoice(); +} + +function commitChoiceIfPossible() { + if (!choiceMarker.dataset.choice) { + return; + } + commitChoice(); +} + +window.addEventListener('keyup', evt => { + if (!listening) { + return; + } + + switch (evt.key) { + case 'ArrowLeft': + makeChoice('golden'); + return; + + case 'ArrowRight': + makeChoice('generated'); + return; + + case 'd': + diffImageContainer.classList.toggle('visible'); + return; + + case 's': + commitSkip(); + return; + + case 'Enter': + commitChoiceIfPossible(); + return; + } +}); + +leftImage.addEventListener('click', () => makeChoice('golden')); +rightImage.addEventListener('click', () => makeChoice('generated')); +testSkip.addEventListener('click', () => commitSkip()); +testConfirm.addEventListener('click', () => commitChoiceIfPossible()); + +setWaiting(); diff --git a/test/screenshots/interactive/no-diff-available.png b/test/screenshots/interactive/no-diff-available.png new file mode 100644 index 0000000000..7c76211816 Binary files /dev/null and b/test/screenshots/interactive/no-diff-available.png differ diff --git a/test/shared/helper.ts b/test/shared/helper.ts index 68c6cf05b1..d93f992540 100644 --- a/test/shared/helper.ts +++ b/test/shared/helper.ts @@ -9,10 +9,12 @@ interface BrowserAndPages { browser: puppeteer.Browser; target: puppeteer.Page; frontend: puppeteer.Page; + screenshot?: puppeteer.Page; } const targetPage = Symbol('TargetPage'); const frontEndPage = Symbol('DevToolsPage'); +const screenshotPage = Symbol('ScreenshotPage'); const browserInstance = Symbol('BrowserInstance'); export let resetPages: (...enabledExperiments: string[]) => void; @@ -149,11 +151,12 @@ export const debuggerStatement = (frontend: puppeteer.Page) => { }; export const store = - (browser: puppeteer.Browser, target: puppeteer.Page, frontend: puppeteer.Page, + (browser: puppeteer.Browser, target: puppeteer.Page, frontend: puppeteer.Page, screenshot: puppeteer.Page | undefined, reset: (...enabledExperiments: string[]) => void) => { globalThis[browserInstance] = browser; globalThis[targetPage] = target; globalThis[frontEndPage] = frontend; + globalThis[screenshotPage] = screenshot; resetPages = reset; }; @@ -174,6 +177,7 @@ export const getBrowserAndPages = (): BrowserAndPages => { browser: globalThis[browserInstance], target: globalThis[targetPage], frontend: globalThis[frontEndPage], + screenshot: globalThis[screenshotPage], }; }; diff --git a/test/shared/resemblejs.d.ts b/test/shared/resemblejs.d.ts deleted file mode 100644 index d5d451a8eb..0000000000 --- a/test/shared/resemblejs.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -declare module 'resemblejs' { - interface ResembleComparisonResult { - dimensionDifference: { width: number, height: number }; - rawMisMatchPercentage: number; - } - - export function compare(file1: string, file2: string, options: {}, - cb:(err: Error, data: ResembleComparisonResult) => void): void; -} diff --git a/test/shared/runner.ts b/test/shared/runner.ts index 31400aaf9e..0172a5e9b5 100644 --- a/test/shared/runner.ts +++ b/test/shared/runner.ts @@ -16,6 +16,8 @@ const envChromeBinary = process.env['CHROME_BIN']; const envDebug = !!process.env['DEBUG']; const envPort = process.env['PORT'] || 9222; const envNoShuffle = !!process.env['NO_SHUFFLE']; +const envInteractive = !!process.env['INTERACTIVE']; +const interactivePage = 'http://localhost:8090/test/screenshots/interactive/index.html'; const blankPage = 'data:text/html,'; const headless = !envDebug; const width = 1280; @@ -97,6 +99,18 @@ interface DevToolsTarget { // 3. Spin up the test environment (async function() { try { + let screenshotPage: puppeteer.Page | undefined; + if (envInteractive) { + const screenshotBrowser = await puppeteer.launch({ + headless: false, + executablePath: envChromeBinary, + defaultViewport: null, + args: [`--window-size=${width},${height}`], + }); + screenshotPage = await screenshotBrowser.newPage(); + await screenshotPage.goto(interactivePage, {waitUntil: ['domcontentloaded']}); + } + const browser = await launchedBrowser; // Load the target page. @@ -156,7 +170,7 @@ interface DevToolsTarget { await frontend.waitForSelector('.elements'); }; - store(browser, srcPage, frontend, resetPages); + store(browser, srcPage, frontend, screenshotPage, resetPages); // 3. Run tests. do { @@ -208,7 +222,7 @@ async function runTests() { } mocha.ui('bdd'); mocha.reporter('list'); - mocha.timeout(envDebug ? 100000 : 4000); + mocha.timeout((envDebug || envInteractive) ? 300000 : 4000); mochaRun = mocha.run(); mochaRun.on('end', () => { diff --git a/test/shared/screenshot.ts b/test/shared/screenshot.ts index c7385fdc4d..fa6f2c7ddc 100644 --- a/test/shared/screenshot.ts +++ b/test/shared/screenshot.ts @@ -3,11 +3,14 @@ // found in the LICENSE file. import * as puppeteer from 'puppeteer'; -import * as resemblejs from 'resemblejs'; import {assert} from 'chai'; import {join} from 'path'; import * as fs from 'fs'; import * as rimraf from 'rimraf'; +import * as childProcess from 'child_process'; +import * as os from 'os'; +import * as path from 'path'; +import {getBrowserAndPages} from './helper.js'; const goldensScreenshotFolder = join(__dirname, '..', 'screenshots', 'goldens'); const generatedScreenshotFolder = join(__dirname, '..', 'screenshots', '.generated'); @@ -34,18 +37,104 @@ export const assertScreenshotUnchanged = async (page: puppeteer.Page, fileName: const opts = {...defaultScreenshotOpts, ...options, path: generatedScreenshotPath}; await page.screenshot(opts); - return new Promise((resolve, reject) => { - resemblejs.compare(generatedScreenshotPath, goldensScreenshotPath, {}, (err, data) => { - if (err) { - reject(err); + return compare(goldensScreenshotPath, generatedScreenshotPath, fileName); +}; + +interface ImageDiff { + rawMisMatchPercentage: number; + diffPath: string; +} + +async function imageDiff(golden: string, generated: string, isInteractive = false) { + let imageDiffDir: string; + + switch (os.platform()) { + case 'darwin': + imageDiffDir = 'mac'; + break; + + case 'win32': + imageDiffDir = 'win32'; + break; + + default: + imageDiffDir = 'linux'; + break; + } + + const imageDiffPath = join(__dirname, '..', 'screenshots', 'image_diff', imageDiffDir, 'image_diff'); + return new Promise(async (resolve, reject) => { + const imageDiff: ImageDiff = {rawMisMatchPercentage: 0, diffPath: ''}; + const diffText = await exec(`${imageDiffPath} --histogram ${golden} ${generated}`); + + // Parse out the number from the cmd output, i.e. diff: 48.9% failed => 48.9 + imageDiff.rawMisMatchPercentage = Number(diffText.replace(/^diff:\s/, '').replace(/%.*/, '')); + + if (Number.isNaN(imageDiff.rawMisMatchPercentage)) { + reject('Unable to compare images'); + } + + // Only create a diff image if necessary. + if (isInteractive || imageDiff.rawMisMatchPercentage > 0) { + imageDiff.diffPath = join(path.dirname(generated), `${path.basename(generated, '.png')}-diff.png`); + await exec(`${imageDiffPath} --diff ${golden} ${generated} ${imageDiff.diffPath}`); + } + + resolve(imageDiff); + }); +} + +async function exec(cmd: string) { + return new Promise((resolve, reject) => { + let commandOutput = ''; + try { + commandOutput = childProcess.execSync(cmd, {encoding: 'utf8'}); + resolve(commandOutput); + } catch (e) { + // image_diff will exit with a status code of 1 if the diff is too big, so + // this needs to be caught, but the outcome is the same - we want to send + // back the string for processing. + if (e.stdout.indexOf('diff') === -1) { + reject(e.stdout); return; } - const {dimensionDifference, rawMisMatchPercentage} = data; - assert.deepEqual(dimensionDifference, { width: 0, height: 0}); - assert.isBelow(rawMisMatchPercentage, 1); - resolve(); - }); - resolve(); + resolve(e.stdout); + } }); -}; +} + +async function compare(golden: string, generated: string, fileName: string) { + const {screenshot} = getBrowserAndPages(); + if (screenshot) { + await screenshot.evaluate(opts => { + (self as any).setState(opts); + }, {type: 'status', msg: `Comparing ${fileName} to generated image...`}); + } + + const isInteractive = typeof screenshot !== 'undefined'; + const {rawMisMatchPercentage, diffPath} = await imageDiff(golden, generated, isInteractive); + + // Interactively allow the user to choose. + if (screenshot && diffPath) { + const root = join(__dirname, '..', '..'); + const left = golden.replace(root, ''); + const right = generated.replace(root, ''); + const diff = diffPath.replace(root, ''); + const type = 'outcome'; + + await screenshot.evaluate(opts => { + (self as any).setState(opts); + }, {type, left, right, diff, rawMisMatchPercentage, fileName}); + + const elementHandle = await screenshot.waitForSelector('.choice', {timeout: 0}); + const choice = await elementHandle.evaluate(node => node.getAttribute('data-choice')); + + // If they choose the test output, copy the generated screenshot over the golden. + if (choice === 'generated') { + fs.copyFileSync(generated, golden); + } + } else { // Assert no change. + assert.isBelow(rawMisMatchPercentage, 1, `There is a ${rawMisMatchPercentage}% difference`); + } +}