[screenshot] Add helper util to screenshots

This adds support for image diffing interactively. A future CL will need
to include image_diff for Linux and win32.

Bug: 1051458
Change-Id: Ifdcb7ee59dff1f8a93fae19db6a1633e5c0bcdcc
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2060004
Commit-Queue: Paul Lewis <aerotwist@chromium.org>
Reviewed-by: Tim van der Lippe <tvanderlippe@chromium.org>

.gitignore

@@ -43,5 +43,6 @@ test/shared/**/helper.d.ts
 test/shared/**/screenshot.d.ts
 test/shared/**/*.tsbuildinfo
 test/screenshots/**/*.js
+!test/screenshots/**/interactive.js
 test/screenshots/**/*.tsbuildinfo
 test/screenshots/.generated

package.json

@@ -34,6 +34,7 @@
     "e2etest": "python scripts/test/run_test_suite.py --test-suite=e2e",
     "generate-jsconfig": "node scripts/generate_jsconfig.js",
     "install-deps": "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true python scripts/deps/manage_node_deps.py",
+    "interactive-screenshottest": "INTERACTIVE=1 python scripts/test/run_test_suite.py --test-suite=screenshots",
     "optimize-svg-images": "python scripts/optimize_svg_images.py",
     "preinstall": "if [ -z ${PUPPETEER_SKIP_CHROMIUM_DOWNLOAD+x} ]; then echo 'Run npm run install-deps instead' && exit 1; fi",
     "screenshottest": "python scripts/test/run_test_suite.py --test-suite=screenshots",
@@ -44,4 +45,4 @@
     "test-local": "node scripts/npm_test.js --layout-tests-directory=test/webtests/",
     "unittest": "NO_TEXT_COVERAGE=1 python scripts/test/run_unittests.py"
   }
-}
+}

test/screenshots/hello-world/hello-world.ts

@@ -13,7 +13,7 @@ describe('hello world', () => {
 
   it('takes a screenshot', async () => {
     const {target, frontend} = getBrowserAndPages();
-    await target.goto(`${resourcesPath}/console/big-int.html`);
+    await target.goto(`${resourcesPath}/console/built-ins.html`);
     await click('#tab-console');
     await frontend.waitForSelector('.console-group-messages');
 

test/screenshots/interactive/index.html

@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Screenshot browser</title>
+  <link rel="stylesheet" href="./interactive.css">
+</head>
+<body>
+  <header>
+    <div class="header-container">
+      <h1>File: <span class="file">...</span></h1>
+      <h2>Percentage difference: <span class="percentage">...</span></h2>
+    </div>
+  </header>
+  <div class="shortcuts">
+    <div class="shortcut-container">
+      <h3>Shortcuts:</h3>
+      <ul>
+        <li class="shortcut shortcut-golden">Choose golden</li>
+        <li class="shortcut shortcut-generated">Choose test output</li>
+        <li class="shortcut shortcut-confirm">Confirm choice</li>
+        <li class="shortcut shortcut-skip">Skip test</li>
+      </ul>
+    </div>
+  </div>
+  <main>
+    <div class="main-container">
+      <div class="golden">
+        <h3>Golden</h3>
+        <div class="image-container"></div>
+      </div>
+
+      <div class="generated">
+        <h3>Test output</h3>
+        <div class="image-container"></div>
+      </div>
+    </div>
+  </main>
+  <footer>
+    <button class="test-skip" disabled>Skip</button> <button class="test-confirm" disabled>Confirm</button>
+  </footer>
+  <div class="diff"></div>
+  <div class="status"></div>
+  <script src="./interactive.js"></script>
+</body>
+</html>

test/screenshots/interactive/interactive.css

@@ -0,0 +1,210 @@
+/**
+ * Copyright 2020 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+* {
+  box-sizing: border-box;
+}
+
+html, body {
+  --max-width: 1600px;
+  width: 100%;
+  height: 100%;
+  margin: 0;
+  padding: 0;
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+body {
+  display: flex;
+  flex-direction: column;
+}
+
+header {
+  height: 85px;
+  background: #0083C3;
+  color: #FFF;
+  box-shadow: 0 2px 4px 0 rgba(0,0,0,0.50);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  position: relative;
+  z-index: 3;
+}
+
+header h1,
+header h2 {
+  font-size: 16px;
+  font-weight: bold;
+  margin: 0;
+  padding: 2px 0;
+}
+
+header h1 .file,
+header h2 .percentage {
+  font-weight: normal;
+}
+
+.header-container,
+.shortcut-container {
+  width: 100%;
+  max-width: var(--max-width);
+  padding: 0 20px;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+}
+
+.shortcuts {
+  background: #003B59;
+  box-shadow: 0 2px 4px 0 rgba(0,0,0,0.50);
+  height: 48px;
+  position: relative;
+  z-index: 2;
+}
+
+.diff {
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  position: fixed;
+  z-index: 10;
+  pointer-events: none;
+  background: rgba(0,0,0,0.70);
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  opacity: 0;
+  transition: opacity 0.2s cubic-bezier(0, 0, 0.4, 1);
+}
+
+.diff.visible {
+  opacity: 1;
+  pointer-events: auto;
+}
+
+main {
+  margin: 0 auto;
+  width: 100%;
+  max-width: var(--max-width);
+  padding: 0 20px;
+  flex: 1;
+}
+
+.main-container {
+  padding: 40px 0;
+  width: 100%;
+  height: 100%;
+  grid-template-columns: 1fr 1fr;
+  grid-gap: 40px;
+  display: grid;
+}
+
+main img,
+.diff img {
+  max-width: 100%;
+  border: 1px solid #D0D0D0;
+  box-shadow: 0 6px 7px 0 rgba(0,0,0,0.11);
+  background: #FFF;
+  cursor: pointer;
+}
+
+main img.selected {
+  outline: 6px solid rgba(255, 0, 120, 0.6);
+}
+
+main h3 {
+  font-size: 14px;
+  color: #303942;
+}
+
+footer {
+  height: 48px;
+  background: #333;
+  padding: 12px;
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: flex-end;
+}
+
+footer button {
+  margin-left: 10px;
+  padding: 4px 8px;
+  border-radius: 2px;
+  background: none;
+  border: 1px solid #FFF;
+  color: #FFF;
+}
+
+footer button[disabled] {
+  opacity: 0.4;;
+}
+
+.status {
+  width: 100%;
+  height: 100%;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  position: fixed;
+  top: 0;
+  left: 0;
+}
+
+.shortcut-container {
+  display: flex;
+  flex-direction: row;
+  align-items: center;
+  justify-content: left;
+  color: #FFF;
+  margin: 0 auto;
+  width: 100%;
+  max-width: var(--max-width);
+}
+
+.shortcut-container h3 {
+  font-size: 10px;
+  padding-right: 30px;
+}
+
+.shortcut-container ul {
+  list-style: none;
+  display: flex;
+  padding: 0;
+}
+
+.shortcut-container li {
+  padding-right: 18px;
+  font-size: 11px;
+  font-weight: bold;
+}
+
+.shortcut-container li::before {
+  content: '';
+  padding: 3px 8px;
+  border-radius: 3px;
+  border: 1px solid rgba(255,255,255,0.7);
+  margin-right: 8px;
+  font-size: 12px;
+}
+.shortcut-container li.shortcut-golden::before {
+  content: '←';
+}
+
+.shortcut-container li.shortcut-generated::before {
+  content: '→';
+}
+
+.shortcut-container li.shortcut-confirm::before {
+  content: 'Enter';
+}
+
+.shortcut-container li.shortcut-skip::before {
+  content: 's';
+}

test/screenshots/interactive/interactive.js

@@ -0,0 +1,137 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+const main = document.querySelector('main');
+const mainContainer = document.querySelector('.main-container');
+const goldenImageContainer = mainContainer.querySelector('.golden .image-container');
+const generatedImageContainer = mainContainer.querySelector('.generated .image-container');
+const status = document.querySelector('.status');
+const file = document.querySelector('.file');
+const percentage = document.querySelector('.percentage');
+const diffImageContainer = document.querySelector('.diff');
+const testSkip = document.querySelector('.test-skip');
+const testConfirm = document.querySelector('.test-confirm');
+
+const choiceMarker = document.createElement('div');
+choiceMarker.classList.add('choice');
+
+const leftImage = new Image();
+const rightImage = new Image();
+const diffImage = new Image();
+
+let listening = false;
+function setState({type, msg, left, right, diff, rawMisMatchPercentage, fileName}) {
+  switch (type) {
+    case 'status':
+      listening = false;
+      document.body.appendChild(status);
+      status.textContent = msg;
+      mainContainer.remove();
+      disableButtons();
+      break;
+
+    case 'outcome':
+      main.appendChild(mainContainer);
+      status.remove();
+      choiceMarker.remove();
+
+      file.textContent = fileName;
+      percentage.textContent = `${rawMisMatchPercentage}%`;
+
+      // Golden.
+      leftImage.src = left;
+      goldenImageContainer.appendChild(leftImage);
+
+      // Generated.
+      rightImage.src = right;
+      generatedImageContainer.appendChild(rightImage);
+
+      // Diff.
+      diffImage.src = diff;
+      diffImageContainer.appendChild(diffImage);
+      diffImage.onerror = () => {
+        diffImage.src = './no-diff-available.png';
+      };
+
+      listening = true;
+      enableSkipButton();
+      break;
+  }
+}
+
+function makeChoice(which) {
+  choiceMarker.dataset.choice = which;
+  leftImage.classList.toggle('selected', which === 'golden');
+  rightImage.classList.toggle('selected', which === 'generated');
+  enableConfirmButton();
+}
+
+function commitChoice() {
+  document.body.appendChild(choiceMarker);
+  setWaiting();
+}
+
+function setWaiting() {
+  setState({type: 'status', msg: 'Waiting...'});
+}
+
+function enableSkipButton() {
+  testSkip.disabled = false;
+}
+
+function enableConfirmButton() {
+  testConfirm.disabled = false;
+}
+
+function disableButtons() {
+  testSkip.disabled = true;
+  testConfirm.disabled = true;
+}
+
+function commitSkip() {
+  makeChoice('skip');
+  commitChoice();
+}
+
+function commitChoiceIfPossible() {
+  if (!choiceMarker.dataset.choice) {
+    return;
+  }
+  commitChoice();
+}
+
+window.addEventListener('keyup', evt => {
+  if (!listening) {
+    return;
+  }
+
+  switch (evt.key) {
+    case 'ArrowLeft':
+      makeChoice('golden');
+      return;
+
+    case 'ArrowRight':
+      makeChoice('generated');
+      return;
+
+    case 'd':
+      diffImageContainer.classList.toggle('visible');
+      return;
+
+    case 's':
+      commitSkip();
+      return;
+
+    case 'Enter':
+      commitChoiceIfPossible();
+      return;
+  }
+});
+
+leftImage.addEventListener('click', () => makeChoice('golden'));
+rightImage.addEventListener('click', () => makeChoice('generated'));
+testSkip.addEventListener('click', () => commitSkip());
+testConfirm.addEventListener('click', () => commitChoiceIfPossible());
+
+setWaiting();

test/shared/helper.ts

@@ -9,10 +9,12 @@ interface BrowserAndPages {
   browser: puppeteer.Browser;
   target: puppeteer.Page;
   frontend: puppeteer.Page;
+  screenshot?: puppeteer.Page;
 }
 
 const targetPage = Symbol('TargetPage');
 const frontEndPage = Symbol('DevToolsPage');
+const screenshotPage = Symbol('ScreenshotPage');
 const browserInstance = Symbol('BrowserInstance');
 
 export let resetPages: (...enabledExperiments: string[]) => void;
@@ -149,11 +151,12 @@ export const debuggerStatement = (frontend: puppeteer.Page) => {
 };
 
 export const store =
-    (browser: puppeteer.Browser, target: puppeteer.Page, frontend: puppeteer.Page,
+    (browser: puppeteer.Browser, target: puppeteer.Page, frontend: puppeteer.Page, screenshot: puppeteer.Page | undefined,
      reset: (...enabledExperiments: string[]) => void) => {
       globalThis[browserInstance] = browser;
       globalThis[targetPage] = target;
       globalThis[frontEndPage] = frontend;
+      globalThis[screenshotPage] = screenshot;
       resetPages = reset;
     };
 
@@ -174,6 +177,7 @@ export const getBrowserAndPages = (): BrowserAndPages => {
     browser: globalThis[browserInstance],
     target: globalThis[targetPage],
     frontend: globalThis[frontEndPage],
+    screenshot: globalThis[screenshotPage],
   };
 };
 

test/shared/resemblejs.d.ts

@@ -1,13 +0,0 @@
-// Copyright 2020 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-declare module 'resemblejs' {
-  interface ResembleComparisonResult {
-    dimensionDifference: { width: number, height: number };
-    rawMisMatchPercentage: number;
-  }
-
-  export function compare(file1: string, file2: string, options: {},
-      cb:(err: Error, data: ResembleComparisonResult) => void): void;
-}

test/shared/runner.ts

@@ -16,6 +16,8 @@ const envChromeBinary = process.env['CHROME_BIN'];
 const envDebug = !!process.env['DEBUG'];
 const envPort = process.env['PORT'] || 9222;
 const envNoShuffle = !!process.env['NO_SHUFFLE'];
+const envInteractive = !!process.env['INTERACTIVE'];
+const interactivePage = 'http://localhost:8090/test/screenshots/interactive/index.html';
 const blankPage = 'data:text/html,';
 const headless = !envDebug;
 const width = 1280;
@@ -97,6 +99,18 @@ interface DevToolsTarget {
 // 3. Spin up the test environment
 (async function() {
   try {
+    let screenshotPage: puppeteer.Page | undefined;
+    if (envInteractive) {
+      const screenshotBrowser = await puppeteer.launch({
+        headless: false,
+        executablePath: envChromeBinary,
+        defaultViewport: null,
+        args: [`--window-size=${width},${height}`],
+      });
+      screenshotPage = await screenshotBrowser.newPage();
+      await screenshotPage.goto(interactivePage, {waitUntil: ['domcontentloaded']});
+    }
+
     const browser = await launchedBrowser;
 
     // Load the target page.
@@ -156,7 +170,7 @@ interface DevToolsTarget {
       await frontend.waitForSelector('.elements');
     };
 
-    store(browser, srcPage, frontend, resetPages);
+    store(browser, srcPage, frontend, screenshotPage, resetPages);
 
     // 3. Run tests.
     do {
@@ -208,7 +222,7 @@ async function runTests() {
     }
     mocha.ui('bdd');
     mocha.reporter('list');
-    mocha.timeout(envDebug ? 100000 : 4000);
+    mocha.timeout((envDebug || envInteractive) ? 300000 : 4000);
 
     mochaRun = mocha.run();
     mochaRun.on('end', () => {

test/shared/screenshot.ts

@@ -3,11 +3,14 @@
 // found in the LICENSE file.
 
 import * as puppeteer from 'puppeteer';
-import * as resemblejs from 'resemblejs';
 import {assert} from 'chai';
 import {join} from 'path';
 import * as fs from 'fs';
 import * as rimraf from 'rimraf';
+import * as childProcess from 'child_process';
+import * as os from 'os';
+import * as path from 'path';
+import {getBrowserAndPages} from './helper.js';
 
 const goldensScreenshotFolder = join(__dirname, '..', 'screenshots', 'goldens');
 const generatedScreenshotFolder = join(__dirname, '..', 'screenshots', '.generated');
@@ -34,18 +37,104 @@ export const assertScreenshotUnchanged = async (page: puppeteer.Page, fileName:
   const opts = {...defaultScreenshotOpts, ...options, path: generatedScreenshotPath};
   await page.screenshot(opts);
 
-  return new Promise((resolve, reject) => {
-    resemblejs.compare(generatedScreenshotPath, goldensScreenshotPath, {}, (err, data) => {
-      if (err) {
-        reject(err);
+  return compare(goldensScreenshotPath, generatedScreenshotPath, fileName);
+};
+
+interface ImageDiff {
+  rawMisMatchPercentage: number;
+  diffPath: string;
+}
+
+async function imageDiff(golden: string, generated: string, isInteractive = false) {
+  let imageDiffDir: string;
+
+  switch (os.platform()) {
+    case 'darwin':
+      imageDiffDir = 'mac';
+      break;
+
+    case 'win32':
+      imageDiffDir = 'win32';
+      break;
+
+    default:
+      imageDiffDir = 'linux';
+      break;
+  }
+
+  const imageDiffPath = join(__dirname, '..', 'screenshots', 'image_diff', imageDiffDir, 'image_diff');
+  return new Promise<ImageDiff>(async (resolve, reject) => {
+    const imageDiff: ImageDiff = {rawMisMatchPercentage: 0, diffPath: ''};
+    const diffText = await exec(`${imageDiffPath} --histogram ${golden} ${generated}`);
+
+    // Parse out the number from the cmd output, i.e. diff: 48.9% failed => 48.9
+    imageDiff.rawMisMatchPercentage = Number(diffText.replace(/^diff:\s/, '').replace(/%.*/, ''));
+
+    if (Number.isNaN(imageDiff.rawMisMatchPercentage)) {
+      reject('Unable to compare images');
+    }
+
+    // Only create a diff image if necessary.
+    if (isInteractive || imageDiff.rawMisMatchPercentage > 0) {
+      imageDiff.diffPath = join(path.dirname(generated), `${path.basename(generated, '.png')}-diff.png`);
+      await exec(`${imageDiffPath} --diff ${golden} ${generated} ${imageDiff.diffPath}`);
+    }
+
+    resolve(imageDiff);
+  });
+}
+
+async function exec(cmd: string) {
+  return new Promise<string>((resolve, reject) => {
+    let commandOutput = '';
+    try {
+      commandOutput = childProcess.execSync(cmd, {encoding: 'utf8'});
+      resolve(commandOutput);
+    } catch (e) {
+      // image_diff will exit with a status code of 1 if the diff is too big, so
+      // this needs to be caught, but the outcome is the same - we want to send
+      // back the string for processing.
+      if (e.stdout.indexOf('diff') === -1) {
+        reject(e.stdout);
         return;
       }
 
-      const {dimensionDifference, rawMisMatchPercentage} = data;
-      assert.deepEqual(dimensionDifference, { width: 0, height: 0});
-      assert.isBelow(rawMisMatchPercentage, 1);
-      resolve();
-    });
-    resolve();
+      resolve(e.stdout);
+    }
   });
-};
+}
+
+async function compare(golden: string, generated: string, fileName: string) {
+  const {screenshot} = getBrowserAndPages();
+  if (screenshot) {
+    await screenshot.evaluate(opts => {
+      (self as any).setState(opts);
+    }, {type: 'status', msg: `Comparing ${fileName} to generated image...`});
+  }
+
+  const isInteractive = typeof screenshot !== 'undefined';
+  const {rawMisMatchPercentage, diffPath} = await imageDiff(golden, generated, isInteractive);
+
+  // Interactively allow the user to choose.
+  if (screenshot && diffPath) {
+    const root = join(__dirname, '..', '..');
+    const left = golden.replace(root, '');
+    const right = generated.replace(root, '');
+    const diff = diffPath.replace(root, '');
+    const type = 'outcome';
+
+    await screenshot.evaluate(opts => {
+      (self as any).setState(opts);
+    }, {type, left, right, diff, rawMisMatchPercentage, fileName});
+
+    const elementHandle = await screenshot.waitForSelector('.choice', {timeout: 0});
+    const choice = await elementHandle.evaluate(node => node.getAttribute('data-choice'));
+
+    // If they choose the test output, copy the generated screenshot over the golden.
+    if (choice === 'generated') {
+      fs.copyFileSync(generated, golden);
+    }
+  } else {  // Assert no change.
+    assert.isBelow(rawMisMatchPercentage, 1, `There is a ${rawMisMatchPercentage}% difference`);
+  }
+}