Support navigating iframes in accessibility tree

This change also introduces e2e test suites for both the new
accessibility tree and the existing accessibility pane.
The fuller accessibility tree now listens for documentUpdated events
to refresh the root nodes. This is a temporary work-around to fix a bug
where the tree would be empty if we attempt to populate it before the
DOM is ready. More fine-grained updates of the tree will be tackled in
a follow-up.

Bug: 1226486
Change-Id: I61edd34724da192225b31348bf9300df212fb11a
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/3043603
Commit-Queue: Johan Bay <jobay@chromium.org>
Reviewed-by: Alex Rudenko <alexrudenko@chromium.org>

front_end/core/sdk/AccessibilityModel.ts

@@ -180,7 +180,7 @@ export class AccessibilityNode {
       return false;
     }
 
-    return !this._childIds.some(id => this._accessibilityModel.axNodeForId(id) !== null);
+    return this._childIds.every(id => this._accessibilityModel.axNodeForId(id) === null);
   }
 }
 

front_end/panels/elements/AccessibilityTreeUtils.ts

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import type * as SDK from '../../core/sdk/sdk.js';
+import * as SDK from '../../core/sdk/sdk.js';
 import type * as TreeOutline from '../../ui/components/tree_outline/tree_outline.js';
 import * as ElementsComponents from './components/components.js';
 import * as LitHtml from '../../ui/lit-html/lit-html.js';
@@ -12,7 +12,8 @@ export type AXTreeNode = TreeOutline.TreeOutlineUtils.TreeNode<AXTreeNodeData>;
 
 export function sdkNodeToAXTreeNode(sdkNode: SDK.AccessibilityModel.AccessibilityNode): AXTreeNode {
   const treeNodeData = sdkNode;
-  if (!sdkNode.numChildren()) {
+  const role = sdkNode.role()?.value;
+  if (!sdkNode.numChildren() && role !== 'Iframe') {
     return {
       treeNodeData,
       id: sdkNode.id(),
@@ -22,8 +23,26 @@ export function sdkNodeToAXTreeNode(sdkNode: SDK.AccessibilityModel.Accessibilit
   return {
     treeNodeData,
     children: async(): Promise<AXTreeNode[]> => {
+      const domNode = await sdkNode.deferredDOMNode()?.resolvePromise();
+      const document = domNode?.contentDocument();
+      if (document) {
+        const axmodel = document.domModel().target().model(SDK.AccessibilityModel.AccessibilityModel);
+        if (!axmodel) {
+          throw new Error('Could not create AccessibilityModel for iframe');
+        }
+        // Check if we have requested the node before:
+        if (!axmodel.axNodeForDOMNode(document)) {
+          // Request root node from backend and add to model
+          await axmodel.requestPartialAXTree(document);
+        }
+        const localRoot = axmodel.axNodeForDOMNode(document);
+        if (!localRoot) {
+          throw new Error('Could not find root node');
+        }
+        return [sdkNodeToAXTreeNode(localRoot)];
+      }
       if (sdkNode.numChildren() === sdkNode.children().length) {
-        return Promise.resolve(sdkNode.children().map(child => sdkNodeToAXTreeNode(child)));
+        return sdkNode.children().map(child => sdkNodeToAXTreeNode(child));
       }
       // numChildren returns the number of children that this node has, whereas node.children()
       // returns only children that have been loaded. If these two don't match, that means that
@@ -40,7 +59,7 @@ export function sdkNodeToAXTreeNode(sdkNode: SDK.AccessibilityModel.Accessibilit
         treeNodeChildren.push(sdkNodeToAXTreeNode(child));
       }
 
-      return Promise.resolve(treeNodeChildren);
+      return treeNodeChildren;
     },
     id: sdkNode.id(),
   };

front_end/panels/elements/AccessibilityTreeView.ts

@@ -7,7 +7,6 @@ import * as SDK from '../../core/sdk/sdk.js';
 import * as TreeOutline from '../../ui/components/tree_outline/tree_outline.js';
 import * as UI from '../../ui/legacy/legacy.js';
 import * as AccessibilityTreeUtils from './AccessibilityTreeUtils.js';
-import type * as LitHtml from '../../ui/lit-html/lit-html.js';
 
 export class AccessibilityTreeView extends UI.Widget.VBox {
   private readonly accessibilityTreeComponent =
@@ -63,20 +62,35 @@ export class AccessibilityTreeView extends UI.Widget.VBox {
   wasShown(): void {
     if (this.selectedTreeNode) {
       this.accessibilityTreeComponent.expandToAndSelectTreeNode(this.selectedTreeNode);
+    } else {
+      // FIXME(jobay): enable once we have a stable way of expanding across reloads
+      // this.accessibilityTreeComponent.expandRecursively(2);
+      this.selectedTreeNode = this.treeData[0];
     }
   }
 
   setAccessibilityModel(model: SDK.AccessibilityModel.AccessibilityModel|null): void {
     this.accessibilityModel = model;
-    this.refreshAccessibilityTree();
   }
 
-  async refreshAccessibilityTree(): Promise<void> {
-    if (!this.accessibilityModel) {
-      return;
-    }
+  wireToDOMModel(domModel: SDK.DOMModel.DOMModel): void {
+    domModel.addEventListener(SDK.DOMModel.Events.DocumentUpdated, this.documentUpdated, this);
+  }
 
-    const root = await this.accessibilityModel.requestRootNode();
+  unwireFromDOMModel(domModel: SDK.DOMModel.DOMModel): void {
+    domModel.removeEventListener(SDK.DOMModel.Events.DocumentUpdated, this.documentUpdated, this);
+  }
+
+  documentUpdated(event: Common.EventTarget.EventTargetEvent): void {
+    const domModel = (event.data as SDK.DOMModel.DOMModel);
+    const axModel = domModel.target().model(SDK.AccessibilityModel.AccessibilityModel);
+    if (domModel.existingDocument() && !domModel.parentModel() && axModel) {
+      this.refreshAccessibilityTree(axModel);
+    }
+  }
+
+  async refreshAccessibilityTree(accessibilityModel: SDK.AccessibilityModel.AccessibilityModel): Promise<void> {
+    const root = await accessibilityModel.requestRootNode();
     if (!root) {
       return;
     }
@@ -85,12 +99,9 @@ export class AccessibilityTreeView extends UI.Widget.VBox {
     this.treeData = [AccessibilityTreeUtils.sdkNodeToAXTreeNode(this.rootAXNode)];
 
     this.accessibilityTreeComponent.data = {
-      defaultRenderer: (node): LitHtml.TemplateResult => AccessibilityTreeUtils.accessibilityNodeRenderer(node),
+      defaultRenderer: AccessibilityTreeUtils.accessibilityNodeRenderer,
       tree: this.treeData,
     };
-
-    this.accessibilityTreeComponent.expandRecursively(2);
-    this.selectedTreeNode = this.treeData[0];
   }
 
   // Given a selected DOM node, asks the model to load the missing subtree from the root to the
@@ -120,7 +131,7 @@ export class AccessibilityTreeView extends UI.Widget.VBox {
     }
 
     this.accessibilityTreeComponent.data = {
-      defaultRenderer: (node): LitHtml.TemplateResult => AccessibilityTreeUtils.accessibilityNodeRenderer(node),
+      defaultRenderer: AccessibilityTreeUtils.accessibilityNodeRenderer,
       tree: this.treeData,
     };
 

front_end/panels/elements/ElementsPanel.ts

@@ -314,9 +314,8 @@ export class ElementsPanel extends UI.Panel.Panel implements UI.SearchableView.S
 
     // Different frames will have different DOMModels, we only want to add the accessibility model
     // for the top level frame, as the accessibility tree does not yet support exploring IFrames.
-    if (!parentModel && this._accessibilityTreeView) {
-      this._accessibilityTreeView.setAccessibilityModel(
-          domModel.target().model(SDK.AccessibilityModel.AccessibilityModel));
+    if (this._accessibilityTreeView) {
+      this._accessibilityTreeView.wireToDOMModel(domModel);
     }
     let treeOutline: ElementsTreeOutline|null = parentModel ? ElementsTreeOutline.forDOMModel(parentModel) : null;
     if (!treeOutline) {

test/e2e/elements/BUILD.gn

@@ -6,6 +6,8 @@ import("../../../third_party/typescript/typescript.gni")
 
 node_ts_library("elements") {
   sources = [
+    "accessibility-pane_test.ts",
+    "accessibility-tree_test.ts",
     "adornment_test.ts",
     "classes-pane_test.ts",
     "computed-pane-properties_test.ts",

test/e2e/elements/accessibility-pane_test.ts

@@ -0,0 +1,32 @@
+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {click, goToResource, waitForAria, waitForElementWithTextContent} from '../../shared/helper.js';
+import {describe, it} from '../../shared/mocha-extensions.js';
+import {toggleAccessibilityPane} from '../helpers/elements-helpers.js';
+
+describe('Accessibility Pane in the Elements Tab', async function() {
+  it('displays the partial accessibility tree', async () => {
+    await goToResource('elements/accessibility-simple-page.html');
+    await toggleAccessibilityPane();
+    waitForAria('Accessibility Tree');
+  });
+
+  it('shows computed name from contents for title element', async () => {
+    await goToResource('elements/accessibility-simple-page.html');
+    await toggleAccessibilityPane();
+    const titleElement = await waitForAria('<h1>');
+    await click(titleElement);
+    await waitForAria('Contents:\xa0"Title"');
+  });
+
+  it('shows name from label for span element', async () => {
+    await goToResource('elements/accessibility-simple-page.html');
+    await toggleAccessibilityPane();
+    const a11yPane = await waitForAria('Accessibility panel');
+    const spanElement = await waitForElementWithTextContent('span-name');
+    await click(spanElement);
+    await waitForAria('Name:\xa0"span-name"', a11yPane);
+  });
+});

test/e2e/elements/accessibility-tree_test.ts

@@ -0,0 +1,36 @@
+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {click, enableExperiment, goToResource, waitForAria} from '../../shared/helper.js';
+import {describe, it} from '../../shared/mocha-extensions.js';
+import {toggleAccessibilityPane, toggleAccessibilityTree} from '../helpers/elements-helpers.js';
+
+describe('Accessibility Tree in the Elements Tab', async function() {
+  beforeEach(async () => {
+    await enableExperiment('fullAccessibilityTree');
+  });
+
+  it('displays the fuller accessibility tree', async () => {
+    await goToResource('elements/accessibility-simple-page.html');
+    await toggleAccessibilityPane();
+    await toggleAccessibilityTree();
+    const root = await waitForAria('RootWebArea\xa0"Simple page with aria labeled element" [role="treeitem"]');
+    await click('.arrow-icon', {root: root});
+    await waitForAria('heading\xa0"Title" [role="treeitem"]');
+    await waitForAria('link\xa0"cats" [role="treeitem"]');
+  });
+
+  it('allows navigating iframes', async () => {
+    await goToResource('elements/accessibility-iframe-page.html');
+    await toggleAccessibilityTree();
+    const root = await waitForAria('RootWebArea\xa0"Page with nested iframe" [role="treeitem"]');
+    await click('.arrow-icon', {root: root});
+    const iframe = await waitForAria('Iframe\xa0"" [role="treeitem"]');
+    await click('.arrow-icon', {root: iframe});
+    const iframeDoc = await waitForAria('RootWebArea\xa0"Simple page with aria labeled element" [role="treeitem"]');
+    await click('.arrow-icon', {root: iframeDoc});
+
+    await waitForAria('link\xa0"cats" [role="treeitem"]');
+  });
+});

test/e2e/helpers/elements-helpers.ts

@@ -5,7 +5,7 @@ import {assert} from 'chai';
 import {performance} from 'perf_hooks';
 import type * as puppeteer from 'puppeteer';
 
-import {$, $$, click, getBrowserAndPages, step, timeout, waitFor, waitForFunction} from '../../shared/helper.js';
+import {$, $$, click, getBrowserAndPages, step, timeout, waitFor, waitForAria, waitForFunction} from '../../shared/helper.js';
 
 const SELECTED_TREE_ELEMENT_SELECTOR = '.selected[role="treeitem"]';
 const CSS_PROPERTY_NAME_SELECTOR = '.webkit-css-property';
@@ -608,3 +608,19 @@ export const assertSelectedNodeClasses = async (expectedClasses: string[]) => {
     assert.include(classes, expectedClass, `Could not find class ${expectedClass} on the element`);
   }
 };
+
+export const toggleAccessibilityPane = async () => {
+  let a11yPane = await $('Accessibility', undefined, 'aria');
+  if (!a11yPane) {
+    const elementsPanel = await waitForAria('Elements panel');
+    const moreTabs = await waitForAria('More tabs', elementsPanel);
+    await click(moreTabs);
+    a11yPane = await waitForAria('Accessibility');
+  }
+  await click(a11yPane);
+};
+
+export const toggleAccessibilityTree = async () => {
+  const treeToggleButton = await waitForAria('Switch to Accessibility Tree view');
+  await click(treeToggleButton);
+};

test/e2e/resources/elements/BUILD.gn

@@ -6,6 +6,8 @@ import("../../../../scripts/build/ninja/copy.gni")
 
 copy_to_gen("elements") {
   sources = [
+    "accessibility-iframe-page.html",
+    "accessibility-simple-page.html",
     "adornment-scroll-snap.html",
     "adornment-shadow.html",
     "adornment.html",

test/e2e/resources/elements/accessibility-iframe-page.html

@@ -0,0 +1,9 @@
+<!--
+  Copyright 2021 The Chromium Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file.
+-->
+<!DOCTYPE html>
+<title>Page with nested iframe</title>
+<h1>Page with an iframe</h1>
+<iframe src="accessibility-simple-page.html" name="simple-page"></iframe>

test/e2e/resources/elements/accessibility-simple-page.html

@@ -0,0 +1,10 @@
+<!--
+  Copyright 2021 The Chromium Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file.
+-->
+<!DOCTYPE html>
+<title>Simple page with aria labeled element</title>
+<h1>Title</h1>
+<span aria-label="span-name"></span>
+<a href="x">cats</a>