Reorganize COEP + reporting tests

The COEP reporting feature has two observable effects. User agents send
reports to the network, and web developers can observe reports via
ReportingObserver. We need to have *some* tests for each way, but we
don't need to have all the test cases for both ways.

As the ReportingObserver way is faster, easier to debug and less flaky,
we would like to test more cases with ReportingObserver. This CL moves
many test cases from reporting.https.html to
reporting-navigation.https.html and
reporting-subresource-corp.https.html, and rename reporting.https.html
to reporting-to-endpoint.https.html. This CL also removes many
duplicated tests (see reporting-subresource-corp.https.html).

Bug: 1052764
Change-Id: I37a567d8cbb6aec79a8426f2f32630c70e4da112
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2208154
Commit-Queue: Yutaka Hirano <yhirano@chromium.org>
Reviewed-by: Makoto Shimazu <shimazu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#770058}

third_party/blink/web_tests/WPTOverrideExpectations

@@ -575,7 +575,7 @@ external/wpt/html/browsers/the-window-object/apis-for-creating-and-navigating-br
 external/wpt/html/browsers/windows/noreferrer-window-name.html [ Timeout ] # wpt_subtest_failure
 external/wpt/html/cross-origin-embedder-policy/coep-frame-javascript.https.html [ Pass Timeout ] # wpt_subtest_failure
 external/wpt/html/cross-origin-embedder-policy/javascript.https.html [ Pass Timeout ] # wpt_subtest_failure
-external/wpt/html/cross-origin-embedder-policy/reporting.https.html [ Pass Failure ] # wpt_subtest_failure
+external/wpt/html/cross-origin-embedder-policy/reporting-to-endpoint.https.html [ Pass Failure ] # wpt_subtest_failure
 external/wpt/html/cross-origin-embedder-policy/require-corp.https.html [ Pass ] # wpt_subtest_failure
 external/wpt/html/cross-origin-opener-policy/blob-popup.https.html [ Pass Failure ]
 external/wpt/html/cross-origin-opener-policy/coep-blob-popup.https.html [ Timeout ] # wpt_subtest_failure

third_party/blink/web_tests/android/ClankWPTOverrideExpectations

@@ -1996,7 +1996,7 @@ crbug.com/1050754 external/wpt/html/browsers/windows/embedded-opener-remove-fram
 crbug.com/1050754 external/wpt/html/browsers/windows/embedded-opener.html [ Failure ]
 crbug.com/1050754 external/wpt/html/browsers/windows/noreferrer-window-name.html [ Timeout ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/cache-storage-reporting.https.html [ Failure ]
-crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/reporting.https.html [ Failure ]
+crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/reporting-to-endpoint.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/require-corp.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-opener-policy/blob-popup.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-opener-policy/coep-blob-popup.https.html [ Timeout ]

third_party/blink/web_tests/android/WeblayerWPTOverrideExpectations

@@ -1637,7 +1637,7 @@ crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/data.https.html
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/dedicated-worker-cache-storage.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/javascript.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/none-sw-from-require-corp.https.html [ Failure ]
-crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/reporting.https.html [ Failure ]
+crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/reporting-to-endpoint.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/require-corp-load-from-cache-storage.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/require-corp-sw-from-none.https.html [ Failure ]
 crbug.com/1050754 external/wpt/html/cross-origin-embedder-policy/require-corp-sw-from-require-corp.https.html [ Failure ]

third_party/blink/web_tests/android/WebviewWPTOverrideExpectations

@@ -2074,7 +2074,7 @@ external/wpt/html/cross-origin-embedder-policy/javascript.https.html [ Failure ]
 external/wpt/html/cross-origin-embedder-policy/none-sw-from-require-corp.https.html [ Failure ]
 external/wpt/html/cross-origin-embedder-policy/none.https.html [ Timeout ]
 external/wpt/html/cross-origin-embedder-policy/report-only-require-corp.https.html [ Timeout ]
-external/wpt/html/cross-origin-embedder-policy/reporting.https.html [ Failure ]
+external/wpt/html/cross-origin-embedder-policy/reporting-to-endpoint.https.html [ Failure ]
 external/wpt/html/cross-origin-embedder-policy/require-corp-load-from-cache-storage.https.html [ Failure ]
 external/wpt/html/cross-origin-embedder-policy/require-corp-sw-from-none.https.html [ Failure ]
 external/wpt/html/cross-origin-embedder-policy/require-corp-sw-from-require-corp.https.html [ Failure ]

third_party/blink/web_tests/external/wpt/html/cross-origin-embedder-policy/reporting-navigation.https.html

@@ -6,9 +6,11 @@
 <script src="/common/get-host-info.sub.js"></script>
 <script>
 const {ORIGIN, REMOTE_ORIGIN} = get_host_info();
-const COEP =
-  '|header(cross-origin-embedder-policy,require-corp)' +
+const COEP = '|header(cross-origin-embedder-policy,require-corp)';
+const COEP_RO =
   '|header(cross-origin-embedder-policy-report-only,require-corp)';
+const CORP_CROSS_ORIGIN =
+  '|header(cross-origin-resource-policy,cross-origin)';
 const FRAME_URL = `${ORIGIN}/common/blank.html?pipe=`;
 const REMOTE_FRAME_URL = `${REMOTE_ORIGIN}/common/blank.html?pipe=`;
 
@@ -19,6 +21,13 @@ function checkCorpReport(report, contextUrl, blockedUrl) {
   assert_equals(report.body['blocked-url'], blockedUrl);
 }
 
+function checkCoepMismatchReport(report, contextUrl, blockedUrl) {
+  assert_equals(report.type, 'coep');
+  assert_equals(report.url, contextUrl);
+  assert_equals(report.body.type, 'navigation');
+  assert_equals(report.body['blocked-url'], blockedUrl);
+}
+
 function loadFrame(document, url) {
   return new Promise((resolve, reject) => {
     const frame = document.createElement('iframe');
@@ -29,49 +38,111 @@ function loadFrame(document, url) {
   });
 }
 
-function observeReportsUpTo(global, upto) {
+// |parentSuffix| is a suffix for the parent frame URL.
+// When |withEmptyFrame| is true, this function creates an empty frame
+// between the parent and target frames.
+// |targetUrl| is a URL for the target frame.
+async function loadFrames(test, parentSuffix, withEmptyFrame, targetUrl) {
+  const frame = await loadFrame(document, FRAME_URL + parentSuffix);
+  test.add_cleanup(() => frame.remove());
+  let parent;
+  if (withEmptyFrame) {
+    parent = frame.contentDocument.createElement('iframe');
+    frame.contentDocument.body.appendChild(parent);
+  } else {
+    parent = frame;
+  }
+  // Here we don't need "await". This loading may or may not succeed, and
+  // we're not interested in the result.
+  loadFrame(parent.contentDocument, targetUrl);
+
+  return parent;
+}
+
+async function observeReports(global) {
   const reports = [];
-  return new Promise(resolve => {
-    const observer = new global.ReportingObserver((rs) => {
-      for (const r of rs) {
-        reports.push(r.toJSON());
-        if (reports.length == upto) {
-          observer.disconnect();
-          resolve(reports);
-        }
-      }
-    });
-    observer.observe();
+  const observer = new global.ReportingObserver((rs) => {
+    for (const r of rs) {
+      reports.push(r.toJSON());
+    }
   });
+  observer.observe();
+
+  // Wait 1500ms for reports to settle.
+  await new Promise(r => step_timeout(r, 1500));
+  return reports;
 }
 
-promise_test(async (t) => {
-  const parent = await loadFrame(document, FRAME_URL + COEP);
-  t.add_cleanup(() => parent.remove());
-  loadFrame(parent.contentDocument, REMOTE_FRAME_URL + COEP);
-
-  // One for COEP, one for COEP-RO.
-  const reports = await observeReportsUpTo(parent.contentWindow, 2);
-
-  assert_equals(reports.length, 2);
-  checkCorpReport(reports[0], parent.src, REMOTE_FRAME_URL + COEP);
-  checkCorpReport(reports[1], parent.src, REMOTE_FRAME_URL + COEP);
-}, 'Navigation CORP');
-
-promise_test(async (t) => {
-  const grandParent = await loadFrame(document, FRAME_URL + COEP);
-  t.add_cleanup(() => grandParent.remove());
-  const parent = grandParent.contentDocument.createElement('iframe');
-  grandParent.contentDocument.body.appendChild(parent);
-  loadFrame(parent.contentDocument, REMOTE_FRAME_URL + COEP);
-
-  // One for COEP, one for COEP-RO.
-  const reports = await observeReportsUpTo(parent.contentWindow, 2);
-
-  assert_equals(reports.length, 2);
-  checkCorpReport(reports[0], 'about:blank', REMOTE_FRAME_URL + COEP);
-  checkCorpReport(reports[1], 'about:blank', REMOTE_FRAME_URL + COEP);
-}, 'Navigation CORP and about:blank');
+// CASES is a list of test case. Each test case consists of:
+//   parent: the suffix of the URL of the parent frame.
+//   target: the suffix of the URL of the target frame.
+//   reports: the expectation of reports to be made. Each report is one of:
+//     'CORP': CORP violation
+//     'CORP-RO' CORP violation (report only)
+//     'NAV': COEP mismatch between the frames.
+//     'NAV-RO': COEP mismatch between the frames (report only).
+//   Currently '-RO' is no-op, e.g., 'CORP' and 'CORP-RO' have the same
+//   expectation. We are planning to introduce "disposition" member in
+//   reports, which will differentiate them each other.
+const CASES = [
+  { parent: '', target: '', reports: [] },
+  { parent: '', target: COEP, reports: [] },
+  { parent: COEP, target: COEP, reports: ['CORP'] },
+  { parent: COEP, target: '', reports: ['CORP'] },
+
+  { parent: '', target: CORP_CROSS_ORIGIN, reports: [] },
+  { parent: COEP, target: CORP_CROSS_ORIGIN, reports: ['NAV'] },
+
+  { parent: '', target: COEP + CORP_CROSS_ORIGIN, reports: [] },
+  { parent: COEP, target: COEP + CORP_CROSS_ORIGIN, reports: [] },
+
+  { parent: COEP_RO, target: COEP, reports: ['CORP-RO'] },
+  { parent: COEP_RO, target: '', reports: ['CORP-RO', 'NAV-RO'] },
+  { parent: COEP_RO, target: CORP_CROSS_ORIGIN, reports: ['NAV-RO'] },
+  { parent: COEP_RO, target: COEP + CORP_CROSS_ORIGIN, reports: [] },
+
+  { parent: COEP, target: COEP_RO + CORP_CROSS_ORIGIN, reports: ['NAV'] },
+];
+
+for (const testcase of CASES) {
+  for (const withEmptyFrame of [false, true]) {
+    function desc(s) {
+      return s === '' ? '(none)' : s;
+    }
+    async_test(async (t) => {
+      try {
+        const targetUrl = REMOTE_FRAME_URL + testcase.target;
+        const parent =
+          await loadFrames(t, testcase.parent, withEmptyFrame, targetUrl);
+
+        const contextUrl = parent.src ? parent.src : 'about:blank';
+        const reports = await observeReports(parent.contentWindow);
+        assert_equals(reports.length, testcase.reports.length);
+
+        for (let i = 0; i < reports.length; i += 1) {
+          const report = reports[i];
+          switch (testcase.reports[i]) {
+            case 'CORP':
+            case 'CORP-RO':
+              checkCorpReport(report, contextUrl, targetUrl);
+              break;
+            case 'NAV':
+            case 'NAV-RO':
+              checkCoepMismatchReport(report, contextUrl, targetUrl);
+              break;
+            default:
+              assert_unreached(
+                'Unexpected report expeaction: ' + testcase.reports[i]);
+          }
+        }
+        t.done();
+      } catch (e) {
+        t.step(() => { throw e; });
+      }
+    }, `parent: ${desc(testcase.parent)}, target: ${desc(testcase.target)}, ` +
+       `with empty frame: ${withEmptyFrame}`);
+  }
+}
 
 </script>
 </body></html>
\ No newline at end of file

third_party/blink/web_tests/external/wpt/html/cross-origin-embedder-policy/reporting-subresource-corp.https-expected.txt

@@ -15,5 +15,9 @@ PASS [service worker] same-origin
 PASS [service worker] blocked by CORP: same-origin
 PASS [service worker] blocked due to COEP
 PASS [service worker] blocked during redirect
+PASS [between service worker and page] same-origin
+PASS [between service worker and page] blocked by CORP: same-origin
+PASS [between service worker and page] blocked due to COEP
+PASS [between service worker and page] blocked during redirect
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/html/cross-origin-embedder-policy/reporting-subresource-corp.https.html

@@ -28,9 +28,9 @@ function checkReport(report, contextUrl, blockedUrl) {
   assert_equals(report.body['blocked-url'], blockedUrl);
 }
 
-async function fetchInFrame(t, url) {
+async function fetchInFrame(t, frameUrl, url) {
   const reports = [];
-  const frame = await with_iframe(FRAME_URL);
+  const frame = await with_iframe(frameUrl);
   t.add_cleanup(() => frame.remove());
 
   const observer = new frame.contentWindow.ReportingObserver((rs) => {
@@ -80,7 +80,7 @@ const ENVIRONMENTS = [{
   tag: 'document',
   contextUrl: FRAME_URL,
   run: async (test, url) => {
-    return await fetchInFrame(test, url);
+    return await fetchInFrame(test, FRAME_URL, url);
   },
 }, {
   tag: 'dedicated worker',
@@ -113,6 +113,21 @@ const ENVIRONMENTS = [{
     worker.addEventListener('error', test.unreached_func('Worker.onerror'));
     return await fetchInWorker(worker, url);
   },
+}, {
+  tag: 'between service worker and page',
+  contextUrl: `${ORIGIN}${BASE}/reporting-empty-frame.html`,
+  run: async (test, url) => {
+    const SCOPE = `${BASE}/reporting-empty-frame.html`;
+    // Here we use a Service Worker without COEP.
+    const WORKER_URL = `${ORIGIN}${BASE}/sw.js`;
+    const reg =
+      await service_worker_unregister_and_register(test, WORKER_URL, SCOPE);
+    test.add_cleanup(() => reg.unregister());
+    const worker = reg.installing || reg.waiting || reg.active;
+    worker.addEventListener('error', test.unreached_func('Worker.onerror'));
+    return await fetchInFrame(
+      test, `${ORIGIN}${BASE}/reporting-empty-frame.html`, url);
+  },
 }];
 
 const CASES = [{

third_party/blink/web_tests/external/wpt/html/cross-origin-embedder-policy/reporting-to-endpoint.https.html

+<!doctype html>
+<html>
+<meta name="timeout" content="long">
+<body>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script src="/common/utils.js"></script>
+<script src="/common/get-host-info.sub.js"></script>
+<script>
+// This file consists of tests for COEP reporting. The tests make COEP
+// violations and see whether reports are sent to the network as specified.
+// We only have basic tests in this file - one for each kind of reports,
+// because we can also test the reporting functionality with ReportingObserver,
+// and that way is faster, easier to debug, and less flaky.
+//
+// For more detailed tests and tests with workers, see tests in other files
+// such as
+//  - reporting-navigation.https.html
+//  - reporting-subresource-corp.https.html
+//  - cache-storage-reporting*.https.html
+// .
+const { REMOTE_ORIGIN } = get_host_info();
+const BASE = new URL("resources", location).pathname
+
+function wait(ms) {
+  return new Promise(resolve => step_timeout(resolve, ms));
+}
+
+async function pollReports(endpoint, reports) {
+  while (true) {
+    await wait(200);
+    const res = await fetch(`resources/report.py?endpoint=${endpoint}`, {cache: 'no-store'});
+    if (res.status !== 200) {
+      continue;
+    }
+    for (const report of await res.json()) {
+      reports.push(report);
+    }
+  }
+}
+
+const reports = [];
+const reportsForReportOnly = [];
+pollReports('endpoint', reports);
+pollReports('report-only-endpoint', reportsForReportOnly);
+
+function checkCorpReportExistence(reports, blockedUrl, contextUrl) {
+  blockedUrl = new URL(blockedUrl, location).href;
+  contextUrl = new URL(contextUrl, location).href;
+  for (const report of reports) {
+    if (report.type !== 'coep' || report.url !== contextUrl ||
+        report.body.type !== 'corp') {
+      continue;
+    }
+    if (report.body['blocked-url'] === blockedUrl) {
+      return;
+    }
+  }
+  assert_unreached(`A report whose blocked-url is ${blockedUrl} and url is ${contextUrl} is not found.`);
+}
+
+function checkNavigationReportExistence(reports, blockedUrl, contextUrl) {
+  blockedUrl = new URL(blockedUrl, location).href;
+  contextUrl = new URL(contextUrl, location).href;
+  for (const report of reports) {
+    if (report.type !== 'coep' || report.url !== contextUrl ||
+        report.body.type !== 'navigation') {
+      continue;
+    }
+    if (report.body['blocked-url'] === blockedUrl) {
+      return;
+    }
+  }
+  assert_unreached(`A report whose blocked-url is ${blockedUrl} and url is ${contextUrl} is not found.`);
+}
+
+async_test(async (t) => {
+  try {
+    const iframe = document.createElement('iframe');
+    t.add_cleanup(() => iframe.remove());
+
+    iframe.src = `resources/reporting-empty-frame.html`
+    document.body.appendChild(iframe);
+    await new Promise(resolve => {
+      iframe.addEventListener('load', resolve, {once: true});
+    });
+
+    const url = `${REMOTE_ORIGIN}/common/text-plain.txt?${token()}`;
+    const init = { mode: 'no-cors', cache: 'no-store' };
+    // The response comes from cross-origin, and doesn't have a CORP
+    // header, so it is blocked.
+    iframe.contentWindow.fetch(url, init).catch(() => {});
+
+    // Wait 3 seconds for reports to settle.
+    await wait(3000);
+
+    checkCorpReportExistence(reports, url, iframe.src);
+    checkCorpReportExistence(reportsForReportOnly, url, iframe.src);
+
+    t.done();
+  } catch (e) {
+    t.step(() => { throw e });
+  }
+}, 'subresource CORP');
+
+async_test(async (t) => {
+  try {
+    const iframe = document.createElement('iframe');
+    t.add_cleanup(() => iframe.remove());
+
+    iframe.src = `resources/reporting-empty-frame.html`
+    document.body.appendChild(iframe);
+    await new Promise(resolve => {
+      iframe.addEventListener('load', resolve, {once: true});
+    });
+
+    const w = iframe.contentWindow;
+
+    function attachFrame(url) {
+      const frame = w.document.createElement('iframe');
+      frame.src = url;
+      w.document.body.appendChild(frame);
+    }
+
+    const url = `${REMOTE_ORIGIN}/common/blank.html?${token()}`;
+    // The nested frame comes from cross-origin and doesn't have a CORP
+    // header, so it is blocked.
+    attachFrame(url);
+
+    // Wait 3 seconds for reports to settle.
+    await wait(3000);
+
+    checkCorpReportExistence(reports, url, iframe.src);
+    checkCorpReportExistence(reportsForReportOnly, url, iframe.src);
+
+    t.done();
+  } catch (e) {
+    t.step(() => { throw e });
+  }
+}, 'navigation CORP');
+
+async_test(async (t) => {
+  try {
+    const iframe = document.createElement('iframe');
+    t.add_cleanup(() => iframe.remove());
+
+    iframe.src = 'resources/reporting-empty-frame.html';
+    const targetUrl = `/common/blank.html?${token()}`;
+    iframe.addEventListener('load', () => {
+      const nested = iframe.contentDocument.createElement('iframe');
+      nested.src = targetUrl;
+      // |nested| doesn't have COEP whereas |iframe| has, so it is blocked.
+      iframe.contentDocument.body.appendChild(nested);
+    }, {once: true});
+
+    document.body.appendChild(iframe);
+
+    // Wait 3 seconds for reports to settle.
+    await wait(3000);
+
+    checkNavigationReportExistence(reports, targetUrl, iframe.src);
+    checkNavigationReportExistence(
+      reportsForReportOnly, targetUrl, iframe.src);
+
+    t.done();
+  } catch (e) {
+    t.step(() => { throw e });
+  }
+}, 'COEP violation on nested frame navigation');
+
+</script>$

third_party/blink/web_tests/external/wpt/html/cross-origin-embedder-policy/reporting.https-expected.txt

-This is a testharness.js-based test.
-PASS subresource CORP
-PASS CORP for subresource requests initiated from a service worker
-PASS navigation CORP
-PASS COEP violation on nested frame navigation
-FAIL subresource requests initiated from DedicatedWorker assert_unreached: A report whose blocked-url is https://www1.web-platform.test:8444/common/text-plain.txt?abc&subresource-corp-from-dedicated-worker and url is https://web-platform.test:8444/html/cross-origin-embedder-policy/resources/fetch-in-dedicated-worker.js is not found. Reached unreachable code
-FAIL subresource requests initiated from DedicatedWorker controlled by a passthrough service worker assert_unreached: A report whose blocked-url is https://www1.web-platform.test:8444/common/text-plain.txt?abc&subresource-corp-from-dedicated-worker-via-passthrough-sw and url is https://web-platform.test:8444/html/cross-origin-embedder-policy/resources/fetch-in-dedicated-worker.js is not found. Reached unreachable code
-PASS subresource CORP in an iframe hosted by a service worker without COEP
-Harness: the test ran to completion.
-