CCA enables multiple streams

Chrome camera app enables multiple streams while recording.

      also pass type checking

Bug: b:151047537
Test: Manually test. preview is 640x360 and recording is 1280x720.
Change-Id: I8171b0061a94ca8653c044857da2598eece43e5f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2409683
Commit-Queue: Heng-ruey Hsu <henryhsu@chromium.org>
Auto-Submit: Heng-ruey Hsu <henryhsu@chromium.org>
Reviewed-by: Inker Kuo <inker@chromium.org>
Cr-Commit-Position: refs/heads/master@{#844700}

chromeos/components/camera_app_ui/resources/BUILD.gn

@@ -121,6 +121,7 @@ copy("chrome_camera_app_js_device") {
     "js/device/camera3_device_info.js",
     "js/device/constraints_preferrer.js",
     "js/device/device_info_updater.js",
+    "js/device/stream_manager.js",
   ]
 
   outputs = [ "$out_camera_app_dir/js/device/{{source_file_part}}" ]

chromeos/components/camera_app_ui/resources/camera_app_resources.grd

@@ -33,6 +33,7 @@
       <structure name="IDR_CAMERA_COMLINK_JS" file="js/lib/comlink.js" type="chrome_html" />
       <structure name="IDR_CAMERA_CONSTRAINTS_PREFERRER_JS" file="js/device/constraints_preferrer.js" type="chrome_html" />
       <structure name="IDR_CAMERA_DEVICE_INFO_UPDATER_JS" file="js/device/device_info_updater.js" type="chrome_html" />
+      <structure name="IDR_CAMERA_STREAM_MANAGER_JS" file="js/device/stream_manager.js" type="chrome_html" />
       <structure name="IDR_CAMERA_DEVICE_OPERATOR_JS" file="js/mojo/device_operator.js" type="chrome_html" />
       <structure name="IDR_CAMERA_DIALOG_JS" file="js/views/dialog.js" type="chrome_html" />
       <structure name="IDR_CAMERA_DOM_JS" file="js/dom.js" type="chrome_html" />

chromeos/components/camera_app_ui/resources/js/BUILD.gn

@@ -63,6 +63,7 @@ js_library("compile_resources") {
     "device/camera3_device_info.js",
     "device/constraints_preferrer.js",
     "device/device_info_updater.js",
+    "device/stream_manager.js",
     "dom.js",
     "error.js",
     "gallerybutton.js",

chromeos/components/camera_app_ui/resources/js/device/constraints_preferrer.js

@@ -171,6 +171,118 @@ export class ConstraintsPreferrer {
   setPreferredResolutionChangeListener(listener) {
     this.preferredResolutionChangeListener_ = listener;
   }
+
+  /**
+   * Sorts the preview resolution (Rp) according to the capture resolution
+   * (Rc) and the screen size (Rs) with the following orders:
+   * If |Rc| <= |Rs|:
+   *   1. All |Rp| <= |Rc|, and the larger, the better.
+   *   2. All |Rp| > |Rc|, and the smaller, the better.
+   *
+   * If |Rc| > |Rs|:
+   *   1. All |Rp| where |Rs| <= |Rp| <= |Rc|, and the smaller, the
+   *   better.
+   *   2. All |Rp| < |Rs|, and the larger, the better.
+   *   3. All |Rp| > |Rc|, and the smaller, the better.
+   *
+   * @param {!ResolutionList} previewResolutions
+   * @param {!Resolution} captureResolution
+   * @return {!ResolutionList}
+   * @protected
+   */
+  sortPreview_(previewResolutions, captureResolution) {
+    if (previewResolutions.length === 0) {
+      return [];
+    }
+
+    const Rs = Math.floor(window.screen.width * window.devicePixelRatio);
+    const Rc = captureResolution.width;
+    const cmpDescending = (r1, r2) => r2.width - r1.width;
+    const cmpAscending = (r1, r2) => r1.width - r2.width;
+
+    if (Rc <= Rs) {
+      const notLargerThanR =
+          previewResolutions.filter((r) => r.width <= Rc).sort(cmpDescending);
+      const largerThanR =
+          previewResolutions.filter((r) => r.width > Rc).sort(cmpAscending);
+      return notLargerThanR.concat(largerThanR);
+    } else {
+      const betweenRsR =
+          previewResolutions.filter((r) => Rs <= r.width && r.width <= Rc)
+              .sort(cmpAscending);
+      const smallerThanRs =
+          previewResolutions.filter((r) => r.width < Rs).sort(cmpDescending);
+      const largerThanR =
+          previewResolutions.filter((r) => r.width > Rc).sort(cmpAscending);
+      return betweenRsR.concat(smallerThanRs).concat(largerThanR);
+    }
+  }
+
+  /**
+   * Sorts prefer resolutions.
+   * @param {!Resolution} prefR Preferred resolution.
+   * @return {function(!CaptureCandidate, !CaptureCandidate): number} Return
+   *     compare function for comparing based on preferred resolution.
+   * @protected
+   */
+  getPreferResolutionSort_(prefR) {
+    /*
+     * @param {!CaptureCandidate} candidate
+     * @param {!CaptureCandidate} candidate2
+     * @return {number}
+     */
+    return ({resolution: r1}, {resolution: r2}) => {
+      if (r1.equals(r2)) {
+        return 0;
+      }
+      // Exactly the preferred resolution.
+      if (r1.equals(prefR)) {
+        return -1;
+      }
+      if (r2.equals(prefR)) {
+        return 1;
+      }
+
+      // Aspect ratio same as preferred resolution.
+      if (!r1.aspectRatioEquals(r2)) {
+        if (r1.aspectRatioEquals(prefR)) {
+          return -1;
+        }
+        if (r2.aspectRatioEquals(prefR)) {
+          return 1;
+        }
+      }
+      return r2.area - r1.area;
+    };
+  }
+
+  /**
+   * Groups resolutions with same ratio into same list.
+   * @param {!ResolutionList} rs
+   * @return {!Map<number, !ResolutionList>} Ratio as key, all resolutions
+   *     with that ratio as value.
+   */
+  groupResolutionRatio_(rs) {
+    /**
+     * @param {!Resolution} r
+     * @return {number}
+     */
+    const toSupportedPreviewRatio = (r) => {
+      // Special aspect ratio mapping rule, see http://b/147986763.
+      if (r.width === 848 && r.height === 480) {
+        return (new Resolution(16, 9)).aspectRatio;
+      }
+      return r.aspectRatio;
+    };
+
+    const result = new Map();
+    for (const r of rs) {
+      const ratio = toSupportedPreviewRatio(r);
+      result.set(ratio, result.get(ratio) || []);
+      result.get(ratio).push(r);
+    }
+    return result;
+  }
 }
 
 /**
@@ -224,6 +336,15 @@ export class VideoConstraintsPreferrer extends ConstraintsPreferrer {
     this.restoreResolutionPreference_('deviceVideoResolution');
     this.restoreFpsPreference_();
 
+    /**
+     * Maps from device id as key to video and preview resolutions of
+     * same aspect ratio supported by that video device as value.
+     * @type {!Map<string, !Array<{videoRs: !ResolutionList, previewRs:
+     *     !ResolutionList}>>}
+     * @private
+     */
+    this.deviceVideoPreviewResolutionMap_ = new Map();
+
     this.toggleFps_.addEventListener('click', (event) => {
       if (!state.get(state.State.STREAMING) || state.get(state.State.TAKING)) {
         event.preventDefault();
@@ -299,12 +420,33 @@ export class VideoConstraintsPreferrer extends ConstraintsPreferrer {
    * @override
    */
   updateDevicesInfo(devices) {
+    this.deviceVideoPreviewResolutionMap_ = new Map();
     this.supportedResolutions_ = new Map();
     this.constFpsInfo_ = {};
 
     devices.forEach(({deviceId, videoResols, videoMaxFps, fpsRanges}) => {
       this.supportedResolutions_.set(
           deviceId, [...videoResols].sort((r1, r2) => r2.area - r1.area));
+
+      // Filter out preview resolution greater than 1920x1080 and 1600x1200.
+      const previewRatios = this.groupResolutionRatio_(videoResols.filter(
+          ({width, height}) => width <= 1920 && height <= 1200));
+      const videoRatios = this.groupResolutionRatio_(videoResols);
+      /**
+       * @type {!Array<{videoRs: !ResolutionList, previewRs:
+       *     !ResolutionList}>}
+       */
+      const pairedResolutions = [];
+      for (const [ratio, videoRs] of videoRatios) {
+        const previewRs = previewRatios.get(/** @type {number} */ (ratio));
+        if (previewRs === undefined) {
+          continue;
+        }
+        pairedResolutions.push(
+            {videoRs: /** @type {!ResolutionList} */ (videoRs), previewRs});
+      }
+      this.deviceVideoPreviewResolutionMap_.set(deviceId, pairedResolutions);
+
       /**
        * @param {number} width
        * @param {number} height
@@ -361,40 +503,8 @@ export class VideoConstraintsPreferrer extends ConstraintsPreferrer {
    * @override
    */
   getSortedCandidates(deviceId) {
-    // Due to the limitation of MediaStream API, preview stream is used directly
-    // to do video recording.
-
     /** @type {!Resolution} */
     const prefR = this.getPrefResolution(deviceId) || new Resolution(0, -1);
-    /**
-     * @param {!Resolution} r1
-     * @param {!Resolution} r2
-     * @return {number}
-     */
-    const sortPrefResol = (r1, r2) => {
-      if (r1.equals(r2)) {
-        return 0;
-      }
-
-      // Exactly the preferred resolution.
-      if (r1.equals(prefR)) {
-        return -1;
-      }
-      if (r2.equals(prefR)) {
-        return 1;
-      }
-
-      // Aspect ratio same as preferred resolution.
-      if (!r1.aspectRatioEquals(r2)) {
-        if (r1.aspectRatioEquals(prefR)) {
-          return -1;
-        }
-        if (r2.aspectRatioEquals(prefR)) {
-          return 1;
-        }
-      }
-      return r2.area - r1.area;
-    };
 
     /**
      * Maps specified video resolution to object of resolution and all supported
@@ -421,27 +531,37 @@ export class VideoConstraintsPreferrer extends ConstraintsPreferrer {
     };
 
     /**
-     * @param {!Resolution} r
-     * @param {number} fps
-     * @return {!MediaStreamConstraints}
+     * @param {{videoRs: !ResolutionList, previewRs: !ResolutionList}} capture
+     * @return {!Array<!CaptureCandidate>}
      */
-    const toConstraints = ({width, height}, fps) => ({
-      audio: {echoCancellation: false},
-      video: {
-        deviceId: {exact: deviceId},
-        frameRate: fps ? {exact: fps} : {min: 20, ideal: 30},
-        width,
-        height,
-      },
-    });
+    const toVideoCandidate = ({videoRs, previewRs}) => {
+      let /** !Resolution */ videoR = prefR;
+      if (!videoRs.some((r) => r.equals(prefR))) {
+        videoR = videoRs.reduce(
+            (videoR, r) => (r.width > videoR.width ? r : videoR));
+      }
 
-    return [...this.supportedResolutions_.get(deviceId)]
-        .sort(sortPrefResol)
-        .flatMap(getFpses)
-        .map(({r, fps}) => ({
-               resolution: r,
-               previewCandidates: [toConstraints(r, fps)],
-             }));
+      return getFpses(videoR).map(
+          ({r, fps}) => ({
+            resolution: videoR,
+            previewCandidates:
+                this.sortPreview_(previewRs, videoR)
+                    .map(({width, height}) => ({
+                           audio: {echoCancellation: false},
+                           video: {
+                             deviceId: {exact: deviceId},
+                             frameRate: fps ? {exact: fps} :
+                                              {min: 20, ideal: 30},
+                             width,
+                             height,
+                           },
+                         })),
+          }));
+    };
+
+    return this.deviceVideoPreviewResolutionMap_.get(deviceId)
+        .flatMap(toVideoCandidate)
+        .sort(this.getPreferResolutionSort_(prefR));
   }
 }
 
@@ -489,36 +609,8 @@ export class PhotoConstraintsPreferrer extends ConstraintsPreferrer {
     this.supportedResolutions_ = new Map();
 
     devices.forEach(({deviceId, photoResols, videoResols: previewResols}) => {
-      /**
-       * @param {!Resolution} r
-       * @return {number}
-       */
-      const toSupportedPreviewRatio = (r) => {
-        // Special aspect ratio mapping rule, see http://b/147986763.
-        if (r.width === 848 && r.height === 480) {
-          return (new Resolution(16, 9)).aspectRatio;
-        }
-        return r.aspectRatio;
-      };
-
-      /**
-       * Groups resolutions with same ratio into same list.
-       * @param {!ResolutionList} rs
-       * @return {!Map<number, !ResolutionList>} Ratio as key, all resolutions
-       *     with that ratio as value.
-       */
-      const groupResolutionRatio = (rs) => {
-        const result = new Map();
-        for (const r of rs) {
-          const ratio = toSupportedPreviewRatio(r);
-          result.set(ratio, result.get(ratio) || []);
-          result.get(ratio).push(r);
-        }
-        return result;
-      };
-
-      const previewRatios = groupResolutionRatio(previewResols);
-      const captureRatios = groupResolutionRatio(photoResols);
+      const previewRatios = this.groupResolutionRatio_(previewResols);
+      const captureRatios = this.groupResolutionRatio_(photoResols);
       /**
        * @type {!Array<{captureRs: !ResolutionList, previewRs:
        *     !ResolutionList}>}
@@ -569,25 +661,6 @@ export class PhotoConstraintsPreferrer extends ConstraintsPreferrer {
     /** @type {!Resolution} */
     const prefR = this.getPrefResolution(deviceId) || new Resolution(0, -1);
 
-    /**
-     * @param {!CaptureCandidate} candidate
-     * @param {!CaptureCandidate} candidate2
-     * @return {number}
-     */
-    const sortPrefResol = ({resolution: r1}, {resolution: r2}) => {
-      if (r1.equals(r2)) {
-        return 0;
-      }
-      // Exactly the preferred resolution.
-      if (r1.equals(prefR)) {
-        return -1;
-      }
-      if (r2.equals(prefR)) {
-        return 1;
-      }
-      return r2.area - r1.area;
-    };
-
     /**
      * @param {{captureRs: !ResolutionList, previewRs: !ResolutionList}} capture
      * @return {!CaptureCandidate}
@@ -599,64 +672,21 @@ export class PhotoConstraintsPreferrer extends ConstraintsPreferrer {
             (captureR, r) => (r.width > captureR.width ? r : captureR));
       }
 
-      /**
-       * @param {!ResolutionList} resolutions
-       * @return {!ResolutionList}
-       */
-      const sortPreview = (resolutions) => {
-        if (resolutions.length === 0) {
-          return [];
-        }
-
-        // Sorts the preview resolution (Rp) according to the capture resolution
-        // (Rc) and the screen size (Rs) with the following orders:
-        // If |Rc| <= |Rs|:
-        //   1. All |Rp| <= |Rc|, and the larger, the better.
-        //   2. All |Rp| > |Rc|, and the smaller, the better.
-        //
-        // If |Rc| > |Rs|:
-        //   1. All |Rp| where |Rs| <= |Rp| <= |Rc|, and the smaller, the
-        //   better.
-        //   2. All |Rp| < |Rs|, and the larger, the better.
-        //   3. All |Rp| > |Rc|, and the smaller, the better.
-        //
-        const Rs = Math.floor(window.screen.width * window.devicePixelRatio);
-        const Rc = captureR.width;
-        const cmpDescending = (r1, r2) => r2.width - r1.width;
-        const cmpAscending = (r1, r2) => r1.width - r2.width;
-
-        if (Rc <= Rs) {
-          const notLargerThanRc =
-              resolutions.filter((r) => r.width <= Rc).sort(cmpDescending);
-          const largerThanRc =
-              resolutions.filter((r) => r.width > Rc).sort(cmpAscending);
-          return notLargerThanRc.concat(largerThanRc);
-        } else {
-          const betweenRsRc =
-              resolutions.filter((r) => Rs <= r.width && r.width <= Rc)
-                  .sort(cmpAscending);
-          const smallerThanRs =
-              resolutions.filter((r) => r.width < Rs).sort(cmpDescending);
-          const largerThanRc =
-              resolutions.filter((r) => r.width > Rc).sort(cmpAscending);
-          return betweenRsRc.concat(smallerThanRs).concat(largerThanRc);
-        }
-      };
-
       const /** !Array<!MediaStreamConstraints> */ previewCandidates =
-          sortPreview(previewRs).map(({width, height}) => ({
-                                       audio: false,
-                                       video: {
-                                         deviceId: {exact: deviceId},
-                                         width,
-                                         height,
-                                       },
-                                     }));
+          this.sortPreview_(previewRs, captureR).map(({width, height}) => ({
+                                                       audio: false,
+                                                       video: {
+                                                         deviceId:
+                                                             {exact: deviceId},
+                                                         width,
+                                                         height,
+                                                       },
+                                                     }));
       return {resolution: captureR, previewCandidates};
     };
 
     return this.deviceCapturePreviewResolutionMap_.get(deviceId)
         .map(toCaptureCandidate)
-        .sort(sortPrefResol);
+        .sort(this.getPreferResolutionSort_(prefR));
   }
 }

chromeos/components/camera_app_ui/resources/js/device/device_info_updater.js

@@ -2,16 +2,16 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import {browserProxy} from '../browser_proxy/browser_proxy.js';
-import {DeviceOperator} from '../mojo/device_operator.js';
 // eslint-disable-next-line no-unused-vars
-import {ResolutionList, VideoConfig} from '../type.js';
-
 import {Camera3DeviceInfo} from './camera3_device_info.js';
 import {
   PhotoConstraintsPreferrer,  // eslint-disable-line no-unused-vars
   VideoConstraintsPreferrer,  // eslint-disable-line no-unused-vars
 } from './constraints_preferrer.js';
+import {
+  DeviceInfo,  // eslint-disable-line no-unused-vars
+  StreamManager,
+} from './stream_manager.js';
 
 /**
  * Contains information of all cameras on the device and will updates its value
@@ -59,44 +59,45 @@ export class DeviceInfoUpdater {
 
     /**
      * MediaDeviceInfo of all available video devices.
-     * @type {!Promise<!Array<!MediaDeviceInfo>>}
-     * @private
-     */
-    this.devicesInfo_ = this.enumerateDevices_();
-
-    /**
-     * Got the permission to run enumerateDevices() or not.
-     * @type {boolean}
+     * @type {!Array<!MediaDeviceInfo>}
      * @private
      */
-    this.canEnumerateDevices_ = false;
+    this.devicesInfo_ = [];
 
     /**
      * Camera3DeviceInfo of all available video devices. Is null on HALv1 device
      * without mojo api support.
-     * @type {!Promise<?Array<!Camera3DeviceInfo>>}
+     * @type {!Array<!Camera3DeviceInfo>}
      * @private
      */
-    this.camera3DevicesInfo_ = this.queryMojoDevicesInfo_();
+    this.camera3DevicesInfo_ = [];
 
     /**
-     * Filter out lagging 720p on grunt. See https://crbug.com/1122852.
-     * @const {!Promise<function(!VideoConfig): boolean>}
+     * Pending device Information.
+     * @type {!Array<!DeviceInfo>}
      * @private
      */
-    this.videoConfigFilter_ = (async () => {
-      const board = await browserProxy.getBoard();
-      return board === 'grunt' ? ({height}) => height < 720 : () => true;
-    })();
+    this.pendingDevicesInfo_ = [];
 
     /**
      * Promise of first update.
      * @type {!Promise}
+     * @private
      */
-    this.firstUpdate_ = this.update_();
+    this.firstUpdate_ = StreamManager.getInstance().deviceUpdate();
 
-    navigator.mediaDevices.addEventListener(
-        'devicechange', this.update_.bind(this));
+    StreamManager.getInstance().addRealDeviceChangeListener(
+        this.deviceUpdate_.bind(this));
+  }
+
+  /**
+   * Handling function for device update
+   * @param {!Array<!DeviceInfo>} devicesInfo devices information
+   * @private
+   */
+  async deviceUpdate_(devicesInfo) {
+    this.pendingDevicesInfo_ = devicesInfo;
+    await this.update_();
   }
 
   /**
@@ -134,60 +135,13 @@ export class DeviceInfoUpdater {
    * @private
    */
   async doUpdate_() {
-    this.devicesInfo_ = this.enumerateDevices_();
-    this.camera3DevicesInfo_ = this.queryMojoDevicesInfo_();
-    try {
-      await this.devicesInfo_;
-      const devices = await this.camera3DevicesInfo_;
-      if (devices) {
-        this.photoPreferrer_.updateDevicesInfo(devices);
-        this.videoPreferrer_.updateDevicesInfo(devices);
-      }
+    this.devicesInfo_ = this.pendingDevicesInfo_.map((d) => d.v1Info);
+    this.camera3DevicesInfo_ = this.pendingDevicesInfo_.map((d) => d.v3Info);
+    if (this.camera3DevicesInfo_.length) {
+      this.photoPreferrer_.updateDevicesInfo(this.camera3DevicesInfo_);
+      this.videoPreferrer_.updateDevicesInfo(this.camera3DevicesInfo_);
       await Promise.all(this.deviceChangeListeners_.map((l) => l(this)));
-    } catch (e) {
-      console.error(e);
-    }
-  }
-
-  /**
-   * Enumerates all available devices and gets their MediaDeviceInfo.
-   * @return {!Promise<!Array<!MediaDeviceInfo>>}
-   * @throws {!Error}
-   * @private
-   */
-  async enumerateDevices_() {
-    if (!this.canEnumerateDevices_) {
-      this.canEnumerateDevices_ =
-          await browserProxy.requestEnumerateDevicesPermission();
-      if (!this.canEnumerateDevices_) {
-        throw new Error('Failed to get the permission for enumerateDevices()');
-      }
-    }
-    const devices = (await navigator.mediaDevices.enumerateDevices())
-                        .filter((device) => device.kind === 'videoinput');
-    if (devices.length === 0) {
-      throw new Error('Device list empty.');
-    }
-    return devices;
-  }
-
-  /**
-   * Queries Camera3DeviceInfo of available devices through private mojo API.
-   * @return {!Promise<?Array<!Camera3DeviceInfo>>} Camera3DeviceInfo
-   *     of available devices. Maybe null on HALv1 devices without supporting
-   *     private mojo api.
-   * @throws {!Error} Thrown when camera unplugging happens between enumerating
-   *     devices and querying mojo APIs with current device info results.
-   * @private
-   */
-  async queryMojoDevicesInfo_() {
-    if (!await DeviceOperator.isSupported()) {
-      return null;
     }
-    const deviceInfos = await this.devicesInfo_;
-    const videoConfigFilter = await this.videoConfigFilter_;
-    return Promise.all(
-        deviceInfos.map((d) => Camera3DeviceInfo.create(d, videoConfigFilter)));
   }
 
   /**

chromeos/components/camera_app_ui/resources/js/device/stream_manager.js

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {browserProxy} from '../browser_proxy/browser_proxy.js';
+import {assertString} from '../chrome_util.js';
+import {DeviceOperator} from '../mojo/device_operator.js';
+import {
+  Facing,
+  VideoConfig,  // eslint-disable-line no-unused-vars
+} from '../type.js';
+import {Camera3DeviceInfo} from './camera3_device_info.js';
+
+/**
+ * The singleton instance of StreamManager. Initialized by the first
+ * invocation of getInstance().
+ * @type {?StreamManager}
+ */
+let instance = null;
+
+/**
+ * Device information includs MediaDeviceInfo and Camera3DeviceInfo.
+ * @typedef {{
+ *   v1Info: !MediaDeviceInfo,
+ *   v3Info: !Camera3DeviceInfo
+ * }}
+ */
+export let DeviceInfo;
+
+/**
+ * Creates extra stream for the current mode.
+ */
+export class CaptureStream {
+  /**
+   * @param {string} deviceId Device id of currently working video device
+   * @param {!MediaStream} stream Capture stream
+   * @private
+   */
+  constructor(deviceId, stream) {
+    /**
+     * Device id of currently working video device.
+     * @type {string}
+     * @private
+     */
+    this.deviceId_ = deviceId;
+
+    /**
+     * Capture stream
+     * @type {!MediaStream}
+     * @protected
+     */
+    this.stream_ = stream;
+  }
+
+  /**
+   * @return {!MediaStream}
+   * @public
+   */
+  get stream() {
+    return this.stream_;
+  }
+
+  /**
+   * Closes stream
+   * @public
+   */
+  async close() {
+    this.stream_.getVideoTracks()[0].stop();
+    try {
+      await StreamManager.getInstance().setMultipleStreamsEnabled(
+          this.deviceId_, false);
+    } catch (e) {
+      console.error(e);
+    }
+  }
+}
+
+/**
+ * Monitors device change and provides different listener callbacks for
+ * device changes. It also provides streams for different modes.
+ */
+export class StreamManager {
+  /**
+   * @private
+   */
+  constructor() {
+    /**
+     * MediaDeviceInfo of all available video devices.
+     * @type {?Promise<!Array<!MediaDeviceInfo>>}
+     * @private
+     */
+    this.devicesInfo_ = null;
+
+    /**
+     * Got the permission to run enumerateDevices() or not.
+     * @type {boolean}
+     * @private
+     */
+    this.canEnumerateDevices_ = false;
+
+    /**
+     * Camera3DeviceInfo of all available video devices. Is null on HALv1 device
+     * without mojo api support.
+     * @type {?Promise<?Array<!DeviceInfo>>}
+     * @private
+     */
+    this.camera3DevicesInfo_ = null;
+
+    /**
+     * Listeners for real device change event.
+     * @type {!Array<function(!Array<!DeviceInfo>): !Promise>}
+     * @private
+     */
+    this.realListeners_ = [];
+
+    /**
+     * Latest result of Camera3DeviceInfo of all real video devices.
+     * @type {!Array<!DeviceInfo>}
+     * @private
+     */
+    this.realDevices_ = [];
+
+    /**
+     * Maps from real device id to corresponding virtual devices id and it is
+     * only available on HALv3.
+     * @type {!Map<string, string>}
+     * @private
+     */
+    this.virtualMap_ = new Map();
+
+    /**
+     * Filter out lagging 720p on grunt. See https://crbug.com/1122852.
+     * @const {!Promise<function(!VideoConfig): boolean>}
+     * @private
+     */
+    this.videoConfigFilter_ = (async () => {
+      const board = await browserProxy.getBoard();
+      return board === 'grunt' ? ({height}) => height < 720 : () => true;
+    })();
+
+    navigator.mediaDevices.addEventListener(
+        'devicechange', this.deviceUpdate.bind(this));
+  }
+
+  /**
+   * Creates a new instance of StreamManager if it is not set. Returns the
+   *     exist instance.
+   * @return {!StreamManager} The singleton instance.
+   */
+  static getInstance() {
+    if (instance === null) {
+      instance = new StreamManager();
+    }
+    return instance;
+  }
+
+  /**
+   * Registers listener to be called when state of available real devices
+   * changes.
+   * @param {function(!Array<!DeviceInfo>)} listener
+   */
+  addRealDeviceChangeListener(listener) {
+    this.realListeners_.push(listener);
+  }
+
+  /**
+   * Creates extra stream according to the constraints.
+   * @param {!MediaStreamConstraints} constraints
+   * @return {!Promise<!CaptureStream>}
+   */
+  async openCaptureStream(constraints) {
+    const realDeviceId = assertString(constraints.video.deviceId.exact);
+    try {
+      await this.setMultipleStreamsEnabled(realDeviceId, true);
+    } catch (e) {
+      console.error(e);
+    }
+
+    constraints.video.deviceId.exact = this.virtualMap_.get(realDeviceId);
+
+    const stream = await navigator.mediaDevices.getUserMedia(constraints);
+    return new CaptureStream(realDeviceId, stream);
+  }
+
+  /**
+   * Handling function for device changing.
+   */
+  async deviceUpdate() {
+    const devices = await this.doDeviceInfoUpdate_();
+    if (devices === null) {
+      return;
+    }
+    await this.doDeviceNotify_(devices);
+  }
+
+  /**
+   * Gets devices information via mojo IPC.
+   * @return {?Promise<?Array<!DeviceInfo>>}
+   * @private
+   */
+  async doDeviceInfoUpdate_() {
+    this.devicesInfo_ = this.enumerateDevices_();
+    this.camera3DevicesInfo_ = this.queryMojoDevicesInfo_();
+    try {
+      return await this.camera3DevicesInfo_;
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+
+  /**
+   * Notifies device changes to listeners and create a mapping for real and
+   * virtual device.
+   * @param {!Array<!DeviceInfo>} devices
+   * @private
+   */
+  async doDeviceNotify_(devices) {
+    const isVirtual = (d) => d.v1Info.label.startsWith('Virtual Camera');
+    const realDevices = devices.filter((d) => !isVirtual(d));
+    const virtualDevices = devices.filter(isVirtual);
+    if (this.isRealDeviceChange_(realDevices)) {
+      await Promise.all(this.realListeners_.map((l) => l(realDevices)));
+    }
+    this.virtualMap_ = new Map();
+    for (const device of virtualDevices) {
+      let dev = null;
+      switch (device.v3Info.facing) {
+        case Facing.VIRTUAL_USER:
+          dev = realDevices.find((d) => d.v3Info.facing === Facing.USER);
+          break;
+        case Facing.VIRTUAL_ENV:
+          dev = realDevices.find((d) => d.v3Info.facing === Facing.ENVIRONMENT);
+          break;
+        case Facing.VIRTUAL_EXT:
+          // TODO(b/172306905): Handle multiple external devices.
+          dev = realDevices.find((d) => d.v3Info.facing === Facing.EXTERNAL);
+          break;
+        default:
+          console.error(`Invalid facing: ${device.v3Info.facing}`);
+      }
+      if (dev) {
+        this.virtualMap_.set(dev.v3Info.deviceId, device.v3Info.deviceId);
+      }
+    }
+    this.realDevices_ = realDevices;
+  }
+
+  /**
+   * Compares devices info to see whether there are only real devices change.
+   * @param {!Array<!DeviceInfo>} devices
+   * @return {boolean}
+   * @private
+   */
+  isRealDeviceChange_(devices) {
+    if (this.realDevices_.length === 0) {
+      return true;
+    }
+    const realChange = (devices1, devices2) => {
+      for (const device of devices1) {
+        const found =
+            devices2.find(
+                (d2) => d2.v3Info.deviceId === device.v3Info.deviceId) ||
+            null;
+        if (found === null) {
+          return true;
+        }
+      }
+      return false;
+    };
+    return realChange(devices, this.realDevices_) ||
+        realChange(this.realDevices_, devices);
+  }
+
+  /**
+   * Enumerates all available devices and gets their MediaDeviceInfo.
+   * @return {!Promise<!Array<!MediaDeviceInfo>>}
+   * @throws {!Error}
+   * @private
+   */
+  async enumerateDevices_() {
+    if (!this.canEnumerateDevices_) {
+      this.canEnumerateDevices_ =
+          await browserProxy.requestEnumerateDevicesPermission();
+      if (!this.canEnumerateDevices_) {
+        throw new Error('Failed to get the permission for enumerateDevices()');
+      }
+    }
+    const devices = (await navigator.mediaDevices.enumerateDevices())
+                        .filter((device) => device.kind === 'videoinput');
+    if (devices.length === 0) {
+      throw new Error('Device list empty.');
+    }
+    return devices;
+  }
+
+  /**
+   * Queries Camera3DeviceInfo of available devices through private mojo API.
+   * @return {!Promise<?Array<!DeviceInfo>>} Camera3DeviceInfo of available
+   *     devices. Maybe null on HALv1 devices without supporting private mojo
+   *     api.
+   * @throws {!Error} Thrown when camera unplugging happens between enumerating
+   *     devices and querying mojo APIs with current device info results.
+   * @private
+   */
+  async queryMojoDevicesInfo_() {
+    if (!await DeviceOperator.isSupported()) {
+      return null;
+    }
+    const deviceInfos = await this.devicesInfo_;
+    const videoConfigFilter = await this.videoConfigFilter_;
+    return Promise.all(deviceInfos.map(
+        async (d) => ({
+          v1Info: d,
+          v3Info: await Camera3DeviceInfo.create(d, videoConfigFilter),
+        })));
+  }
+
+  /**
+   * Enables/Disables multiple streams on target camera device. The extra
+   * stream will be reported as virtual video device from
+   * navigator.mediaDevices.enumerateDevices().
+   * @param {string} deviceId The id of target camera device.
+   * @param {boolean} enabled True for eanbling multiple streams.
+   */
+  async setMultipleStreamsEnabled(deviceId, enabled) {
+    const deviceOperator = await DeviceOperator.getInstance();
+    await deviceOperator.setMultipleStreamsEnabled(deviceId, enabled);
+    await this.deviceUpdate();
+    if (this.virtualMap_.has(deviceId) !== enabled) {
+      throw new Error(`${deviceId} set multiple streams to ${enabled} failed`);
+    }
+  }
+}

chromeos/components/camera_app_ui/resources/js/mojo/device_operator.js

@@ -271,6 +271,12 @@ export class DeviceOperator {
         return Facing.USER;
       case cros.mojom.CameraFacing.CAMERA_FACING_EXTERNAL:
         return Facing.EXTERNAL;
+      case cros.mojom.CameraFacing.CAMERA_FACING_VIRTUAL_BACK:
+        return Facing.VIRTUAL_ENV;
+      case cros.mojom.CameraFacing.CAMERA_FACING_VIRTUAL_FRONT:
+        return Facing.VIRTUAL_USER;
+      case cros.mojom.CameraFacing.CAMERA_FACING_VIRTUAL_EXTERNAL:
+        return Facing.VIRTUAL_EXT;
       default:
         assertNotReached(`Unexpected facing value: ${facing}`);
     }
@@ -479,6 +485,19 @@ export class DeviceOperator {
     return event.wait();
   }
 
+  /**
+   * Enables/Disables multiple streams on target camera device. The extra
+   * stream will be reported as virtual video device from
+   * navigator.mediaDevices.enumerateDevices().
+   * @param {?string} deviceId The id of target camera device.
+   * @param {boolean} enabled True for eanbling multiple streams.
+   */
+  async setMultipleStreamsEnabled(deviceId, enabled) {
+    if (deviceId) {
+      await this.deviceProvider_.setMultipleStreamsEnabled(deviceId, enabled);
+    }
+  }
+
   /**
    * Creates a new instance of DeviceOperator if it is not set. Returns the
    *     exist instance.

chromeos/components/camera_app_ui/resources/js/type.js

@@ -95,6 +95,11 @@ export const Facing = {
   USER: 'user',
   ENVIRONMENT: 'environment',
   EXTERNAL: 'external',
+  // VIRTUAL_{facing} is for labeling video device for configuring extra stream
+  // from corresponding {facing} video device.
+  VIRTUAL_USER: 'virtual_user',
+  VIRTUAL_ENV: 'virtual_environment',
+  VIRTUAL_EXT: 'virtual_external',
   NOT_SET: '(not set)',
   UNKNOWN: 'unknown',
 };

chromeos/components/camera_app_ui/resources/js/views/camera.js

@@ -527,6 +527,9 @@ export class Camera extends View {
           await this.endTake_();
         }
       } finally {
+        // Stopping preview will wait device close. Therefore, we clear
+        // mode before stopping preview to close extra stream first.
+        await this.modes_.clear();
         await this.preview_.close();
       }
       return this.start_();
@@ -562,9 +565,8 @@ export class Camera extends View {
         }
         const factory = this.modes_.getModeFactory(mode);
         try {
-          factory.setCaptureResolution(captureR);
           if (deviceOperator !== null) {
-            factory.prepareDevice(deviceOperator, constraints);
+            factory.prepareDevice(deviceOperator, constraints, captureR);
           }
           const stream = await this.preview_.open(constraints);
           this.facingMode_ = await this.options_.updateValues(stream);
@@ -576,7 +578,10 @@ export class Camera extends View {
           nav.close(ViewName.WARNING, WarningType.NO_CAMERA);
           return true;
         } catch (e) {
-          factory.clear();
+          // Stopping preview will wait device close. Therefore, we clear
+          // mode before stopping preview to close extra stream first.
+          await factory.clear();
+          await this.modes_.clear();
           this.preview_.close();
           console.error(e);
         }

chromeos/components/camera_app_ui/resources/js/views/camera/mode/index.js

@@ -366,7 +366,7 @@ export class Modes {
   }
 
   /**
-   * Creates and updates new current mode object.
+   * Creates and updates current mode object.
    * @param {!Mode} mode Classname of mode to be updated.
    * @param {!ModeFactory} factory The factory ready for producing mode capture
    *     object.
@@ -390,6 +390,17 @@ export class Modes {
     await this.updateSaveMetadata_();
   }
 
+  /**
+   * Clears everything when mode is not needed anymore.
+   * @return {!Promise}
+   */
+  async clear() {
+    if (this.current !== null) {
+      await this.current.clear();
+    }
+    this.current = null;
+  }
+
   /**
    * Checks whether to save image metadata or not.
    * @return {!Promise} Promise for the operation.

chromeos/components/camera_app_ui/resources/js/views/camera/mode/mode_base.js

@@ -79,11 +79,19 @@ export class ModeBase {
   async addMetadataObserver() {}
 
   /**
-   * Remove the observer that saves metadata.
+   * Removes the observer that saves metadata.
    * @return {!Promise} Promise for the operation.
    */
   async removeMetadataObserver() {}
 
+  /**
+   * Clears everything when mode is not needed anymore.
+   * @return {!Promise}
+   */
+  async clear() {
+    await this.stopCapture();
+  }
+
   /**
    * Initiates video/photo capture operation under this mode.
    * @return {!Promise}
@@ -137,13 +145,6 @@ export class ModeFactory {
     return assertInstanceof(this.stream_, MediaStream);
   }
 
-  /**
-   * @param {!Resolution} resolution
-   */
-  setCaptureResolution(resolution) {
-    this.captureResolution_ = resolution;
-  }
-
   /**
    * @param {!Facing} facing
    */
@@ -164,10 +165,11 @@ export class ModeFactory {
    *     capture device.
    * @param {!MediaStreamConstraints} constraints Constraints for preview
    *     stream.
+   * @param {!Resolution} resolution Capture resolution
    * @return {!Promise}
    * @abstract
    */
-  prepareDevice(deviceOperator, constraints) {}
+  prepareDevice(deviceOperator, constraints, resolution) {}
 
   /**
    * @return {!ModeBase}

chromeos/components/camera_app_ui/resources/js/views/camera/mode/photo.js

@@ -243,7 +243,8 @@ export class PhotoFactory extends ModeFactory {
   /**
    * @override
    */
-  async prepareDevice(deviceOperator, constraints) {
+  async prepareDevice(deviceOperator, constraints, resolution) {
+    this.captureResolution_ = resolution;
     const deviceId = assertString(constraints.video.deviceId.exact);
     await deviceOperator.setCaptureIntent(
         deviceId, cros.mojom.CaptureIntent.STILL_CAPTURE);

chromeos/components/camera_app_ui/resources/js/views/camera/mode/video.js

@@ -4,7 +4,15 @@
 
 import {AsyncJobQueue} from '../../../async_job_queue.js';
 import {browserProxy} from '../../../browser_proxy/browser_proxy.js';
-import {assert, assertString} from '../../../chrome_util.js';
+import {
+  assert,
+  assertInstanceof,
+  assertString,
+} from '../../../chrome_util.js';
+import {
+  CaptureStream,
+  StreamManager,
+} from '../../../device/stream_manager.js';
 import {Filenamer} from '../../../models/file_namer.js';
 import {
   VideoSaver,  // eslint-disable-line no-unused-vars
@@ -118,12 +126,12 @@ export class VideoHandler {
  */
 export class Video extends ModeBase {
   /**
-   * @param {!MediaStream} stream
+   * @param {!CaptureStream} stream
    * @param {!Facing} facing
    * @param {!VideoHandler} handler
    */
   constructor(stream, facing, handler) {
-    super(stream, facing, null);
+    super(stream.stream, facing, null);
 
     /**
      * @const {!VideoHandler}
@@ -171,6 +179,20 @@ export class Video extends ModeBase {
      * Whether current recording ever paused/resumed before it ended.
      */
     this.everPaused_ = false;
+
+    /**
+     * @type {!CaptureStream}
+     * @private
+     */
+    this.captureStream_ = stream;
+  }
+
+  /**
+   * @override
+   */
+  async clear() {
+    await this.stopCapture();
+    await this.captureStream_.close();
   }
 
   /**
@@ -377,12 +399,20 @@ export class VideoFactory extends ModeFactory {
      * @private
      */
     this.handler_ = handler;
+
+    /**
+     * Extra stream for video capturing.
+     * @type {?CaptureStream}
+     * @private
+     */
+    this.extraStream_ = null;
   }
 
   /**
    * @override
    */
-  async prepareDevice(deviceOperator, constraints) {
+  async prepareDevice(deviceOperator, constraints, resolution) {
+    this.captureResolution_ = resolution;
     const deviceId = assertString(constraints.video.deviceId.exact);
     await deviceOperator.setCaptureIntent(
         deviceId, cros.mojom.CaptureIntent.VIDEO_RECORD);
@@ -403,12 +433,26 @@ export class VideoFactory extends ModeFactory {
       // range.
     }
     await deviceOperator.setFpsRange(deviceId, minFrameRate, maxFrameRate);
+
+    const captureConstraints = {
+      audio: constraints.audio,
+      video: {
+        deviceId: constraints.video.deviceId,
+        frameRate: constraints.video.frameRate,
+        width: resolution.width,
+        height: resolution.height,
+      },
+    };
+    this.extraStream_ =
+        await StreamManager.getInstance().openCaptureStream(captureConstraints);
   }
 
   /**
    * @override
    */
   produce_() {
-    return new Video(this.previewStream_, this.facing_, this.handler_);
+    return new Video(
+        assertInstanceof(this.extraStream_, CaptureStream), this.facing_,
+        this.handler_);
   }
 }

chromeos/components/camera_app_ui/resources/js/views/settings.js

@@ -262,7 +262,7 @@ export class ResolutionSettings extends BaseSettings {
     this.frontSetting_ = null;
 
     /**
-     * Device setting of back camera. Null if no front camera.
+     * Device setting of back camera. Null if no back camera.
      * @type {?DeviceSetting}
      * @private
      */