Fix NativeDesktopMediaList to set aura_id for chrome Aura windows for Linux and windows.

Previously NativeDesktopMediaList was returning only native window ID, now it also sets aura_id for chrome in Linux and windows. This allows to capture browser windows directly from Aura, which is faster and more reliable.

BUG=581790, 289779, 590915

Review URL: https://codereview.chromium.org/1763753003

Cr-Commit-Position: refs/heads/master@{#381711}

chrome/browser/media/desktop_media_list_base.cc

@@ -8,8 +8,10 @@
 
 #include "chrome/browser/media/desktop_media_list_observer.h"
 #include "content/public/browser/browser_thread.h"
+#include "ui/gfx/image/image.h"
 
 using content::BrowserThread;
+using content::DesktopMediaID;
 
 DesktopMediaListBase::DesktopMediaListBase(base::TimeDelta update_period)
     : update_period_(update_period), weak_factory_(this) {}
@@ -25,8 +27,7 @@ void DesktopMediaListBase::SetThumbnailSize(const gfx::Size& thumbnail_size) {
   thumbnail_size_ = thumbnail_size;
 }
 
-void DesktopMediaListBase::SetViewDialogWindowId(
-    content::DesktopMediaID dialog_id) {
+void DesktopMediaListBase::SetViewDialogWindowId(DesktopMediaID dialog_id) {
   view_dialog_id_ = dialog_id;
 }
 
@@ -49,13 +50,13 @@ const DesktopMediaList::Source& DesktopMediaListBase::GetSource(
 }
 
 DesktopMediaListBase::SourceDescription::SourceDescription(
-    content::DesktopMediaID id,
+    DesktopMediaID id,
     const base::string16& name)
     : id(id), name(name) {}
 
 void DesktopMediaListBase::UpdateSourcesList(
     const std::vector<SourceDescription>& new_sources) {
-  typedef std::set<content::DesktopMediaID> SourceSet;
+  typedef std::set<DesktopMediaID> SourceSet;
   SourceSet new_source_set;
   for (size_t i = 0; i < new_sources.size(); ++i) {
     new_source_set.insert(new_sources[i].id);
@@ -115,7 +116,7 @@ void DesktopMediaListBase::UpdateSourcesList(
   }
 }
 
-void DesktopMediaListBase::UpdateSourceThumbnail(content::DesktopMediaID id,
+void DesktopMediaListBase::UpdateSourceThumbnail(DesktopMediaID id,
                                                  const gfx::ImageSkia& image) {
   for (size_t i = 0; i < sources_.size(); ++i) {
     if (sources_[i].id == id) {
@@ -132,3 +133,14 @@ void DesktopMediaListBase::ScheduleNextRefresh() {
       base::Bind(&DesktopMediaListBase::Refresh, weak_factory_.GetWeakPtr()),
       update_period_);
 }
+
+// static
+uint32_t DesktopMediaListBase::GetImageHash(const gfx::Image& image) {
+  SkBitmap bitmap = image.AsBitmap();
+  bitmap.lockPixels();
+  uint32_t value =
+      base::Hash(reinterpret_cast<char*>(bitmap.getPixels()), bitmap.getSize());
+  bitmap.unlockPixels();
+
+  return value;
+}

chrome/browser/media/desktop_media_list_base.h

@@ -8,6 +8,10 @@
 #include "chrome/browser/media/desktop_media_list.h"
 #include "content/public/browser/desktop_media_id.h"
 
+namespace gfx {
+class Image;
+}
+
 // Thumbnail size is 100*100 pixels
 static const int kDefaultThumbnailSize = 100;
 
@@ -27,6 +31,8 @@ class DesktopMediaListBase : public DesktopMediaList {
   int GetSourceCount() const override;
   const Source& GetSource(int index) const override;
 
+  static uint32_t GetImageHash(const gfx::Image& image);
+
  protected:
   struct SourceDescription {
     SourceDescription(content::DesktopMediaID id, const base::string16& name);

chrome/browser/media/native_desktop_media_list.cc

@@ -8,6 +8,9 @@
 #include "base/strings/utf_string_conversions.h"
 #include "base/threading/sequenced_worker_pool.h"
 #include "chrome/browser/media/desktop_media_list_observer.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_list.h"
+#include "chrome/browser/ui/browser_window.h"
 #include "chrome/grit/generated_resources.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/video_util.h"
@@ -16,7 +19,11 @@
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
+#include "ui/aura/window.h"
+#include "ui/aura/window_tree_host.h"
 #include "ui/base/l10n/l10n_util.h"
+#include "ui/gfx/native_widget_types.h"
+#include "ui/snapshot/snapshot.h"
 
 using content::BrowserThread;
 using content::DesktopMediaID;
@@ -30,7 +37,7 @@ const int kDefaultUpdatePeriod = 1000;
 // media source has changed.
 uint32_t GetFrameHash(webrtc::DesktopFrame* frame) {
   int data_size = frame->stride() * frame->size().height();
-  return base::SuperFastHash(reinterpret_cast<char*>(frame->data()), data_size);
+  return base::Hash(reinterpret_cast<char*>(frame->data()), data_size);
 }
 
 gfx::ImageSkia ScaleDesktopFrame(scoped_ptr<webrtc::DesktopFrame> frame,
@@ -67,6 +74,33 @@ gfx::ImageSkia ScaleDesktopFrame(scoped_ptr<webrtc::DesktopFrame> frame,
   return gfx::ImageSkia::CreateFrom1xBitmap(result);
 }
 
+#if defined(USE_AURA)
+
+NativeDesktopMediaList::NativeAuraIdMap GetBrowserNativeAuraIdMap() {
+  NativeDesktopMediaList::NativeAuraIdMap id_map;
+  for (auto* browser : *BrowserList::GetInstance()) {
+    aura::Window* aura_window = browser->window()->GetNativeWindow();
+    if (!aura_window)
+      continue;
+    aura::WindowTreeHost* host = aura_window->GetHost();
+    if (!host)
+      continue;
+    gfx::AcceleratedWidget widget = host->GetAcceleratedWidget();
+#if defined(OS_WIN)
+    DesktopMediaID::Id native_id = reinterpret_cast<DesktopMediaID::Id>(widget);
+#else
+    DesktopMediaID::Id native_id = widget;
+#endif
+    DesktopMediaID media_id = DesktopMediaID::RegisterAuraWindow(
+        DesktopMediaID::TYPE_WINDOW, aura_window);
+    id_map[native_id] = media_id.aura_id;
+  }
+
+  return id_map;
+}
+
+#endif  // defined(USE_AURA)
+
 }  // namespace
 
 class NativeDesktopMediaList::Worker
@@ -78,7 +112,8 @@ class NativeDesktopMediaList::Worker
   ~Worker() override;
 
   void Refresh(const gfx::Size& thumbnail_size,
-               content::DesktopMediaID::Id view_dialog_id);
+               const DesktopMediaID::Id& view_dialog_id,
+               const NativeAuraIdMap& native_aura_id_map);
 
  private:
   typedef std::map<DesktopMediaID, uint32_t> ImageHashesMap;
@@ -115,8 +150,10 @@ NativeDesktopMediaList::Worker::~Worker() {}
 
 void NativeDesktopMediaList::Worker::Refresh(
     const gfx::Size& thumbnail_size,
-    content::DesktopMediaID::Id view_dialog_id) {
+    const DesktopMediaID::Id& view_dialog_id,
+    const NativeAuraIdMap& native_aura_id_map) {
   std::vector<SourceDescription> sources;
+  std::vector<DesktopMediaID> aura_media_ids;
 
   if (screen_capturer_) {
     webrtc::ScreenCapturer::ScreenList screens;
@@ -145,13 +182,22 @@ void NativeDesktopMediaList::Worker::Refresh(
            it != windows.end(); ++it) {
         // Skip the picker dialog window.
         if (it->id != view_dialog_id) {
-          sources.push_back(SourceDescription(
-              DesktopMediaID(DesktopMediaID::TYPE_WINDOW, it->id),
-              base::UTF8ToUTF16(it->title)));
+          DesktopMediaID media_id(DesktopMediaID::TYPE_WINDOW, it->id);
+#if defined(USE_AURA)
+          // Associate aura id with native id.
+          auto aura_id = native_aura_id_map.find(media_id.id);
+          if (aura_id != native_aura_id_map.end()) {
+            media_id.aura_id = aura_id->second;
+            aura_media_ids.push_back(media_id);
+          }
+#endif
+          sources.push_back(
+              SourceDescription(media_id, base::UTF8ToUTF16(it->title)));
         }
       }
     }
   }
+
   // Update list of windows before updating thumbnails.
   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
                           base::Bind(&NativeDesktopMediaList::UpdateSourcesList,
@@ -162,6 +208,7 @@ void NativeDesktopMediaList::Worker::Refresh(
   // Get a thumbnail for each source.
   for (size_t i = 0; i < sources.size(); ++i) {
     SourceDescription& source = sources[i];
+
     switch (source.id.type) {
       case DesktopMediaID::TYPE_SCREEN:
         if (!screen_capturer_->SelectScreen(source.id.id))
@@ -170,6 +217,12 @@ void NativeDesktopMediaList::Worker::Refresh(
         break;
 
       case DesktopMediaID::TYPE_WINDOW:
+#if defined(USE_AURA)
+        // Aura window thumbmail capture is skipped here. It will be done
+        // asynchronously in the UI thread.
+        if (source.id.aura_id > DesktopMediaID::kNullId)
+          continue;
+#endif
         if (!window_capturer_->SelectWindow(source.id.id))
           continue;
         window_capturer_->Capture(webrtc::DesktopRegion());
@@ -193,17 +246,21 @@ void NativeDesktopMediaList::Worker::Refresh(
             ScaleDesktopFrame(std::move(current_frame_), thumbnail_size);
         BrowserThread::PostTask(
             BrowserThread::UI, FROM_HERE,
-            base::Bind(&NativeDesktopMediaList::OnSourceThumbnail, media_list_,
-                       i, thumbnail));
+            base::Bind(&NativeDesktopMediaList::OnSourceThumbnailCaptured,
+                       media_list_, i, thumbnail));
       }
     }
   }
 
   image_hashes_.swap(new_image_hashes);
 
+  // Aura thumbnail captures have to be done in UI thread. After they are done,
+  // a refresh will be scheduled.
   BrowserThread::PostTask(
       BrowserThread::UI, FROM_HERE,
-      base::Bind(&NativeDesktopMediaList::ScheduleNextRefresh, media_list_));
+      base::Bind(
+          &NativeDesktopMediaList::FinishRefreshOnUiThreadAndScheduleNext,
+          media_list_, aura_media_ids));
 }
 
 void NativeDesktopMediaList::Worker::OnCaptureCompleted(
@@ -231,12 +288,80 @@ NativeDesktopMediaList::~NativeDesktopMediaList() {
 }
 
 void NativeDesktopMediaList::Refresh() {
+  NativeAuraIdMap native_aura_id_map;
+#if defined(USE_AURA)
+  native_aura_id_map = GetBrowserNativeAuraIdMap();
+  pending_aura_capture_requests_ = 0;
+  new_aura_thumbnail_hashes_.clear();
+#endif
+
   capture_task_runner_->PostTask(
-      FROM_HERE, base::Bind(&Worker::Refresh, base::Unretained(worker_.get()),
-                            thumbnail_size_, view_dialog_id_.id));
+      FROM_HERE,
+      base::Bind(&Worker::Refresh, base::Unretained(worker_.get()),
+                 thumbnail_size_, view_dialog_id_.id, native_aura_id_map));
 }
 
-void NativeDesktopMediaList::OnSourceThumbnail(int index,
-                                               const gfx::ImageSkia& image) {
+void NativeDesktopMediaList::OnSourceThumbnailCaptured(
+    int index,
+    const gfx::ImageSkia& image) {
   UpdateSourceThumbnail(GetSource(index).id, image);
 }
+
+void NativeDesktopMediaList::FinishRefreshOnUiThreadAndScheduleNext(
+    const std::vector<DesktopMediaID>& aura_ids) {
+  // Schedule a refresh here when there is no aura thumbanil capture or schedule
+  // a refresh in OnAuraThumbnailCaptured() after all aura thumbnails are
+  // captured.
+  if (aura_ids.size() == 0) {
+    ScheduleNextRefresh();
+    return;
+  }
+
+#if defined(USE_AURA)
+  DCHECK_EQ(pending_aura_capture_requests_, 0);
+  for (const auto& aura_id : aura_ids) {
+    CaptureAuraWindowThumbnail(aura_id);
+  }
+#endif
+}
+
+#if defined(USE_AURA)
+
+void NativeDesktopMediaList::CaptureAuraWindowThumbnail(
+    const DesktopMediaID& id) {
+  gfx::NativeWindow window = DesktopMediaID::GetAuraWindowById(id);
+  if (!window)
+    return;
+
+  gfx::Rect window_rect(window->bounds().width(), window->bounds().height());
+  gfx::Rect scaled_rect = media::ComputeLetterboxRegion(
+      gfx::Rect(thumbnail_size_), window_rect.size());
+
+  pending_aura_capture_requests_++;
+  ui::GrabWindowSnapshotAndScaleAsync(
+      window, window_rect, scaled_rect.size(), BrowserThread::GetBlockingPool(),
+      base::Bind(&NativeDesktopMediaList::OnAuraThumbnailCaptured,
+                 weak_factory_.GetWeakPtr(), id));
+}
+
+void NativeDesktopMediaList::OnAuraThumbnailCaptured(const DesktopMediaID& id,
+                                                     const gfx::Image& image) {
+  if (!image.IsEmpty()) {
+    // Only new or changed thumbnail need update.
+    new_aura_thumbnail_hashes_[id] = GetImageHash(image);
+    if (!previous_aura_thumbnail_hashes_.count(id) ||
+        previous_aura_thumbnail_hashes_[id] != new_aura_thumbnail_hashes_[id]) {
+      UpdateSourceThumbnail(id, image.AsImageSkia());
+    }
+  }
+
+  // After all aura windows are processed, schedule next refresh;
+  pending_aura_capture_requests_--;
+  DCHECK_GE(pending_aura_capture_requests_, 0);
+  if (pending_aura_capture_requests_ == 0) {
+    previous_aura_thumbnail_hashes_ = std::move(new_aura_thumbnail_hashes_);
+    ScheduleNextRefresh();
+  }
+}
+
+#endif  // defined(USE_AURA)

chrome/browser/media/native_desktop_media_list.h

@@ -10,6 +10,7 @@
 #include "base/sequenced_task_runner.h"
 #include "chrome/browser/media/desktop_media_list_base.h"
 #include "content/public/browser/desktop_media_id.h"
+#include "ui/gfx/image/image.h"
 
 namespace webrtc {
 class ScreenCapturer;
@@ -20,6 +21,9 @@ class WindowCapturer;
 // native windows.
 class NativeDesktopMediaList : public DesktopMediaListBase {
  public:
+  typedef std::map<content::DesktopMediaID::Id, content::DesktopMediaID::Id>
+      NativeAuraIdMap;
+
   // Caller may pass NULL for either of the arguments in case when only some
   // types of sources the model should be populated with (e.g. it will only
   // contain windows, if |screen_capturer| is NULL).
@@ -29,6 +33,8 @@ class NativeDesktopMediaList : public DesktopMediaListBase {
   ~NativeDesktopMediaList() override;
 
  private:
+  typedef std::map<content::DesktopMediaID, uint32_t> ImageHashesMap;
+
   class Worker;
   friend class Worker;
 
@@ -38,7 +44,16 @@ class NativeDesktopMediaList : public DesktopMediaListBase {
   // OnSourceThumbnail() for each changed thumbnail and then calls
   // DelayLaunchNextRefersh() at the end.
   void Refresh() override;
-  void OnSourceThumbnail(int index, const gfx::ImageSkia& image);
+
+  void OnSourceThumbnailCaptured(int index, const gfx::ImageSkia& image);
+  void FinishRefreshOnUiThreadAndScheduleNext(
+      const std::vector<content::DesktopMediaID>& aura_ids);
+
+#if defined(USE_AURA)
+  void CaptureAuraWindowThumbnail(const content::DesktopMediaID& id);
+  void OnAuraThumbnailCaptured(const content::DesktopMediaID& id,
+                               const gfx::Image& image);
+#endif
 
   // Task runner used for the |worker_|.
   scoped_refptr<base::SequencedTaskRunner> capture_task_runner_;
@@ -48,6 +63,17 @@ class NativeDesktopMediaList : public DesktopMediaListBase {
   // after the model is destroyed.
   scoped_ptr<Worker> worker_;
 
+#if defined(USE_AURA)
+  // previous_aura_thumbnail_hashes_ holds thumbanil hash values of aura windows
+  // in the previous refresh. While new_aura_thumbnail_hashes_ has hash values
+  // of the ongoing refresh. Those two maps are used to detect new thumbnails
+  // and changed thumbnails from the previous refresh.
+  ImageHashesMap previous_aura_thumbnail_hashes_;
+  ImageHashesMap new_aura_thumbnail_hashes_;
+
+  int pending_aura_capture_requests_;
+#endif
+
   base::WeakPtrFactory<NativeDesktopMediaList> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(NativeDesktopMediaList);

chrome/browser/media/tab_desktop_media_list.cc

@@ -25,18 +25,6 @@ using content::DesktopMediaID;
 
 namespace {
 
-// Returns a hash of a favicon to detect when the favicon of media source has
-// changed.
-uint32_t GetImageHash(const gfx::Image& favicon) {
-  SkBitmap bitmap = favicon.AsBitmap();
-  bitmap.lockPixels();
-  uint32_t value =
-      base::Hash(reinterpret_cast<char*>(bitmap.getPixels()), bitmap.getSize());
-  bitmap.unlockPixels();
-
-  return value;
-}
-
 gfx::ImageSkia CreateEnlargedFaviconImage(gfx::Size size,
                                           const gfx::ImageSkia& favicon) {
   DCHECK_GE(size.width(), 20);

content/browser/renderer_host/media/media_stream_manager.cc

@@ -38,6 +38,7 @@
 #include "content/browser/renderer_host/render_process_host_impl.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/content_browser_client.h"
+#include "content/public/browser/desktop_media_id.h"
 #include "content/public/browser/media_observer.h"
 #include "content/public/browser/media_request_state.h"
 #include "content/public/browser/render_process_host.h"
@@ -2052,15 +2053,24 @@ void MediaStreamManager::OnMediaStreamUIWindowId(MediaStreamType video_type,
   if (video_type != MEDIA_DESKTOP_VIDEO_CAPTURE)
     return;
 
-  // Pass along for desktop screen and window capturing.
+  // Pass along for desktop screen and window capturing when
+  // DesktopCaptureDevice is used.
   for (const StreamDeviceInfo& device_info : devices) {
-    if (device_info.device.type == MEDIA_DESKTOP_VIDEO_CAPTURE &&
-        !WebContentsMediaCaptureId::IsWebContentsDeviceId(
-            device_info.device.id)) {
-      video_capture_manager_->SetDesktopCaptureWindowId(device_info.session_id,
-                                                        window_id);
-      break;
-    }
+    if (device_info.device.type != MEDIA_DESKTOP_VIDEO_CAPTURE)
+      continue;
+
+    DesktopMediaID media_id = DesktopMediaID::Parse(device_info.device.id);
+    // WebContentsVideoCaptureDevice is used for tab/webcontents.
+    if (media_id.type == DesktopMediaID::TYPE_WEB_CONTENTS)
+      continue;
+#if defined(USE_AURA)
+    // DesktopCaptureDevicAura is used when aura_id is valid.
+    if (media_id.aura_id > DesktopMediaID::kNullId)
+      continue;
+#endif
+    video_capture_manager_->SetDesktopCaptureWindowId(device_info.session_id,
+                                                      window_id);
+    break;
   }
 }