CrOS: Implement multipaste virtual keyboard api functions

This change implements the functions added to the virtual keyboard
private api for interacting with the clipboard history. The functions
implemented are GetClipboardHistory, PasteClipboardItem, and
DeleteClipboardItem.

Bug: 1099318
Change-Id: I8d19947def36359164bdd97f55a182f6fac30d73
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2569100Reviewed-by: Darren Shen <shend@chromium.org>
Reviewed-by: Devlin <rdevlin.cronin@chromium.org>
Reviewed-by: Alex Newcomer <newcomer@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Commit-Queue: Matthew Mourgos <mmourgos@chromium.org>
Cr-Commit-Position: refs/heads/master@{#840387}

ash/clipboard/clipboard_history_controller_impl.cc

@@ -13,9 +13,13 @@
 #include "ash/clipboard/clipboard_nudge_controller.h"
 #include "ash/clipboard/scoped_clipboard_history_pause_impl.h"
 #include "ash/public/cpp/clipboard_image_model_factory.h"
+#include "ash/public/cpp/file_icon_util.h"
 #include "ash/public/cpp/window_tree_host_lookup.h"
 #include "ash/resources/vector_icons/vector_icons.h"
 #include "ash/shell.h"
+#include "ash/style/ash_color_provider.h"
+#include "base/base64.h"
+#include "base/files/file_path.h"
 #include "base/location.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/utf_string_conversions.h"
@@ -32,11 +36,13 @@
 #include "ui/base/models/menu_separator_types.h"
 #include "ui/base/models/simple_menu_model.h"
 #include "ui/base/resource/resource_bundle.h"
+#include "ui/base/webui/web_ui_util.h"
 #include "ui/display/screen.h"
 #include "ui/events/event.h"
 #include "ui/events/event_constants.h"
 #include "ui/events/keycodes/keyboard_codes_posix.h"
 #include "ui/events/types/event_type.h"
+#include "ui/gfx/codec/png_codec.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/strings/grit/ui_strings.h"
 #include "ui/views/controls/menu/menu_controller.h"
@@ -45,6 +51,9 @@ namespace ash {
 
 namespace {
 
+constexpr char kImageDataKey[] = "imageData";
+constexpr char kTextDataKey[] = "textData";
+
 ui::ClipboardNonBacked* GetClipboard() {
   auto* clipboard = ui::ClipboardNonBacked::GetForCurrentThread();
   DCHECK(clipboard);
@@ -269,6 +278,82 @@ ClipboardHistoryControllerImpl::CreateScopedPause() {
       clipboard_history_.get());
 }
 
+base::Value ClipboardHistoryControllerImpl::GetHistoryValues(
+    const std::set<std::string>& item_id_filter) const {
+  base::Value item_results(base::Value::Type::LIST);
+
+  // Get the clipboard data for each clipboard history item.
+  for (const auto& item : history()->GetItems()) {
+    // If the |item_id_filter| contains values, then only return the clipboard
+    // items included in it.
+    if (!item_id_filter.empty() &&
+        item_id_filter.find(item.id().ToString()) == item_id_filter.end()) {
+      continue;
+    }
+
+    base::Value item_value(base::Value::Type::DICTIONARY);
+    switch (ash::ClipboardHistoryUtil::CalculateDisplayFormat(item.data())) {
+      case ash::ClipboardHistoryUtil::ClipboardHistoryDisplayFormat::kBitmap:
+        item_value.SetKey(
+            kImageDataKey,
+            base::Value(webui::GetBitmapDataUrl(item.data().bitmap())));
+        break;
+      case ash::ClipboardHistoryUtil::ClipboardHistoryDisplayFormat::kHtml: {
+        const SkBitmap& bitmap =
+            *(resource_manager_->GetImageModel(item).GetImage().ToSkBitmap());
+        item_value.SetKey(kImageDataKey,
+                          base::Value(webui::GetBitmapDataUrl(bitmap)));
+        break;
+      }
+      case ash::ClipboardHistoryUtil::ClipboardHistoryDisplayFormat::kText:
+        item_value.SetKey(kTextDataKey, base::Value(item.data().text()));
+        break;
+      case ash::ClipboardHistoryUtil::ClipboardHistoryDisplayFormat::kFile: {
+        item_value.SetKey(
+            kTextDataKey,
+            base::Value(base::UTF16ToUTF8(resource_manager_->GetLabel(item))));
+        gfx::ImageSkia image = GetIconForPath(
+            base::FilePath(item.data().text()),
+            ash::AshColorProvider::Get()->GetContentLayerColor(
+                AshColorProvider::ContentLayerType::kIconColorPrimary));
+        item_value.SetKey(
+            kImageDataKey,
+            base::Value(webui::GetBitmapDataUrl(*image.bitmap())));
+        break;
+      }
+    }
+    item_value.SetKey("idToken", base::Value(item.id().ToString()));
+    item_results.Append(std::move(item_value));
+  }
+
+  return item_results;
+}
+
+bool ClipboardHistoryControllerImpl::PasteClipboardItemById(
+    const std::string& item_id) {
+  if (currently_pasting_)
+    return false;
+
+  for (const auto& item : history()->GetItems()) {
+    if (item.id().ToString() == item_id) {
+      PasteClipboardHistoryItem(item, /*paste_plain_text=*/false);
+      return true;
+    }
+  }
+  return false;
+}
+
+bool ClipboardHistoryControllerImpl::DeleteClipboardItemById(
+    const std::string& item_id) {
+  for (const auto& item : history()->GetItems()) {
+    if (item.id().ToString() == item_id) {
+      DeleteClipboardHistoryItem(item);
+      return true;
+    }
+  }
+  return false;
+}
+
 void ClipboardHistoryControllerImpl::OnClipboardHistoryCleared() {
   // Prevent clipboard contents getting restored if the Clipboard is cleared
   // soon after a `PasteMenuItemData()`.
@@ -332,6 +417,12 @@ void ClipboardHistoryControllerImpl::PasteMenuItemData(int command_id,
   const ClipboardHistoryItem& selected_item =
       context_menu_->GetItemFromCommandId(command_id);
 
+  PasteClipboardHistoryItem(selected_item, paste_plain_text);
+}
+
+void ClipboardHistoryControllerImpl::PasteClipboardHistoryItem(
+    const ClipboardHistoryItem& item,
+    bool paste_plain_text) {
   auto* clipboard = GetClipboard();
   std::unique_ptr<ui::ClipboardData> original_data;
 
@@ -339,19 +430,19 @@ void ClipboardHistoryControllerImpl::PasteMenuItemData(int command_id,
   // we can paste the selected history item.
   ui::DataTransferEndpoint data_dst(ui::EndpointType::kClipboardHistory);
   if (paste_plain_text ||
-      selected_item.data() != *clipboard->GetClipboardData(&data_dst)) {
+      item.data() != *clipboard->GetClipboardData(&data_dst)) {
     std::unique_ptr<ui::ClipboardData> temp_data;
     if (paste_plain_text) {
       // When the shift key is pressed, we only paste plain text.
       temp_data = std::make_unique<ui::ClipboardData>();
-      temp_data->set_text(selected_item.data().text());
-      ui::DataTransferEndpoint* data_src = selected_item.data().source();
+      temp_data->set_text(item.data().text());
+      ui::DataTransferEndpoint* data_src = item.data().source();
       if (data_src) {
         temp_data->set_source(
             std::make_unique<ui::DataTransferEndpoint>(*data_src));
       }
     } else {
-      temp_data = std::make_unique<ui::ClipboardData>(selected_item.data());
+      temp_data = std::make_unique<ui::ClipboardData>(item.data());
     }
 
     // Pause clipboard history when manipulating the clipboard for a paste.
@@ -367,6 +458,8 @@ void ClipboardHistoryControllerImpl::PasteMenuItemData(int command_id,
   if (!original_data)
     return;
 
+  currently_pasting_ = true;
+
   // Replace the original item back on top of the clipboard. Some apps take a
   // long time to receive the paste event, also some apps will read from the
   // clipboard multiple times per paste. Wait a bit before replacing the item
@@ -382,6 +475,7 @@ void ClipboardHistoryControllerImpl::PasteMenuItemData(int command_id,
             // should actually be opaque to the user.
             std::unique_ptr<ScopedClipboardHistoryPauseImpl> scoped_pause;
             if (weak_ptr) {
+              weak_ptr->currently_pasting_ = false;
               scoped_pause = std::make_unique<ScopedClipboardHistoryPauseImpl>(
                   weak_ptr->clipboard_history_.get());
             }
@@ -413,8 +507,7 @@ void ClipboardHistoryControllerImpl::DeleteItemWithCommandId(int command_id) {
   // process the key event.
   const auto& to_be_deleted_item =
       context_menu_->GetItemFromCommandId(command_id);
-  ClipboardHistoryUtil::RecordClipboardHistoryItemDeleted(to_be_deleted_item);
-  clipboard_history_->RemoveItemForId(to_be_deleted_item.id());
+  DeleteClipboardHistoryItem(to_be_deleted_item);
 
   // If the item to be deleted is the last one, close the whole menu.
   if (context_menu_->GetMenuItemsCount() == 1) {
@@ -426,6 +519,12 @@ void ClipboardHistoryControllerImpl::DeleteItemWithCommandId(int command_id) {
   context_menu_->RemoveMenuItemWithCommandId(command_id);
 }
 
+void ClipboardHistoryControllerImpl::DeleteClipboardHistoryItem(
+    const ClipboardHistoryItem& item) {
+  ClipboardHistoryUtil::RecordClipboardHistoryItemDeleted(item);
+  clipboard_history_->RemoveItemForId(item.id());
+}
+
 void ClipboardHistoryControllerImpl::AdvancePseudoFocus(bool reverse) {
   DCHECK(context_menu_);
   context_menu_->AdvancePseudoFocus(reverse);

ash/clipboard/clipboard_history_controller_impl.h

@@ -80,6 +80,10 @@ class ASH_EXPORT ClipboardHistoryControllerImpl
   void ShowMenu(const gfx::Rect& anchor_rect,
                 ui::MenuSourceType source_type) override;
   std::unique_ptr<ScopedClipboardHistoryPause> CreateScopedPause() override;
+  base::Value GetHistoryValues(
+      const std::set<std::string>& item_id_filter) const override;
+  bool PasteClipboardItemById(const std::string& item_id) override;
+  bool DeleteClipboardItemById(const std::string& item_id) override;
 
   // ClipboardHistory::Observer:
   void OnClipboardHistoryCleared() override;
@@ -94,6 +98,10 @@ class ASH_EXPORT ClipboardHistoryControllerImpl
   // clipboard data should be pasted.
   void PasteMenuItemData(int command_id, bool paste_plain_text);
 
+  // Pastes the specified clipboard history item.
+  void PasteClipboardHistoryItem(const ClipboardHistoryItem& item,
+                                 bool paste_plain_text);
+
   // Delete the menu item being selected and its corresponding data. If no item
   // is selected, do nothing.
   void DeleteSelectedMenuItemIfAny();
@@ -101,6 +109,9 @@ class ASH_EXPORT ClipboardHistoryControllerImpl
   // Delete the menu item specified by `command_id` and its corresponding data.
   void DeleteItemWithCommandId(int command_id);
 
+  // Deletes the specified clipboard history item.
+  void DeleteClipboardHistoryItem(const ClipboardHistoryItem& item);
+
   // Advances the pseudo focus (backward if `reverse` is true).
   void AdvancePseudoFocus(bool reverse);
 
@@ -127,6 +138,9 @@ class ASH_EXPORT ClipboardHistoryControllerImpl
   // Controller that shows contextual nudges for multipaste.
   std::unique_ptr<ClipboardNudgeController> nudge_controller_;
 
+  // Whether a paste is currently being performed.
+  bool currently_pasting_ = false;
+
   base::WeakPtrFactory<ClipboardHistoryControllerImpl> weak_ptr_factory_{this};
 };
 

ash/public/cpp/clipboard_history_controller.h

@@ -6,17 +6,21 @@
 #define ASH_PUBLIC_CPP_CLIPBOARD_HISTORY_CONTROLLER_H_
 
 #include <memory>
+#include <set>
 
 #include "ash/public/cpp/ash_public_export.h"
 #include "base/observer_list_types.h"
 #include "ui/base/ui_base_types.h"
 
+namespace base {
+class Value;
+}  // namespace base
+
 namespace gfx {
 class Rect;
 }  // namespace gfx
 
 namespace ash {
-
 class ScopedClipboardHistoryPause;
 
 // An interface implemented in Ash to enable the Chrome side to show the
@@ -49,6 +53,18 @@ class ASH_PUBLIC_EXPORT ClipboardHistoryController {
   // its lifetime.
   virtual std::unique_ptr<ScopedClipboardHistoryPause> CreateScopedPause() = 0;
 
+  // Returns the history which tracks what is being copied to the clipboard.
+  // Only the items listed in |item_id_filter| are returned. If |item_id_filter|
+  // is empty, then all items in the history are returned.
+  virtual base::Value GetHistoryValues(
+      const std::set<std::string>& item_id_filter) const = 0;
+
+  // Pastes the clipboard item specified by the item id.
+  virtual bool PasteClipboardItemById(const std::string& item_id) = 0;
+
+  // Deletes the clipboard item specified by the item id.
+  virtual bool DeleteClipboardItemById(const std::string& item_id) = 0;
+
  protected:
   ClipboardHistoryController();
   virtual ~ClipboardHistoryController();

chrome/browser/extensions/api/virtual_keyboard_private/chrome_virtual_keyboard_delegate.cc

@@ -8,6 +8,7 @@
 #include <string>
 #include <utility>
 
+#include "ash/public/cpp/clipboard_history_controller.h"
 #include "ash/public/cpp/keyboard/keyboard_switches.h"
 #include "ash/public/cpp/keyboard/keyboard_types.h"
 #include "base/bind.h"
@@ -332,6 +333,40 @@ bool ChromeVirtualKeyboardDelegate::SetWindowBoundsInScreen(
   return keyboard_client->SetWindowBoundsInScreen(bounds_in_screen);
 }
 
+void ChromeVirtualKeyboardDelegate::GetClipboardHistory(
+    const std::set<std::string>& item_ids_filter,
+    OnGetClipboardHistoryCallback get_history_callback) {
+  ash::ClipboardHistoryController* clipboard_history_controller =
+      ash::ClipboardHistoryController::Get();
+  if (!clipboard_history_controller)
+    return;
+
+  std::move(get_history_callback)
+      .Run(clipboard_history_controller->GetHistoryValues(item_ids_filter));
+}
+
+bool ChromeVirtualKeyboardDelegate::PasteClipboardItem(
+    const std::string& clipboard_item_id) {
+  ash::ClipboardHistoryController* clipboard_history_controller =
+      ash::ClipboardHistoryController::Get();
+  if (!clipboard_history_controller)
+    return false;
+
+  return clipboard_history_controller->PasteClipboardItemById(
+      clipboard_item_id);
+}
+
+bool ChromeVirtualKeyboardDelegate::DeleteClipboardItem(
+    const std::string& clipboard_item_id) {
+  ash::ClipboardHistoryController* clipboard_history_controller =
+      ash::ClipboardHistoryController::Get();
+  if (!clipboard_history_controller)
+    return false;
+
+  return clipboard_history_controller->DeleteClipboardItemById(
+      clipboard_item_id);
+}
+
 bool ChromeVirtualKeyboardDelegate::SetDraggableArea(
     const api::virtual_keyboard_private::Bounds& rect) {
   auto* keyboard_client = ChromeKeyboardControllerClient::Get();

chrome/browser/extensions/api/virtual_keyboard_private/chrome_virtual_keyboard_delegate.h

@@ -55,7 +55,11 @@ class ChromeVirtualKeyboardDelegate : public VirtualKeyboardDelegate {
   bool SetHitTestBounds(const std::vector<gfx::Rect>& bounds) override;
   bool SetAreaToRemainOnScreen(const gfx::Rect& bounds) override;
   bool SetWindowBoundsInScreen(const gfx::Rect& bounds_in_screen) override;
-
+  void GetClipboardHistory(
+      const std::set<std::string>& item_ids_filter,
+      OnGetClipboardHistoryCallback get_history_callback) override;
+  bool PasteClipboardItem(const std::string& clipboard_item_id) override;
+  bool DeleteClipboardItem(const std::string& clipboard_item_id) override;
   api::virtual_keyboard::FeatureRestrictions RestrictFeatures(
       const api::virtual_keyboard::RestrictFeatures::Params& params) override;
 

extensions/browser/api/virtual_keyboard_private/virtual_keyboard_delegate.h

@@ -28,6 +28,9 @@ class VirtualKeyboardDelegate {
 
   using OnSetModeCallback = base::OnceCallback<void(bool success)>;
 
+  using OnGetClipboardHistoryCallback =
+      base::Callback<void(base::Value history)>;
+
   // Fetch information about the preferred configuration of the keyboard. On
   // exit, |settings| is populated with the keyboard configuration if execution
   // is successful, otherwise it's set to nullptr.
@@ -106,6 +109,22 @@ class VirtualKeyboardDelegate {
   // Sets the bounds of the keyboard window in screen coordinates.
   virtual bool SetWindowBoundsInScreen(const gfx::Rect& bounds_in_screen) = 0;
 
+  // Calls the |get_history_callback| function and passes a value containing the
+  // current cipboard history items. Only clipboard items which have an id in
+  // the |item_ids_filter| are included. If the filter is empty then all
+  // clipboard items are included.
+  virtual void GetClipboardHistory(
+      const std::set<std::string>& item_ids_filter,
+      OnGetClipboardHistoryCallback get_history_callback) = 0;
+
+  // Paste a clipboard item from the clipboard history. Returns whether the
+  // paste is successful.
+  virtual bool PasteClipboardItem(const std::string& clipboard_item_id) = 0;
+
+  // Delete a clipboard item from the clipboard history. Returns whether the
+  // deletion is successful.
+  virtual bool DeleteClipboardItem(const std::string& clipboard_item_id) = 0;
+
   // Restricts the virtual keyboard IME features.
   // Returns the values which were updated.
   virtual api::virtual_keyboard::FeatureRestrictions RestrictFeatures(

extensions/browser/api/virtual_keyboard_private/virtual_keyboard_private_api.cc

@@ -32,6 +32,8 @@ const char kSetAreaToRemainOnScreenFailed[] =
 const char kSetWindowBoundsInScreenFailed[] =
     "Setting bounds of the virtual keyboard failed";
 const char kUnknownError[] = "Unknown error.";
+const char kPasteClipboardItemFailed[] = "Pasting the clipboard item failed";
+const char kDeleteClipboardItemFailed[] = "Deleting the clipboard item failed";
 
 namespace keyboard = api::virtual_keyboard_private;
 
@@ -247,7 +249,27 @@ VirtualKeyboardPrivateSetWindowBoundsInScreenFunction ::
 
 ExtensionFunction::ResponseAction
 VirtualKeyboardPrivateGetClipboardHistoryFunction::Run() {
-  return RespondNow(Error("Not implemented"));
+  std::unique_ptr<keyboard::GetClipboardHistory::Params> params =
+      keyboard::GetClipboardHistory::Params::Create(*args_);
+  EXTENSION_FUNCTION_VALIDATE(params);
+  std::set<std::string> item_id_filter;
+  if (params->options.item_ids) {
+    for (const auto& id : *(params->options.item_ids)) {
+      item_id_filter.insert(id);
+    }
+  }
+
+  delegate()->GetClipboardHistory(
+      item_id_filter,
+      base::Bind(&VirtualKeyboardPrivateGetClipboardHistoryFunction::
+                     OnGetClipboardHistory,
+                 this));
+  return did_respond() ? AlreadyResponded() : RespondLater();
+}
+
+void VirtualKeyboardPrivateGetClipboardHistoryFunction::OnGetClipboardHistory(
+    base::Value results) {
+  Respond(OneArgument(std::move(results)));
 }
 
 VirtualKeyboardPrivateGetClipboardHistoryFunction ::
@@ -255,7 +277,13 @@ VirtualKeyboardPrivateGetClipboardHistoryFunction ::
 
 ExtensionFunction::ResponseAction
 VirtualKeyboardPrivatePasteClipboardItemFunction::Run() {
-  return RespondNow(Error("Not implemented"));
+  std::unique_ptr<keyboard::PasteClipboardItem::Params> params =
+      keyboard::PasteClipboardItem::Params::Create(*args_);
+  EXTENSION_FUNCTION_VALIDATE(params);
+
+  if (!delegate()->PasteClipboardItem(params->item_id))
+    return RespondNow(Error(kPasteClipboardItemFailed));
+  return RespondNow(NoArguments());
 }
 
 VirtualKeyboardPrivatePasteClipboardItemFunction ::
@@ -263,7 +291,13 @@ VirtualKeyboardPrivatePasteClipboardItemFunction ::
 
 ExtensionFunction::ResponseAction
 VirtualKeyboardPrivateDeleteClipboardItemFunction::Run() {
-  return RespondNow(Error("Not implemented"));
+  std::unique_ptr<keyboard::DeleteClipboardItem::Params> params =
+      keyboard::DeleteClipboardItem::Params::Create(*args_);
+  EXTENSION_FUNCTION_VALIDATE(params);
+
+  if (!delegate()->DeleteClipboardItem(params->item_id))
+    return RespondNow(Error(kDeleteClipboardItemFailed));
+  return RespondNow(NoArguments());
 }
 
 VirtualKeyboardPrivateDeleteClipboardItemFunction ::

extensions/browser/api/virtual_keyboard_private/virtual_keyboard_private_api_unittest.cc

@@ -75,6 +75,16 @@ class MockVirtualKeyboardDelegate : public VirtualKeyboardDelegate {
   }
   const gfx::Rect& GetWindowBounds() { return window_bounds_; }
 
+  void GetClipboardHistory(
+      const std::set<std::string>& item_ids_filter,
+      OnGetClipboardHistoryCallback get_history_callback) override {}
+  bool PasteClipboardItem(const std::string& clipboard_item_id) override {
+    return false;
+  }
+  bool DeleteClipboardItem(const std::string& clipboard_item_id) override {
+    return false;
+  }
+
   api::virtual_keyboard::FeatureRestrictions RestrictFeatures(
       const api::virtual_keyboard::RestrictFeatures::Params& params) override {
     return api::virtual_keyboard::FeatureRestrictions();

extensions/shell/browser/shell_virtual_keyboard_delegate.cc

@@ -101,6 +101,24 @@ bool ShellVirtualKeyboardDelegate::SetWindowBoundsInScreen(
   return false;
 }
 
+void ShellVirtualKeyboardDelegate::GetClipboardHistory(
+    const std::set<std::string>& item_ids_filter,
+    OnGetClipboardHistoryCallback get_history_callback) {
+  NOTIMPLEMENTED();
+}
+
+bool ShellVirtualKeyboardDelegate::PasteClipboardItem(
+    const std::string& clipboard_item_id) {
+  NOTIMPLEMENTED();
+  return false;
+}
+
+bool ShellVirtualKeyboardDelegate::DeleteClipboardItem(
+    const std::string& clipboard_item_id) {
+  NOTIMPLEMENTED();
+  return false;
+}
+
 api::virtual_keyboard::FeatureRestrictions
 ShellVirtualKeyboardDelegate::RestrictFeatures(
     const api::virtual_keyboard::RestrictFeatures::Params& params) {

extensions/shell/browser/shell_virtual_keyboard_delegate.h

@@ -46,6 +46,11 @@ class ShellVirtualKeyboardDelegate : public VirtualKeyboardDelegate {
   bool SetHitTestBounds(const std::vector<gfx::Rect>& bounds) override;
   bool SetAreaToRemainOnScreen(const gfx::Rect& bounds) override;
   bool SetWindowBoundsInScreen(const gfx::Rect& bounds_in_screen) override;
+  void GetClipboardHistory(
+      const std::set<std::string>& item_ids_filter,
+      OnGetClipboardHistoryCallback get_history_callback) override;
+  bool PasteClipboardItem(const std::string& clipboard_item_id) override;
+  bool DeleteClipboardItem(const std::string& clipboard_item_id) override;
 
   api::virtual_keyboard::FeatureRestrictions RestrictFeatures(
       const api::virtual_keyboard::RestrictFeatures::Params& params) override;