Capture canvas clicks as a form of XR input

This forms the basis of the WebXR Input system when used in Magic Window
mode.

Bug: 812287
Change-Id: Ibea7df8b173e378c563e011de23ab6bc5448af9f
Reviewed-on: https://chromium-review.googlesource.com/952230
Commit-Queue: Brandon Jones <bajones@chromium.org>
Reviewed-by: Bill Orr <billorr@chromium.org>
Reviewed-by: Michael Thiessen <mthiesse@chromium.org>
Cr-Commit-Position: refs/heads/master@{#543860}

third_party/WebKit/Source/modules/xr/BUILD.gn

@@ -8,6 +8,8 @@ blink_modules_sources("xr") {
   sources = [
     "XR.cpp",
     "XR.h",
+    "XRCanvasInputProvider.cpp",
+    "XRCanvasInputProvider.h",
     "XRCoordinateSystem.cpp",
     "XRCoordinateSystem.h",
     "XRDevice.cpp",

third_party/WebKit/Source/modules/xr/XRCanvasInputProvider.cpp

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "modules/xr/XRCanvasInputProvider.h"
+
+#include "core/events/MouseEvent.h"
+#include "core/html/canvas/HTMLCanvasElement.h"
+#include "modules/xr/XRDevice.h"
+#include "modules/xr/XRFrameProvider.h"
+#include "modules/xr/XRInputSource.h"
+#include "modules/xr/XRSession.h"
+#include "modules/xr/XRView.h"
+
+namespace blink {
+
+namespace {
+
+class XRCanvasInputEventListener : public EventListener {
+ public:
+  XRCanvasInputEventListener(XRCanvasInputProvider* input_provider)
+      : EventListener(kCPPEventListenerType), input_provider_(input_provider) {}
+
+  bool operator==(const EventListener& that) const override {
+    return this == &that;
+  }
+
+  void handleEvent(ExecutionContext* execution_context, Event* event) override {
+    if (!input_provider_->ShouldProcessEvents())
+      return;
+
+    if (event->type() == EventTypeNames::click) {
+      input_provider_->OnClick(ToMouseEvent(event));
+    }
+  }
+
+  void Trace(blink::Visitor* visitor) override {
+    visitor->Trace(input_provider_);
+    EventListener::Trace(visitor);
+  }
+
+ private:
+  Member<XRCanvasInputProvider> input_provider_;
+};
+
+}  // namespace
+
+XRCanvasInputProvider::XRCanvasInputProvider(XRSession* session,
+                                             HTMLCanvasElement* canvas)
+    : session_(session), canvas_(canvas) {
+  listener_ = new XRCanvasInputEventListener(this);
+  canvas->addEventListener(EventTypeNames::click, listener_);
+}
+
+XRCanvasInputProvider::~XRCanvasInputProvider() {
+  Stop();
+}
+
+void XRCanvasInputProvider::Stop() {
+  if (!listener_) {
+    return;
+  }
+  canvas_->removeEventListener(EventTypeNames::click, listener_);
+  canvas_ = nullptr;
+  listener_ = nullptr;
+}
+
+bool XRCanvasInputProvider::ShouldProcessEvents() {
+  // Don't process canvas gestures if there's an active exclusive session.
+  return !(session_->device()->frameProvider()->exclusive_session());
+}
+
+void XRCanvasInputProvider::OnClick(MouseEvent* event) {
+  UpdateInputSource(event);
+  session_->OnSelect(input_source_);
+  ClearInputSource();
+}
+
+XRInputSource* XRCanvasInputProvider::GetInputSource() {
+  return input_source_;
+}
+
+void XRCanvasInputProvider::UpdateInputSource(MouseEvent* event) {
+  if (!canvas_)
+    return;
+
+  if (!input_source_) {
+    input_source_ = new XRInputSource(session_, 0);
+    input_source_->SetPointerOrigin(XRInputSource::kOriginScreen);
+  }
+
+  // Get the event location relative to the canvas element.
+  double element_x = event->pageX() - canvas_->OffsetLeft();
+  double element_y = event->pageY() - canvas_->OffsetTop();
+
+  // Unproject the event location into a pointer matrix. This takes the 2D
+  // position of the screen interaction and shoves it backwards through the
+  // projection matrix to get a 3D point in space, which is then returned in
+  // matrix form so we can use it as an XRInputSource's pointerMatrix.
+  XRView* view = session_->views()[0];
+  std::unique_ptr<TransformationMatrix> pointer_transform_matrix =
+      view->UnprojectPointer(element_x, element_y, canvas_->OffsetWidth(),
+                             canvas_->OffsetHeight());
+
+  // Update the input source's pointer matrix.
+  input_source_->SetPointerTransformMatrix(std::move(pointer_transform_matrix));
+}
+
+void XRCanvasInputProvider::ClearInputSource() {
+  input_source_ = nullptr;
+}
+
+void XRCanvasInputProvider::Trace(blink::Visitor* visitor) {
+  visitor->Trace(session_);
+  visitor->Trace(canvas_);
+  visitor->Trace(listener_);
+  visitor->Trace(input_source_);
+}
+
+void XRCanvasInputProvider::TraceWrappers(
+    const blink::ScriptWrappableVisitor* visitor) const {
+  visitor->TraceWrappers(input_source_);
+}
+
+}  // namespace blink

third_party/WebKit/Source/modules/xr/XRCanvasInputProvider.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef XRCanvasInputProvider_h
+#define XRCanvasInputProvider_h
+
+#include "platform/bindings/ScriptWrappable.h"
+#include "platform/bindings/TraceWrapperMember.h"
+#include "platform/heap/Handle.h"
+#include "platform/transforms/TransformationMatrix.h"
+#include "platform/wtf/Forward.h"
+#include "platform/wtf/text/WTFString.h"
+
+namespace blink {
+
+class EventListener;
+class HTMLCanvasElement;
+class MouseEvent;
+class XRInputSource;
+class XRSession;
+
+class XRCanvasInputProvider
+    : public GarbageCollectedFinalized<XRCanvasInputProvider>,
+      public TraceWrapperBase {
+ public:
+  XRCanvasInputProvider(XRSession*, HTMLCanvasElement*);
+  virtual ~XRCanvasInputProvider();
+
+  XRSession* session() const { return session_; }
+
+  // Remove all event listeners.
+  void Stop();
+
+  bool ShouldProcessEvents();
+
+  void OnClick(MouseEvent*);
+
+  XRInputSource* GetInputSource();
+
+  virtual void Trace(blink::Visitor*);
+  virtual void TraceWrappers(const blink::ScriptWrappableVisitor*) const;
+  const char* NameInHeapSnapshot() const override {
+    return "XRCanvasInputProvider";
+  }
+
+ private:
+  void UpdateInputSource(MouseEvent*);
+  void ClearInputSource();
+
+  const Member<XRSession> session_;
+  Member<HTMLCanvasElement> canvas_;
+  Member<EventListener> listener_;
+  TraceWrapperMember<XRInputSource> input_source_;
+};
+
+}  // namespace blink
+
+#endif  // XRCanvasInputProvider_h

third_party/WebKit/Source/modules/xr/XRSession.cpp

@@ -13,6 +13,7 @@
 #include "core/resize_observer/ResizeObserverEntry.h"
 #include "modules/EventTargetModules.h"
 #include "modules/xr/XR.h"
+#include "modules/xr/XRCanvasInputProvider.h"
 #include "modules/xr/XRDevice.h"
 #include "modules/xr/XRFrameOfReference.h"
 #include "modules/xr/XRFrameOfReferenceOptions.h"
@@ -97,6 +98,11 @@ XRSession::XRSession(XRDevice* device,
           canvas->GetDocument(), new XRSessionResizeObserverDelegate(this));
       resize_observer_->observe(canvas);
 
+      // Begin processing input events on the output context's canvas.
+      if (!exclusive_) {
+        canvas_input_provider_ = new XRCanvasInputProvider(this, canvas);
+      }
+
       // Get the initial canvas dimensions
       UpdateCanvasDimensions(canvas);
     }
@@ -198,6 +204,22 @@ void XRSession::cancelAnimationFrame(int id) {
   callback_collection_.CancelCallback(id);
 }
 
+HeapVector<Member<XRInputSource>> XRSession::getInputSources() const {
+  HeapVector<Member<XRInputSource>> source_array;
+  for (const auto& input_source : input_sources_.Values()) {
+    source_array.push_back(input_source);
+  }
+
+  if (canvas_input_provider_) {
+    XRInputSource* input_source = canvas_input_provider_->GetInputSource();
+    if (input_source) {
+      source_array.push_back(input_source);
+    }
+  }
+
+  return source_array;
+}
+
 ScriptPromise XRSession::end(ScriptState* script_state) {
   // Don't allow a session to end twice.
   if (ended_) {
@@ -222,6 +244,11 @@ void XRSession::ForceEnd() {
   ended_ = true;
   pending_frame_ = false;
 
+  if (canvas_input_provider_) {
+    canvas_input_provider_->Stop();
+    canvas_input_provider_ = nullptr;
+  }
+
   // If this session is the active exclusive session for the device, notify the
   // frameProvider that it's ended.
   if (device_->frameProvider()->exclusive_session() == this) {
@@ -233,7 +260,7 @@ void XRSession::ForceEnd() {
 
 DoubleSize XRSession::IdealFramebufferSize() const {
   if (!exclusive_) {
-    return DoubleSize(output_width_, output_height_);
+    return OutputCanvasSize();
   }
 
   double width = device_->xrDisplayInfoPtr()->leftEye->renderWidth +
@@ -243,6 +270,14 @@ DoubleSize XRSession::IdealFramebufferSize() const {
   return DoubleSize(width, height);
 }
 
+DoubleSize XRSession::OutputCanvasSize() const {
+  if (!output_context_) {
+    return DoubleSize();
+  }
+
+  return DoubleSize(output_width_, output_height_);
+}
+
 void XRSession::OnFocus() {
   if (!blurred_)
     return;
@@ -529,6 +564,7 @@ void XRSession::Trace(blink::Visitor* visitor) {
   visitor->Trace(views_);
   visitor->Trace(input_sources_);
   visitor->Trace(resize_observer_);
+  visitor->Trace(canvas_input_provider_);
   visitor->Trace(callback_collection_);
   EventTargetWithInlineData::Trace(visitor);
 }

third_party/WebKit/Source/modules/xr/XRSession.h

@@ -22,6 +22,7 @@ namespace blink {
 class Element;
 class ResizeObserver;
 class V8XRFrameRequestCallback;
+class XRCanvasInputProvider;
 class XRDevice;
 class XRFrameOfReferenceOptions;
 class XRInputSourceEvent;
@@ -70,13 +71,7 @@ class XRSession final : public EventTargetWithInlineData {
   using InputSourceMap =
       HeapHashMap<uint32_t, TraceWrapperMember<XRInputSource>>;
 
-  HeapVector<Member<XRInputSource>> getInputSources() const {
-    HeapVector<Member<XRInputSource>> source_array;
-    for (const auto& input_source : input_sources_.Values()) {
-      source_array.push_back(input_source);
-    }
-    return source_array;
-  }
+  HeapVector<Member<XRInputSource>> getInputSources() const;
 
   // Called by JavaScript to manually end the session.
   ScriptPromise end(ScriptState*);
@@ -96,6 +91,10 @@ class XRSession final : public EventTargetWithInlineData {
   // as the size which gives 1:1 pixel ratio at the center of the user's view.
   DoubleSize IdealFramebufferSize() const;
 
+  // Reports the size of the output context's, if one is available. If not
+  // reports (0, 0);
+  DoubleSize OutputCanvasSize() const;
+
   // EventTarget overrides.
   ExecutionContext* GetExecutionContext() const override;
   const AtomicString& InterfaceName() const override;
@@ -109,6 +108,10 @@ class XRSession final : public EventTargetWithInlineData {
 
   const HeapVector<Member<XRView>>& views();
 
+  void OnSelectStart(XRInputSource*);
+  void OnSelectEnd(XRInputSource*);
+  void OnSelect(XRInputSource*);
+
   void Trace(blink::Visitor*) override;
   virtual void TraceWrappers(const blink::ScriptWrappableVisitor*) const;
 
@@ -121,9 +124,6 @@ class XRSession final : public EventTargetWithInlineData {
   void UpdateInputSourceState(
       XRInputSource*,
       const device::mojom::blink::XRInputSourceStatePtr&);
-  void OnSelectStart(XRInputSource*);
-  void OnSelectEnd(XRInputSource*);
-  void OnSelect(XRInputSource*);
   XRInputSourceEvent* CreateInputSourceEvent(const AtomicString&,
                                              XRInputSource*);
 
@@ -134,6 +134,7 @@ class XRSession final : public EventTargetWithInlineData {
   HeapVector<Member<XRView>> views_;
   InputSourceMap input_sources_;
   Member<ResizeObserver> resize_observer_;
+  Member<XRCanvasInputProvider> canvas_input_provider_;
 
   XRFrameRequestCallbackCollection callback_collection_;
   std::unique_ptr<TransformationMatrix> base_pose_matrix_;

third_party/WebKit/Source/modules/xr/XRView.cpp

@@ -52,6 +52,8 @@ void XRView::UpdateProjectionMatrixFromFoV(float up_rad,
   out[13] = 0.0f;
   out[14] = (2.0f * far_depth * near_depth) * inv_nf;
   out[15] = 0.0f;
+
+  inv_projection_dirty_ = true;
 }
 
 void XRView::UpdateProjectionMatrixFromAspect(float fovy,
@@ -78,12 +80,68 @@ void XRView::UpdateProjectionMatrixFromAspect(float fovy,
   out[13] = 0.0f;
   out[14] = (2.0f * far_depth * near_depth) * inv_nf;
   out[15] = 0.0f;
+
+  inv_projection_dirty_ = true;
 }
 
 void XRView::UpdateOffset(float x, float y, float z) {
   offset_.Set(x, y, z);
 }
 
+std::unique_ptr<TransformationMatrix> XRView::UnprojectPointer(
+    double x,
+    double y,
+    double canvas_width,
+    double canvas_height) {
+  // Recompute the inverse projection matrix if needed.
+  if (inv_projection_dirty_) {
+    float* m = projection_matrix_->Data();
+    std::unique_ptr<TransformationMatrix> projection =
+        TransformationMatrix::Create(m[0], m[1], m[2], m[3], m[4], m[5], m[6],
+                                     m[7], m[8], m[9], m[10], m[11], m[12],
+                                     m[13], m[14], m[15]);
+    inv_projection_ = TransformationMatrix::Create(projection->Inverse());
+    inv_projection_dirty_ = false;
+  }
+
+  // Transform the x/y coordinate into WebGL normalized device coordinates.
+  // Z coordinate of -1 means the point will be projected onto the projection
+  // matrix near plane.
+  FloatPoint3D point_in_projection_space(
+      x / canvas_width * 2.0 - 1.0,
+      (canvas_height - y) / canvas_height * 2.0 - 1.0, -1.0);
+
+  FloatPoint3D point_in_view_space =
+      inv_projection_->MapPoint(point_in_projection_space);
+
+  const FloatPoint3D kOrigin(0.0, 0.0, 0.0);
+  const FloatPoint3D kUp(0.0, 1.0, 0.0);
+
+  // Generate a "Look At" matrix
+  FloatPoint3D z_axis = kOrigin - point_in_view_space;
+  z_axis.Normalize();
+
+  FloatPoint3D x_axis = kUp.Cross(z_axis);
+  x_axis.Normalize();
+
+  FloatPoint3D y_axis = z_axis.Cross(x_axis);
+  y_axis.Normalize();
+
+  // TODO(bajones): There's probably a more efficent way to do this?
+  TransformationMatrix inv_pointer(x_axis.X(), y_axis.X(), z_axis.X(), 0.0,
+                                   x_axis.Y(), y_axis.Y(), z_axis.Y(), 0.0,
+                                   x_axis.Z(), y_axis.Z(), z_axis.Z(), 0.0, 0.0,
+                                   0.0, 0.0, 1.0);
+  inv_pointer.Translate3d(-point_in_view_space.X(), -point_in_view_space.Y(),
+                          -point_in_view_space.Z());
+
+  // LookAt matrices are view matrices (inverted), so invert before returning.
+  std::unique_ptr<TransformationMatrix> pointer =
+      TransformationMatrix::Create(inv_pointer.Inverse());
+
+  return pointer;
+}
+
 void XRView::Trace(blink::Visitor* visitor) {
   visitor->Trace(session_);
   visitor->Trace(projection_matrix_);

third_party/WebKit/Source/modules/xr/XRView.h

@@ -43,6 +43,11 @@ class XRView final : public ScriptWrappable {
                                         float near_depth,
                                         float far_depth);
 
+  std::unique_ptr<TransformationMatrix> UnprojectPointer(double x,
+                                                         double y,
+                                                         double canvas_width,
+                                                         double canvas_height);
+
   // TODO(bajones): Should eventually represent this as a full transform.
   const FloatPoint3D& offset() const { return offset_; }
   void UpdateOffset(float x, float y, float z);
@@ -55,6 +60,8 @@ class XRView final : public ScriptWrappable {
   Member<XRSession> session_;
   Member<DOMFloat32Array> projection_matrix_;
   FloatPoint3D offset_;
+  std::unique_ptr<TransformationMatrix> inv_projection_;
+  bool inv_projection_dirty_ = true;
 };
 
 }  // namespace blink