Add deferral to offscreen canvas

This mostly means that we need to be more explicit about when we
finalize frame. This is covered in prior CLs:
  https://chromium-review.googlesource.com/c/chromium/src/+/1848934
  https://chromium-review.googlesource.com/c/chromium/src/+/1814457
  Next step is to create a CanvasDeferral class that mixes in to the
generic CanvasRenderingContext class and manages deferral for all types
of canvases. Also, finding a way to get write pixels within deferral by
using putImageData would be nice.

Bug: 1002523
Change-Id: Ia5aa50e3082177df1f422d14fc7899f8d173a874
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1762467
Commit-Queue: Aaron Krajeski <aaronhk@chromium.org>
Commit-Queue: Fernando Serboncini <fserb@chromium.org>
Reviewed-by: Fernando Serboncini <fserb@chromium.org>
Reviewed-by: Kai Ninomiya <kainino@chromium.org>
Cr-Commit-Position: refs/heads/master@{#704738}

third_party/blink/renderer/core/html/canvas/canvas_rendering_context_host.cc

@@ -226,6 +226,12 @@ ScriptPromise CanvasRenderingContextHost::convertToBlob(
     return ScriptPromise();
   }
 
+  // It's possible that there are recorded commands that have not been resolved
+  // Finalize frame will be called in GetImage, but if there's no
+  // resourceProvider yet then the IsPaintable check will fail
+  if (RenderingContext())
+    RenderingContext()->FinalizeFrame();
+
   if (!this->IsPaintable() || Size().IsEmpty()) {
     error_msg << "The size of " << object_name << " is zero.";
     exception_state.ThrowDOMException(DOMExceptionCode::kIndexSizeError,
@@ -240,8 +246,6 @@ ScriptPromise CanvasRenderingContextHost::convertToBlob(
     return ScriptPromise();
   }
 
-  // It's possible that there are recorded commands that have not been resolved
-  RenderingContext()->FinalizeFrame();
   base::TimeTicks start_time = base::TimeTicks::Now();
   scoped_refptr<StaticBitmapImage> image_bitmap =
       RenderingContext()->GetImage(kPreferNoAcceleration);

third_party/blink/renderer/core/offscreencanvas/offscreen_canvas.cc

@@ -215,6 +215,8 @@ ScriptPromise OffscreenCanvas::CreateImageBitmap(
     EventTarget&,
     base::Optional<IntRect> crop_rect,
     const ImageBitmapOptions* options) {
+  if (context_)
+    context_->FinalizeFrame();
   return ImageBitmapSource::FulfillImageBitmap(
       script_state,
       IsPaintable() ? ImageBitmap::Create(this, crop_rect, options) : nullptr);

third_party/blink/renderer/modules/canvas/offscreencanvas2d/offscreen_canvas_rendering_context_2d.cc

@@ -5,6 +5,7 @@
 #include "third_party/blink/renderer/modules/canvas/offscreencanvas2d/offscreen_canvas_rendering_context_2d.h"
 
 #include "base/metrics/histogram_functions.h"
+#include "third_party/blink/public/common/features.h"
 #include "third_party/blink/renderer/bindings/modules/v8/offscreen_rendering_context.h"
 #include "third_party/blink/renderer/core/css/offscreen_font_selector.h"
 #include "third_party/blink/renderer/core/css/parser/css_parser.h"
@@ -19,6 +20,7 @@
 #include "third_party/blink/renderer/platform/graphics/canvas_resource_provider.h"
 #include "third_party/blink/renderer/platform/graphics/graphics_types.h"
 #include "third_party/blink/renderer/platform/graphics/paint/paint_canvas.h"
+#include "third_party/blink/renderer/platform/graphics/skia/skia_utils.h"
 #include "third_party/blink/renderer/platform/graphics/static_bitmap_image.h"
 #include "third_party/blink/renderer/platform/heap/heap.h"
 #include "third_party/blink/renderer/platform/text/bidi_text_run.h"
@@ -79,8 +81,26 @@ OffscreenCanvasRenderingContext2D::OffscreenCanvasRenderingContext2D(
     OffscreenCanvas* canvas,
     const CanvasContextCreationAttributesCore& attrs)
     : CanvasRenderingContext(canvas, attrs),
+      is_deferral_enabled_(
+          base::FeatureList::IsEnabled(features::kCanvasAlwaysDeferral)),
       random_generator_((uint32_t)base::RandUint64()),
       bernoulli_distribution_(kUMASampleProbability) {
+  is_valid_size_ = IsValidImageSize(Host()->Size());
+
+  if (is_deferral_enabled_) {
+    StartRecording();
+
+    // Clear the background transparent or opaque. Similar code at
+    // CanvasResourceProvider::Clear().
+    if (IsCanvas2DBufferValid()) {
+      DCHECK(recorder_);
+      recorder_->getRecordingCanvas()->clear(
+          ColorParams().GetOpacityMode() == kOpaque ? SK_ColorBLACK
+                                                    : SK_ColorTRANSPARENT);
+      DidDraw();
+    }
+  }
+
   ExecutionContext* execution_context = canvas->GetTopExecutionContext();
   if (auto* document = DynamicTo<Document>(execution_context)) {
     Settings* settings = document->GetSettings();
@@ -104,10 +124,50 @@ void OffscreenCanvasRenderingContext2D::commit() {
   // TODO(fserb): consolidate this with PushFrame
   SkIRect damage_rect(dirty_rect_for_commit_);
   dirty_rect_for_commit_.setEmpty();
+  FinalizeFrame();
   Host()->Commit(ProduceCanvasResource(), damage_rect);
   GetOffscreenFontCache().PruneLocalFontCache(kMaxCachedFonts);
 }
 
+void OffscreenCanvasRenderingContext2D::StartRecording() {
+  DCHECK(is_deferral_enabled_);
+  recorder_ = std::make_unique<PaintRecorder>();
+
+  cc::PaintCanvas* canvas = recorder_->beginRecording(Width(), Height());
+  // Always save an initial frame, to support resetting the top level matrix
+  // and clip.
+  canvas->save();
+
+  RestoreMatrixClipStack(canvas);
+}
+
+void OffscreenCanvasRenderingContext2D::FlushRecording() {
+  if (!have_recorded_draw_commands_)
+    return;
+
+  {  // Make a new scope so that PaintRecord gets deleted and that gets timed
+    CanvasResourceProvider* resource_provider = GetCanvasResourceProvider();
+    cc::PaintCanvas* canvas = resource_provider->Canvas();
+    canvas->drawPicture(recorder_->finishRecordingAsPicture());
+    resource_provider->FlushSkia();
+  }
+  GetCanvasResourceProvider()->ReleaseLockedImages();
+
+  if (is_deferral_enabled_)
+    StartRecording();
+  have_recorded_draw_commands_ = false;
+}
+
+void OffscreenCanvasRenderingContext2D::FinalizeFrame() {
+  TRACE_EVENT0("blink", "OffscreenCanvasRenderingContext2D::FinalizeFrame");
+
+  // Make sure surface is ready for painting: fix the rendering mode now
+  // because it will be too late during the paint invalidation phase.
+  if (!GetOrCreateCanvasResourceProvider())
+    return;
+  FlushRecording();
+}
+
 // BaseRenderingContext2D implementation
 bool OffscreenCanvasRenderingContext2D::OriginClean() const {
   return Host()->OriginClean();
@@ -134,7 +194,13 @@ bool OffscreenCanvasRenderingContext2D::CanCreateCanvas2dResourceProvider()
     const {
   if (!Host() || Host()->Size().IsEmpty())
     return false;
-  return !!offscreenCanvasForBinding()->GetOrCreateResourceProvider();
+  return !!GetOrCreateCanvasResourceProvider();
+}
+
+CanvasResourceProvider*
+OffscreenCanvasRenderingContext2D::GetOrCreateCanvasResourceProvider() const {
+  // TODO(aaronhk) use Host() instead of offscreenCanvasForBinding() here
+  return offscreenCanvasForBinding()->GetOrCreateResourceProvider();
 }
 
 CanvasResourceProvider*
@@ -144,11 +210,15 @@ OffscreenCanvasRenderingContext2D::GetCanvasResourceProvider() const {
 void OffscreenCanvasRenderingContext2D::Reset() {
   Host()->DiscardResourceProvider();
   BaseRenderingContext2D::Reset();
+  if (is_deferral_enabled_)
+    StartRecording();
+  // Because the host may have changed to a zero size
+  is_valid_size_ = IsValidImageSize(Host()->Size());
 }
 
 scoped_refptr<CanvasResource>
 OffscreenCanvasRenderingContext2D::ProduceCanvasResource() {
-  if (!CanCreateCanvas2dResourceProvider())
+  if (!GetOrCreateCanvasResourceProvider())
     return nullptr;
   scoped_refptr<CanvasResource> frame =
       GetCanvasResourceProvider()->ProduceCanvasResource();
@@ -164,6 +234,7 @@ bool OffscreenCanvasRenderingContext2D::PushFrame() {
     return false;
 
   SkIRect damage_rect(dirty_rect_for_commit_);
+  FinalizeFrame();
   bool ret = Host()->PushFrame(ProduceCanvasResource(), damage_rect);
   dirty_rect_for_commit_.setEmpty();
   GetOffscreenFontCache().PruneLocalFontCache(kMaxCachedFonts);
@@ -175,10 +246,9 @@ ImageBitmap* OffscreenCanvasRenderingContext2D::TransferToImageBitmap(
   WebFeature feature = WebFeature::kOffscreenCanvasTransferToImageBitmap2D;
   UseCounter::Count(ExecutionContext::From(script_state), feature);
 
-  if (!CanCreateCanvas2dResourceProvider())
+  if (!GetOrCreateCanvasResourceProvider())
     return nullptr;
-  scoped_refptr<StaticBitmapImage> image =
-      GetCanvasResourceProvider()->Snapshot();
+  scoped_refptr<StaticBitmapImage> image = GetImage(kPreferAcceleration);
   if (!image)
     return nullptr;
   image->SetOriginClean(this->OriginClean());
@@ -193,12 +263,21 @@ ImageBitmap* OffscreenCanvasRenderingContext2D::TransferToImageBitmap(
       return nullptr;
     }
   }
-  Host()->DiscardResourceProvider();  // "Transfer" means no retained buffer.
+
+  // "Transfer" means no retained buffer. Matrix transformations need to be
+  // preserved though.
+  Host()->DiscardResourceProvider();
+  if (is_deferral_enabled_) {
+    recorder_->getRecordingCanvas()->restore();
+    recorder_->getRecordingCanvas()->save();
+  }
+
   return ImageBitmap::Create(std::move(image));
 }
 
 scoped_refptr<StaticBitmapImage> OffscreenCanvasRenderingContext2D::GetImage(
     AccelerationHint hint) {
+  FinalizeFrame();
   if (!IsPaintable())
     return nullptr;
   scoped_refptr<StaticBitmapImage> image =
@@ -219,6 +298,10 @@ bool OffscreenCanvasRenderingContext2D::ParseColorOrCurrentColor(
 }
 
 cc::PaintCanvas* OffscreenCanvasRenderingContext2D::DrawingCanvas() const {
+  if (!is_valid_size_)
+    return nullptr;
+  if (is_deferral_enabled_)
+    return recorder_->getRecordingCanvas();
   if (!CanCreateCanvas2dResourceProvider())
     return nullptr;
   return GetCanvasResourceProvider()->Canvas();
@@ -226,17 +309,25 @@ cc::PaintCanvas* OffscreenCanvasRenderingContext2D::DrawingCanvas() const {
 
 cc::PaintCanvas* OffscreenCanvasRenderingContext2D::ExistingDrawingCanvas()
     const {
+  if (!is_valid_size_)
+    return nullptr;
+  if (is_deferral_enabled_)
+    return recorder_->getRecordingCanvas();
   if (!IsPaintable())
     return nullptr;
   return GetCanvasResourceProvider()->Canvas();
 }
 
 void OffscreenCanvasRenderingContext2D::DidDraw() {
+  if (is_deferral_enabled_)
+    have_recorded_draw_commands_ = true;
   Host()->DidDraw();
   dirty_rect_for_commit_.setWH(Width(), Height());
 }
 
 void OffscreenCanvasRenderingContext2D::DidDraw(const SkIRect& dirty_rect) {
+  if (is_deferral_enabled_)
+    have_recorded_draw_commands_ = true;
   dirty_rect_for_commit_.join(dirty_rect);
   Host()->DidDraw(SkRect::Make(dirty_rect_for_commit_));
 }
@@ -289,6 +380,17 @@ bool OffscreenCanvasRenderingContext2D::WritePixels(
     int x,
     int y) {
   DCHECK(IsPaintable());
+
+  FinalizeFrame();
+  // WritePixels is not supported by deferral. Since we are directly rendering,
+  // we can't do deferral on top of the canvas. Disable deferral completely.
+  is_deferral_enabled_ = false;
+  have_recorded_draw_commands_ = false;
+  recorder_.reset();
+  // install the current matrix/clip stack onto the immediate canvas
+  if (GetOrCreateCanvasResourceProvider())
+    RestoreMatrixClipStack(GetCanvasResourceProvider()->Canvas());
+
   return offscreenCanvasForBinding()->ResourceProvider()->WritePixels(
       orig_info, pixels, row_bytes, x, y);
 }
@@ -519,8 +621,7 @@ void OffscreenCanvasRenderingContext2D::DrawTextInternal(
 
   Draw(
       [&font, &text_run_paint_info, &location](
-          cc::PaintCanvas* c, const PaintFlags* flags)  // draw lambda
-      {
+          cc::PaintCanvas* c, const PaintFlags* flags) /* draw lambda */ {
         font.DrawBidiText(c, text_run_paint_info, location,
                           Font::kUseFallbackIfFontNotReady, kCDeviceScaleFactor,
                           *flags);

third_party/blink/renderer/modules/canvas/offscreencanvas2d/offscreen_canvas_rendering_context_2d.h

@@ -12,6 +12,7 @@
 #include "third_party/blink/renderer/core/html/canvas/canvas_rendering_context.h"
 #include "third_party/blink/renderer/core/html/canvas/canvas_rendering_context_factory.h"
 #include "third_party/blink/renderer/modules/canvas/canvas2d/base_rendering_context_2d.h"
+#include "third_party/blink/renderer/platform/graphics/paint/paint_recorder.h"
 
 namespace blink {
 
@@ -95,6 +96,7 @@ class MODULES_EXPORT OffscreenCanvasRenderingContext2D final
   int Height() const final;
 
   bool CanCreateCanvas2dResourceProvider() const final;
+  CanvasResourceProvider* GetOrCreateCanvasResourceProvider() const;
   CanvasResourceProvider* GetCanvasResourceProvider() const;
 
   bool ParseColorOrCurrentColor(Color&, const String& color_string) const final;
@@ -120,6 +122,8 @@ class MODULES_EXPORT OffscreenCanvasRenderingContext2D final
 
   bool PushFrame() override;
 
+  bool HasRecordedDrawCommands() { return have_recorded_draw_commands_; }
+
  protected:
   void NeedsFinalizeFrame() override {
     CanvasRenderingContext::NeedsFinalizeFrame();
@@ -132,6 +136,13 @@ class MODULES_EXPORT OffscreenCanvasRenderingContext2D final
                    int y) override;
 
  private:
+  void StartRecording();
+  bool is_deferral_enabled_;
+  std::unique_ptr<PaintRecorder> recorder_;
+  bool have_recorded_draw_commands_;
+  void FinalizeFrame() final;
+  void FlushRecording();
+
   bool IsPaintable() const final;
   bool IsCanvas2DBufferValid() const override;
 
@@ -148,6 +159,8 @@ class MODULES_EXPORT OffscreenCanvasRenderingContext2D final
   CanvasPixelFormat PixelFormat() const override;
   SkIRect dirty_rect_for_commit_;
 
+  bool is_valid_size_ = false;
+
   std::mt19937 random_generator_;
   std::bernoulli_distribution bernoulli_distribution_;
 };