Add Exif and IPC compatibility to JEA interface

This CL adds 2 things to JEA interface.

1. Exif. Currently HW JEA only encodes the image with APP0 JFIF segment.
After this CL, it accepts Exif buffer and would insert the APP1 segment
if Exif buffer is provided.

2. IPC compatibility.
IPC from ChromeOS side relies on the buffer ID to identify the encoded
results. The ID of VideoFrame is generated randomly at Chromium side
which ChromeOS doesn't know. The CL changes JEA interface to use output
buffer ID to identify each encode attemp.

BUG=819847
TEST=Verified that JEA can be used from ChromeOS side with Exif buffer.

Cq-Include-Trybots: master.tryserver.chromium.android:android_optional_gpu_tests_rel;master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win_optional_gpu_tests_rel
Change-Id: I87f13a435f9875999ca79dd5769fb925a9c59fb8
Reviewed-on: https://chromium-review.googlesource.com/954411
Commit-Queue: Sheng-hao Tsao <shenghao@google.com>
Reviewed-by: Ricky Liang <jcliang@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#542200}

media/filters/jpeg_parser.h

@@ -44,7 +44,8 @@ enum JpegMarker {
   JPEG_SOS = 0xDA,    // start of scan
   JPEG_DQT = 0xDB,    // define quantization table
   JPEG_DRI = 0xDD,    // define restart internal
-  JPEG_APP0 = 0xE0,   // start of application segment
+  JPEG_APP0 = 0xE0,   // start of application segment (APP0)
+  JPEG_APP1 = 0xE1,   // start of application segment (APP1)
   JPEG_MARKER_PREFIX = 0xFF,  // jpeg marker prefix
 };
 

media/gpu/jpeg_encode_accelerator_unittest.cc

@@ -254,9 +254,8 @@ class JpegClient : public JpegEncodeAccelerator::Client {
   void StartEncode(int32_t bitstream_buffer_id);
 
   // JpegEncodeAccelerator::Client implementation.
-  void VideoFrameReady(int video_frame_id,
-                       size_t encoded_picture_size) override;
-  void NotifyError(int video_frame_id,
+  void VideoFrameReady(int32_t buffer_id, size_t encoded_picture_size) override;
+  void NotifyError(int32_t buffer_id,
                    JpegEncodeAccelerator::Status status) override;
 
  private:
@@ -283,10 +282,8 @@ class JpegClient : public JpegEncodeAccelerator::Client {
   // JpegClient doesn't own |test_image_files_|.
   const std::vector<TestImageFile*>& test_image_files_;
 
-  // A map that stores HW encoding start timestamp for each video frame id.
-  std::map<int, base::TimeTicks> video_frame_id_to_start_time_;
-
-  std::map<int, TestImageFile*> video_frame_id_to_image_;
+  // A map that stores HW encoding start timestamp for each output buffer id.
+  std::map<int, base::TimeTicks> buffer_id_to_start_time_;
 
   std::unique_ptr<JpegEncodeAccelerator> encoder_;
   ClientState state_;
@@ -311,7 +308,6 @@ class JpegClient : public JpegEncodeAccelerator::Client {
 JpegClient::JpegClient(const std::vector<TestImageFile*>& test_image_files,
                        ClientStateNotification<ClientState>* note)
     : test_image_files_(test_image_files),
-      video_frame_id_to_image_(),
       state_(ClientState::CREATED),
       note_(note) {}
 
@@ -344,12 +340,12 @@ void JpegClient::DestroyJpegEncoder() {
   encoder_.reset();
 }
 
-void JpegClient::VideoFrameReady(int video_frame_id, size_t hw_encoded_size) {
+void JpegClient::VideoFrameReady(int32_t buffer_id, size_t hw_encoded_size) {
   base::TimeTicks hw_encode_end = base::TimeTicks::Now();
   base::TimeDelta elapsed_hw =
-      hw_encode_end - video_frame_id_to_start_time_[video_frame_id];
+      hw_encode_end - buffer_id_to_start_time_[buffer_id];
 
-  TestImageFile* test_image = video_frame_id_to_image_[video_frame_id];
+  TestImageFile* test_image = test_image_files_[buffer_id];
   size_t sw_encoded_size = 0;
   base::TimeDelta elapsed_sw;
   LOG_ASSERT(GetSoftwareEncodeResult(test_image->visible_size.width(),
@@ -457,10 +453,10 @@ double JpegClient::GetMeanAbsoluteDifference(uint8_t* hw_yuv_result,
   return total_difference / yuv_size;
 }
 
-void JpegClient::NotifyError(int video_frame_id,
+void JpegClient::NotifyError(int32_t buffer_id,
                              JpegEncodeAccelerator::Status status) {
-  LOG(ERROR) << "Notifying of error " << status << " for video frame id "
-             << video_frame_id;
+  LOG(ERROR) << "Notifying of error " << status << " for output buffer id "
+             << buffer_id;
   SetState(ClientState::ERROR);
   encoded_buffer_.reset(nullptr);
 }
@@ -537,10 +533,9 @@ void JpegClient::StartEncode(int32_t bitstream_buffer_id) {
 
   LOG_ASSERT(input_frame_.get());
 
-  video_frame_id_to_image_[input_frame_->unique_id()] = image_file;
-  video_frame_id_to_start_time_[input_frame_->unique_id()] =
-      base::TimeTicks::Now();
-  encoder_->Encode(input_frame_, kJpegDefaultQuality, *encoded_buffer_);
+  buffer_id_to_start_time_[bitstream_buffer_id] = base::TimeTicks::Now();
+  encoder_->Encode(input_frame_, kJpegDefaultQuality, nullptr,
+                   *encoded_buffer_);
 }
 
 class JpegEncodeAcceleratorTest : public ::testing::Test {

media/gpu/vaapi/vaapi_jpeg_encode_accelerator.h

@@ -41,19 +41,24 @@ class MEDIA_GPU_EXPORT VaapiJpegEncodeAccelerator
   // Currently only I420 format is supported for |video_frame|.
   void Encode(scoped_refptr<media::VideoFrame> video_frame,
               int quality,
-              const BitstreamBuffer& bitstream_buffer) override;
+              const BitstreamBuffer* exif_buffer,
+              const BitstreamBuffer& output_buffer) override;
 
  private:
   // An input video frame and the corresponding output buffer awaiting
   // consumption, provided by the client.
   struct EncodeRequest {
-    EncodeRequest(scoped_refptr<media::VideoFrame> video_frame,
-                  std::unique_ptr<SharedMemoryRegion> shm,
+    EncodeRequest(int32_t buffer_id,
+                  scoped_refptr<media::VideoFrame> video_frame,
+                  std::unique_ptr<SharedMemoryRegion> exif_shm,
+                  std::unique_ptr<SharedMemoryRegion> output_shm,
                   int quality);
     ~EncodeRequest();
 
+    int32_t buffer_id;
     scoped_refptr<media::VideoFrame> video_frame;
-    std::unique_ptr<SharedMemoryRegion> shm;
+    std::unique_ptr<SharedMemoryRegion> exif_shm;
+    std::unique_ptr<SharedMemoryRegion> output_shm;
     int quality;
 
     DISALLOW_COPY_AND_ASSIGN(EncodeRequest);
@@ -65,9 +70,9 @@ class MEDIA_GPU_EXPORT VaapiJpegEncodeAccelerator
 
   // Notifies the client that an error has occurred and encoding cannot
   // continue.
-  void NotifyError(int video_frame_id, Status status);
+  void NotifyError(int32_t buffer_id, Status status);
 
-  void VideoFrameReady(int video_frame_id, size_t encoded_picture_size);
+  void VideoFrameReady(int32_t buffer_id, size_t encoded_picture_size);
 
   // ChildThread's task runner.
   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;

media/gpu/vaapi/vaapi_jpeg_encoder.cc

@@ -32,10 +32,11 @@ const size_t kNumDcRunSizeBits = 16;
 const size_t kNumAcRunSizeBits = 16;
 const size_t kNumDcCodeWordsHuffVal = 12;
 const size_t kNumAcCodeWordsHuffVal = 162;
-const size_t kJpegHeaderSize = 83 + (kDctSize2 * 2) + (kNumDcRunSizeBits * 2) +
-                               (kNumDcCodeWordsHuffVal * 2) +
-                               (kNumAcRunSizeBits * 2) +
+const size_t kJpegDefaultHeaderSize =
+    67 + (kDctSize2 * 2) + (kNumDcRunSizeBits * 2) +
+    (kNumDcCodeWordsHuffVal * 2) + (kNumAcRunSizeBits * 2) +
     (kNumAcCodeWordsHuffVal * 2);
+const size_t kJFIFApp0Size = 16;
 
 const uint8_t kZigZag8x8[64] = {
     0,  1,  8,  16, 9,  2,  3,  10, 17, 24, 32, 25, 18, 11, 4,  5,
@@ -62,8 +63,6 @@ const JpegQuantizationTable kDefaultQuantTable[2] = {
     },
 };
 
-using JPEGHeader = uint8_t[kJpegHeaderSize];
-
 void FillPictureParameters(const gfx::Size& input_size,
                            int quality,
                            VABufferID output_buffer_id,
@@ -172,8 +171,10 @@ void FillSliceParameters(VAEncSliceParameterBufferJPEG* slice_param) {
 }
 
 size_t FillJpegHeader(const gfx::Size& input_size,
+                      const uint8_t* exif_buffer,
+                      size_t exif_buffer_size,
                       int quality,
-                      JPEGHeader& header) {
+                      uint8_t* header) {
   unsigned int width = input_size.width();
   unsigned int height = input_size.height();
 
@@ -184,8 +185,18 @@ size_t FillJpegHeader(const gfx::Size& input_size,
   memcpy(header, kSOI, sizeof(kSOI));
   idx += sizeof(kSOI);
 
+  if (exif_buffer_size > 0) {
+    // Application Segment for Exif data.
+    uint16_t exif_segment_size = static_cast<uint16_t>(exif_buffer_size + 2);
+    const uint8_t kAppSegment[] = {
+        0xFF, JPEG_APP1, static_cast<uint8_t>(exif_segment_size / 256),
+        static_cast<uint8_t>(exif_segment_size % 256)};
+    memcpy(header + idx, kAppSegment, sizeof(kAppSegment));
+    idx += sizeof(kAppSegment);
+    memcpy(header + idx, exif_buffer, exif_buffer_size);
+    idx += exif_buffer_size;
+  } else {
     // Application Segment - JFIF standard 1.01.
-  // TODO(shenghao): Use Exif (JPEG_APP1) instead.
     static const uint8_t kAppSegment[] = {
         0xFF, JPEG_APP0, 0x00,
         0x10,  // Segment length:16 (2-byte).
@@ -207,6 +218,7 @@ size_t FillJpegHeader(const gfx::Size& input_size,
     };
     memcpy(header + idx, kAppSegment, sizeof(kAppSegment));
     idx += sizeof(kAppSegment);
+  }
 
   if (quality <= 0) {
     quality = 1;
@@ -349,10 +361,12 @@ VaapiJpegEncoder::VaapiJpegEncoder(scoped_refptr<VaapiWrapper> vaapi_wrapper)
 VaapiJpegEncoder::~VaapiJpegEncoder() {}
 
 size_t VaapiJpegEncoder::GetMaxCodedBufferSize(const gfx::Size& size) {
-  return size.GetArea() * 3 / 2 + kJpegHeaderSize;
+  return size.GetArea() * 3 / 2 + kJpegDefaultHeaderSize;
 }
 
 bool VaapiJpegEncoder::Encode(const gfx::Size& input_size,
+                              const uint8_t* exif_buffer,
+                              size_t exif_buffer_size,
                               int quality,
                               VASurfaceID surface_id,
                               VABufferID output_buffer_id) {
@@ -402,8 +416,13 @@ bool VaapiJpegEncoder::Encode(const gfx::Size& input_size,
     return false;
   }
 
-  JPEGHeader header_data;
-  size_t length_in_bits = FillJpegHeader(input_size, quality, header_data);
+  std::vector<uint8_t> jpeg_header;
+  size_t jpeg_header_size = exif_buffer_size > 0
+                                ? kJpegDefaultHeaderSize + exif_buffer_size
+                                : kJpegDefaultHeaderSize + kJFIFApp0Size;
+  jpeg_header.resize(jpeg_header_size);
+  size_t length_in_bits = FillJpegHeader(
+      input_size, exif_buffer, exif_buffer_size, quality, jpeg_header.data());
 
   VAEncPackedHeaderParameterBuffer header_param;
   memset(&header_param, 0, sizeof(header_param));
@@ -416,7 +435,8 @@ bool VaapiJpegEncoder::Encode(const gfx::Size& input_size,
   }
 
   if (!vaapi_wrapper_->SubmitBuffer(VAEncPackedHeaderDataBufferType,
-                                    (length_in_bits + 7) / 8, header_data)) {
+                                    (length_in_bits + 7) / 8,
+                                    jpeg_header.data())) {
     return false;
   }
 

media/gpu/vaapi/vaapi_jpeg_encoder.h

@@ -32,6 +32,9 @@ class MEDIA_GPU_EXPORT VaapiJpegEncoder {
 
   // Encode a JPEG picture. It will fill VA-API parameters and call
   // corresponding VA-API methods according to |input_size|.
+  // |exif_buffer| contains the EXIF data that will be inserted to the JPEG
+  // image.
+  // |exif_buffer_size| is the size of |exif_buffer|.
   // |quality| is the JPEG image quality
   // |surface_id| is the VA surface that contains input image.
   // |output_buffer_id| is the ID of VA buffer that encoded image will be
@@ -39,6 +42,8 @@ class MEDIA_GPU_EXPORT VaapiJpegEncoder {
   // GetMaxCodedBufferSize().
   // Return false on failure.
   bool Encode(const gfx::Size& input_size,
+              const uint8_t* exif_buffer,
+              size_t exif_buffer_size,
               int quality,
               VASurfaceID surface_id,
               VABufferID output_buffer_id);

media/video/jpeg_encode_accelerator.h

@@ -48,11 +48,10 @@ class MEDIA_EXPORT JpegEncodeAccelerator {
    public:
     // Callback called after each successful Encode().
     // Parameters:
-    //  |video_frame_id| is |VideoFrame.unique_id()| of the input VideoFrame in
-    //  corresponding Encode() call.
+    //  |buffer_id| is |output_buffer.id()| of the corresponding Encode() call.
     //  |encoded_picture_size| is the actual size of encoded JPEG image in
     //  the BitstreamBuffer provided through encode().
-    virtual void VideoFrameReady(int video_frame_id,
+    virtual void VideoFrameReady(int32_t buffer_id,
                                  size_t encoded_picture_size) = 0;
 
     // Callback to notify errors. Client is responsible for destroying JEA when
@@ -60,10 +59,10 @@ class MEDIA_EXPORT JpegEncodeAccelerator {
     // is informed about the buffer that failed to encode and may continue
     // using the same instance of JEA.
     // Parameters:
-    //  |video_frame_id| is |video_frame_id.unique_id()| of the input VideoFrame
+    //  |buffer_id| is |output_buffer.id()| of the corresponding Encode() call
     //  that resulted in the error.
     //  |status| would be one of the values of Status except ENCODE_OK.
-    virtual void NotifyError(int video_frame_id, Status status) = 0;
+    virtual void NotifyError(int32_t buffer_id, Status status) = 0;
 
    protected:
     virtual ~Client() {}
@@ -88,18 +87,21 @@ class MEDIA_EXPORT JpegEncodeAccelerator {
 
   // Encodes the given |video_frame| that contains a YUV image. Client will
   // receive the encoded result in Client::VideoFrameReady() callback with the
-  // corresponding |unique_id_| in |video_frame|, or receive
+  // corresponding |output_buffer.id()|, or receive
   // Client::NotifyError() callback.
   // Parameters:
   //  |video_frame| contains the YUV image to be encoded.
   //  |quality| of JPEG image.
-  //  |bitstream_buffer| that contains output buffer for encoded result. Clients
+  //  |exif_buffer| contains Exif data to be inserted into JPEG image. If it's
+  //  nullptr, the JFIF APP0 segment will be inserted.
+  //  |output_buffer| that contains output buffer for encoded result. Clients
   //  should call GetMaxCodedBufferSize() and allocate the buffer accordingly.
   //  The buffer needs to be valid until VideoFrameReady() or NotifyError() is
   //  called.
   virtual void Encode(scoped_refptr<media::VideoFrame> video_frame,
                       int quality,
-                      const BitstreamBuffer& bitstream_buffer) = 0;
+                      const BitstreamBuffer* exif_buffer,
+                      const BitstreamBuffer& output_buffer) = 0;
 };
 
 }  // namespace media