Feed refactor: Proto conversion

Add StreamModelUpdateRequest and functions for converting wire protos into store protos

Bug: 1044139
Change-Id: I243ebabc331884500dec480c69d843d38546a34e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2020545
Commit-Queue: Ian Wells <iwells@chromium.org>
Reviewed-by: Dan H <harringtond@chromium.org>
Cr-Commit-Position: refs/heads/master@{#747983}

components/feed/core/proto/BUILD.gn

@@ -19,6 +19,7 @@ proto_library("proto") {
 proto_library("proto_v2") {
   proto_in_dir = "../../../../"
   sources = [
+    "libraries/api/internal/stream_data.proto",
     "ui/action/feed_action_payload.proto",
     "ui/action/ui_feed_action.proto",
     "ui/piet/accessibility.proto",
@@ -39,6 +40,7 @@ proto_library("proto_v2") {
     "ui/piet/text.proto",
     "ui/stream/stream_structure.proto",
     "v2/store.proto",
+    "v2/ui.proto",
     "wire/action_payload.proto",
     "wire/action_payload_for_test.proto",
     "wire/action_request.proto",

components/feed/core/proto/v2/ui_extensions.proto

-// Copyright 2020 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-syntax = "proto2";
-
-package feedui;
-
-option optimize_for = LITE_RUNTIME;
-
-import "components/feed/core/proto/ui/piet/actions.proto";
-import "components/feed/core/proto/v2/ui.proto";
-
-message ActionExtension {
-  extend components.feed.core.proto.ui.piet.Action {
-    optional feedui.Action chrome_ui_action_extension = 100002;
-  }
-}

components/feed/core/v2/BUILD.gn

@@ -27,6 +27,8 @@ source_set("feed_core_v2") {
     "stream_model/ephemeral_change.h",
     "stream_model/feature_tree.cc",
     "stream_model/feature_tree.h",
+    "stream_model_update_request.cc",
+    "stream_model_update_request.h",
   ]
   deps = [
     "//components/feed/core:feed_core",
@@ -55,10 +57,12 @@ source_set("core_unit_tests") {
     "feed_network_impl_unittest.cc",
     "feed_stream_unittest.cc",
     "stream_model_unittest.cc",
+    "stream_model_update_request_unittest.cc",
   ]
 
   deps = [
     ":feed_core_v2",
+    ":unit_tests_bundle_data",
     "//base",
     "//base/test:test_support",
     "//components/feed/core:feed_core",
@@ -74,3 +78,11 @@ source_set("core_unit_tests") {
     "//third_party/zlib/google:compression_utils",
   ]
 }
+
+bundle_data("unit_tests_bundle_data") {
+  visibility = [ ":core_unit_tests" ]
+  testonly = true
+  sources = [ "//components/test/data/feed/response.binarypb" ]
+  outputs = [ "{{bundle_resources_dir}}/" +
+              "{{source_root_relative_dir}}/{{source_file_part}}" ]
+}

components/feed/core/v2/stream_model_update_request.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_
+#define COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_
+
+#include <vector>
+#include "base/optional.h"
+#include "base/time/time.h"
+#include "components/feed/core/proto/v2/store.pb.h"
+#include "components/feed/core/proto/wire/response.pb.h"
+
+namespace feed {
+
+// Data for updating StreamModel. This can be sourced from the network or
+// persistent storage.
+struct StreamModelUpdateRequest {
+ public:
+  enum class Source {
+    kNetworkUpdate,
+    kInitialLoadFromStore,
+  };
+
+  StreamModelUpdateRequest();
+  ~StreamModelUpdateRequest();
+  StreamModelUpdateRequest(const StreamModelUpdateRequest&);
+  StreamModelUpdateRequest& operator=(const StreamModelUpdateRequest&);
+
+  // Whether this data originates is from the initial load of content from
+  // the local data store.
+  Source source = Source::kNetworkUpdate;
+
+  // The set of Contents marked UPDATE_OR_APPEND in the response, in the order
+  // in which they were received.
+  std::vector<feedstore::Content> content;
+
+  // Contains the root ContentId, tokens, a timestamp for when the most recent
+  // content was added, and a list of ContentIds for clusters in the response.
+  feedstore::StreamData stream_data;
+
+  // The set of StreamSharedStates marked UPDATE_OR_APPEND in the order in which
+  // they were received.
+  std::vector<feedstore::StreamSharedState> shared_states;
+
+  // If this data originates from the network, this is the server-reported time
+  // at which the request was fulfilled.
+  int64_t server_response_time;
+
+  // If this data originates from the network, this is the time taken by the
+  // server to produce the response.
+  base::TimeDelta response_time;
+};
+
+base::Optional<feedstore::DataOperation> TranslateDataOperation(
+    feedwire::DataOperation wire_operation);
+
+std::unique_ptr<StreamModelUpdateRequest> TranslateWireResponse(
+    feedwire::Response response,
+    base::TimeDelta response_time);
+
+}  // namespace feed
+
+#endif  // COMPONENTS_FEED_CORE_V2_STREAM_MODEL_UPDATE_REQUEST_H_

components/feed/core/v2/stream_model_update_request_unittest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/feed/core/v2/stream_model_update_request.h"
+
+#include "base/base_paths.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/time/time.h"
+#include "components/feed/core/proto/libraries/api/internal/stream_data.pb.h"
+#include "components/feed/core/proto/wire/feed_response.pb.h"
+#include "components/feed/core/proto/wire/response.pb.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace feed {
+namespace {
+
+const char kResponsePbPath[] = "components/test/data/feed/response.binarypb";
+constexpr base::TimeDelta kResponseTime = base::TimeDelta::FromSeconds(42);
+
+const int kExpectedStreamStructureCount = 34;
+const size_t kExpectedContentCount = 10;
+const size_t kExpectedSharedStateCount = 1;
+
+std::string ContentIdToString(const feedwire::ContentId& content_id) {
+  return "{content_domain: \"" + content_id.content_domain() +
+         "\", id: " + base::NumberToString(content_id.id()) + ", table: \"" +
+         content_id.table() + "\"}";
+}
+
+feedwire::Response TestWireResponse() {
+  // Read and parse response.binarypb.
+  base::FilePath response_file_path;
+  DCHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &response_file_path));
+  response_file_path = response_file_path.AppendASCII(kResponsePbPath);
+
+  DCHECK(base::PathExists(response_file_path))
+      << "Path doesn't exist: " << response_file_path;
+
+  std::string response_data;
+  DCHECK(base::ReadFileToString(response_file_path, &response_data));
+
+  feedwire::Response response;
+  DCHECK(response.ParseFromString(response_data));
+  return response;
+}
+
+}  // namespace
+
+// TODO(iwells): Test failure cases once the new protos are ready.
+
+TEST(StreamModelUpdateRequestTest, TranslateRealResponse) {
+  // Tests how proto translation works on a real response from the server.
+  //
+  // The response will periodically need to be updated as changes are made to
+  // the server. Update testdata/response.textproto and then run
+  // tools/generate_test_response_binarypb.sh.
+
+  feedwire::Response response = TestWireResponse();
+  feedwire::FeedResponse feed_response =
+      response.GetExtension(feedwire::FeedResponse::feed_response);
+  ASSERT_EQ(feed_response.data_operation_size(), kExpectedStreamStructureCount);
+
+  std::unique_ptr<StreamModelUpdateRequest> translated =
+      TranslateWireResponse(response, kResponseTime);
+
+  ASSERT_TRUE(translated);
+  ASSERT_EQ(translated->stream_data.structures_size(),
+            kExpectedStreamStructureCount);
+
+  const google::protobuf::RepeatedPtrField<feedstore::StreamStructure>&
+      structures = translated->stream_data.structures();
+
+  // Check CLEAR_ALL:
+  EXPECT_EQ(structures[0].operation(), feedstore::StreamStructure::CLEAR_ALL);
+
+  // Check UPDATE_OR_APPEND for a shared state:
+  EXPECT_EQ(structures[1].operation(),
+            feedstore::StreamStructure::UPDATE_OR_APPEND);
+  EXPECT_EQ(structures[1].type(), feedstore::StreamStructure::UNKNOWN_TYPE);
+  EXPECT_TRUE(structures[1].has_content_id());
+  ASSERT_TRUE(translated->shared_states.size() > 0);
+  EXPECT_EQ(ContentIdToString(translated->shared_states[0].content_id()),
+            ContentIdToString(structures[1].content_id()));
+  // TODO(iwells): More checks on shared_state here
+
+  // Check UPDATE_OR_APPEND for the root:
+  EXPECT_EQ(structures[2].operation(),
+            feedstore::StreamStructure::UPDATE_OR_APPEND);
+  EXPECT_EQ(structures[2].type(), feedstore::StreamStructure::STREAM);
+  EXPECT_TRUE(structures[2].has_content_id());
+  EXPECT_FALSE(structures[2].has_parent_id());
+
+  feedwire::ContentId root_content_id = structures[2].content_id();
+
+  // Content:
+  EXPECT_EQ(structures[3].operation(),
+            feedstore::StreamStructure::UPDATE_OR_APPEND);
+  EXPECT_EQ(structures[3].type(), feedstore::StreamStructure::CONTENT);
+  EXPECT_TRUE(structures[3].has_content_id());
+  EXPECT_TRUE(structures[3].has_parent_id());
+  EXPECT_TRUE(structures[3].has_content_info());
+  EXPECT_NE(structures[3].content_info().score(), 0.);
+  EXPECT_NE(structures[3].content_info().availability_time_seconds(), 0);
+  EXPECT_TRUE(structures[3].content_info().has_representation_data());
+  EXPECT_TRUE(structures[3].content_info().has_offline_metadata());
+
+  ASSERT_TRUE(translated->content.size() > 0);
+  EXPECT_EQ(ContentIdToString(translated->content[0].content_id()),
+            ContentIdToString(structures[3].content_id()));
+  // TODO: Check content.frame() once this is available.
+
+  // Non-content structures:
+  EXPECT_EQ(structures[4].operation(),
+            feedstore::StreamStructure::UPDATE_OR_APPEND);
+  EXPECT_EQ(structures[4].type(), feedstore::StreamStructure::CARD);
+  EXPECT_TRUE(structures[4].has_content_id());
+  EXPECT_TRUE(structures[4].has_parent_id());
+
+  EXPECT_EQ(structures[5].operation(),
+            feedstore::StreamStructure::UPDATE_OR_APPEND);
+  EXPECT_EQ(structures[5].type(), feedstore::StreamStructure::CLUSTER);
+  EXPECT_TRUE(structures[5].has_content_id());
+  EXPECT_TRUE(structures[5].has_parent_id());
+  EXPECT_EQ(ContentIdToString(structures[5].parent_id()),
+            ContentIdToString(root_content_id));
+
+  // The other members:
+  EXPECT_EQ(translated->content.size(), kExpectedContentCount);
+  EXPECT_EQ(translated->shared_states.size(), kExpectedSharedStateCount);
+
+  EXPECT_EQ(translated->response_time, kResponseTime);
+}
+
+}  // namespace feed

components/feed/core/v2/tools/generate_test_response_binarypb.sh

+#!/bin/bash
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+CHROMIUM_SRC=$(realpath $(dirname $(readlink -f $0))/../../../../..)
+OUT_DIR=$CHROMIUM_SRC/components/test/data/feed
+
+if [ ! -d $OUT_DIR ]; then
+  echo "Output directory $OUT_DIR doesn't exist."
+  exit 1
+fi
+
+python3 $CHROMIUM_SRC/components/feed/core/v2/tools/textpb_to_binarypb.py \
+  --chromium_path=$CHROMIUM_SRC \
+  --output_file=$OUT_DIR/response.binarypb \
+  --source_file=\
+$CHROMIUM_SRC/components/feed/core/v2/testdata/response.textproto

components/feed/core/v2/tools/protoc_util.py

+#!/usr/bin/python3
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Lint as: python3
+"""The tools provides lot of protoc related helper functions."""
+
+import glob
+import os
+import subprocess
+
+_protoc_path = None
+
+def run_command(args, input):
+  """Uses subprocess to execute the command line args."""
+  proc = subprocess.run(
+      args,
+      input=input,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.PIPE,
+      check=True)
+  return proc.stdout
+
+
+def get_protoc_common_args(root_dir, proto_path):
+  """Returns a list of protoc common args as a list."""
+  result = [
+      '-I' + os.path.join(root_dir)
+  ]
+  for root, _, files in os.walk(os.path.join(root_dir, proto_path)):
+    result += [os.path.join(root, f) for f in files if f.endswith('.proto')]
+
+  return result
+
+
+def encode_proto(text, message_name, root_dir, proto_path):
+  """Calls a command line to encode the text string and returns binary bytes."""
+  return run_command([protoc_path(root_dir), '--encode=' + message_name]
+                     + get_protoc_common_args(root_dir, proto_path),
+                     text.encode())
+
+
+def decode_proto(data, message_name, root_dir, proto_path):
+  """Calls a command line to decode the binary bytes array into text string."""
+  return run_command([protoc_path(root_dir), '--decode=' + message_name
+                     ] + get_protoc_common_args(root_dir, proto_path),
+                     data).decode('utf-8')
+
+
+def protoc_path(root_dir):
+  """Returns the path to the proto compiler, protoc."""
+  global _protoc_path
+  if not _protoc_path:
+    protoc_list = list(
+        glob.glob(os.path.join(root_dir, "out") + "/*/protoc")) + list(
+            glob.glob(os.path.join(root_dir, "out") + "/*/*/protoc"))
+    if not len(protoc_list):
+      print("Can't find a suitable build output directory",
+            "(it should have protoc)")
+      sys.exit(1)
+    _protoc_path = protoc_list[0]
+  return _protoc_path

components/feed/core/v2/tools/textpb_to_binarypb.py

+#!/usr/bin/python3
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Lint as: python3
+"""The tool converts a textpb into a binary proto using chromium protoc binary.
+
+Make sure you have absl-py installed via 'python3 -m pip install absl-py'.
+
+Usage example:
+    python3 ./textpb_to_binarypb.py
+    --chromium_path ~/chromium/src
+    --output_file /tmp/binary.pb
+    --source_file /tmp/original.textpb
+"""
+
+import glob
+import os
+import protoc_util
+import subprocess
+
+from absl import app
+from absl import flags
+
+DEFAULT_MESSAGE = 'feedwire.Response'
+
+FLAGS = flags.FLAGS
+FLAGS = flags.FLAGS
+flags.DEFINE_string('chromium_path', '', 'The path of your chromium depot.')
+flags.DEFINE_string('output_file', '', 'The target output binary file path.')
+flags.DEFINE_string('source_file', '',
+                    'The source proto file, in textpb format, path.')
+flags.DEFINE_string('message',
+                    DEFAULT_MESSAGE,
+                    'The message to look for in source_file.')
+
+COMPONENT_FEED_PROTO_PATH = 'components/feed/core/proto'
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError('Too many command-line arguments.')
+  if not FLAGS.chromium_path:
+    raise app.UsageError('chromium_path flag must be set.')
+  if not FLAGS.source_file:
+    raise app.UsageError('source_file flag must be set.')
+  if not FLAGS.output_file:
+    raise app.UsageError('output_file flag must be set.')
+
+  with open(FLAGS.source_file) as file:
+    value_text_proto = file.read()
+
+  encoded = protoc_util.encode_proto(value_text_proto, FLAGS.message,
+                                     FLAGS.chromium_path,
+                                     COMPONENT_FEED_PROTO_PATH)
+  with open(FLAGS.output_file, 'wb') as file:
+    file.write(encoded)
+
+
+if __name__ == '__main__':
+  app.run(main)