Add real time lookup request/response to chrome://safe-browsing

Screen shot: https://screenshot.googleplex.com/My1OamJqnYy

The CL only includes web UI logic, the code that triggers this UI
will be included in a follow up CL.

Bug: 1013370
Change-Id: I2b785ea711a0c62c9b7fa3c3dcac811307289d1f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1856804
Commit-Queue: Xinghui Lu <xinghuilu@chromium.org>
Reviewed-by: Daniel Rubery <drubery@chromium.org>
Reviewed-by: Varun Khaneja <vakh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#705693}

components/safe_browsing/browser/safe_browsing_url_checker_impl.cc

@@ -455,7 +455,7 @@ void SafeBrowsingUrlCheckerImpl::OnRTLookupResponse(
     OnUrlResult(url, SB_THREAT_TYPE_SAFE, ThreatMetadata());
   }
 
-  // TODO(vakh): Log |response| to chrome://safe-browsing
+  // TODO(crbug.com/1013370): Log |response| to chrome://safe-browsing
 }
 
 }  // namespace safe_browsing

components/safe_browsing/web_ui/BUILD.gn

@@ -19,6 +19,7 @@ static_library("web_ui") {
     "//components/safe_browsing:buildflags",
     "//components/safe_browsing:csd_proto",
     "//components/safe_browsing:features",
+    "//components/safe_browsing:realtimeapi_proto",
     "//components/safe_browsing/browser:network_context",
     "//components/safe_browsing/browser:referrer_chain_provider",
     "//components/safe_browsing/common:safe_browsing_prefs",

components/safe_browsing/web_ui/resources/safe_browsing.html

@@ -25,6 +25,7 @@
         <tab id="csbrr">ClientSafeBrowsingReportRequests</tab>
         <tab id="download-protection">Download Protection</tab>
         <tab id="password-protection">Password Protection</tab>
+        <tab id="rt-lookup">RT Lookup</tab>
         <tab id="referrer-chain">Referrer Chain</tab>
         <tab id="log">Log Messages</tab>
       </tabs>
@@ -87,6 +88,10 @@
           <h2>Password Protection Pings</h2>
           <table id="pg-ping-list" class="request-response"></table>
         </tabpanel>
+        <tabpanel>
+          <h2>RT Lookup Pings</h2>
+          <table id="rt-lookup-ping-list" class="request-response"></table>
+        </tabpanel>
         <tabpanel>
           <h2>Referrer Chain</h2>
           <form id="get-referrer-chain-form">

components/safe_browsing/web_ui/resources/safe_browsing.js

@@ -89,6 +89,24 @@ cr.define('safe_browsing', function() {
       addPGResponse(result);
     });
 
+    cr.sendWithPromise('getRTLookupPings', [])
+        .then((rtLookupPings) => { rtLookupPings
+            .forEach(function (rtLookupPing) {
+          addRTLookupPing(rtLookupPing);
+        })});
+    cr.addWebUIListener('rt-lookup-pings-update', function(result) {
+      addRTLookupPing(result);
+    });
+
+    cr.sendWithPromise('getRTLookupResponses', [])
+        .then((rtLookupResponses) => { rtLookupResponses
+            .forEach(function (rtLookupResponse) {
+          addRTLookupResponse(rtLookupResponse);
+        })});
+    cr.addWebUIListener('rt-lookup-responses-update', function(result) {
+      addRTLookupResponse(result);
+    });
+
     cr.sendWithPromise('getLogMessages', [])
         .then((logMessages) => { logMessages.forEach(function (message) {
           addLogMessage(message);
@@ -204,36 +222,40 @@ cr.define('safe_browsing', function() {
     appendChildWithInnerText(logDiv, eventFormatted);
   }
 
-  function addPGPingRow(token) {
-    var row = $('pg-ping-list').insertRow();
+  function insertTokenToTable(tableId, token) {
+    var row = $(tableId).insertRow();
     row.className = 'content';
-    row.id = 'pg-ping-list-' + token;
+    row.id = tableId + '-' + token;
     row.insertCell().className = 'content';
     row.insertCell().className = 'content';
   }
 
-  function addPGPing(result) {
+  function addResultToTable(tableId, result, position) {
     var token = result[0];
     var request = result[1];
 
-    if ($('pg-ping-list-'+token) == undefined) {
-      addPGPingRow(token);
+    if ($(tableId + '-' + token) == undefined) {
+      insertTokenToTable(tableId, token);
     }
 
-    var cell = $('pg-ping-list-'+token).cells[0];
+    var cell = $(tableId + '-' + token).cells[position];
     appendChildWithInnerText(cell, request);
   }
 
+  function addPGPing(result) {
+    addResultToTable('pg-ping-list', result, 0);
+  }
+
   function addPGResponse(result) {
-    var token = result[0];
-    var response = result[1];
+    addResultToTable('pg-ping-list', result, 1);
+  }
 
-    if ($('pg-ping-list-'+token) == undefined) {
-      addPGPingRow(token);
-    }
+  function addRTLookupPing(result) {
+    addResultToTable('rt-lookup-ping-list', result, 0);
+  }
 
-    var cell = $('pg-ping-list-'+token).cells[1];
-    appendChildWithInnerText(cell, response);
+  function addRTLookupResponse(result) {
+    addResultToTable('rt-lookup-ping-list', result, 1);
   }
 
   function addLogMessage(result) {

components/safe_browsing/web_ui/safe_browsing_ui.cc

@@ -176,6 +176,36 @@ void WebUIInfoSingleton::ClearPGPings() {
   std::map<int, LoginReputationClientResponse>().swap(pg_responses_);
 }
 
+int WebUIInfoSingleton::AddToRTLookupPings(const RTLookupRequest request) {
+  if (!HasListener())
+    return -1;
+
+  for (auto* webui_listener : webui_instances_)
+    webui_listener->NotifyRTLookupPingJsListener(rt_lookup_pings_.size(),
+                                                 request);
+
+  rt_lookup_pings_.push_back(request);
+
+  return rt_lookup_pings_.size() - 1;
+}
+
+void WebUIInfoSingleton::AddToRTLookupResponses(
+    int token,
+    const RTLookupResponse response) {
+  if (!HasListener())
+    return;
+
+  for (auto* webui_listener : webui_instances_)
+    webui_listener->NotifyRTLookupResponseJsListener(token, response);
+
+  rt_lookup_responses_[token] = response;
+}
+
+void WebUIInfoSingleton::ClearRTLookupPings() {
+  std::vector<RTLookupRequest>().swap(rt_lookup_pings_);
+  std::map<int, RTLookupResponse>().swap(rt_lookup_responses_);
+}
+
 void WebUIInfoSingleton::LogMessage(const std::string& message) {
   if (!HasListener())
     return;
@@ -213,6 +243,7 @@ void WebUIInfoSingleton::UnregisterWebUIInstance(SafeBrowsingUIHandler* webui) {
     ClearClientDownloadResponsesReceived();
     ClearPGEvents();
     ClearPGPings();
+    ClearRTLookupPings();
     ClearLogMessages();
   }
 }
@@ -942,6 +973,53 @@ base::Value SerializeChromeUserPopulation(
   return std::move(population_dict);
 }
 
+base::Value SerializeRTThreatInfo(
+    const RTLookupResponse::ThreatInfo& threat_info) {
+  base::DictionaryValue threat_info_dict;
+  std::string threat_type;
+  switch (threat_info.threat_type()) {
+    case RTLookupResponse::ThreatInfo::THREAT_TYPE_UNSPECIFIED:
+      threat_type = "THREAT_TYPE_UNSPECIFIED";
+      break;
+    case RTLookupResponse::ThreatInfo::WEB_MALWARE:
+      threat_type = "WEB_MALWARE";
+      break;
+    case RTLookupResponse::ThreatInfo::SOCIAL_ENGINEERING:
+      threat_type = "SOCIAL_ENGINEERING";
+      break;
+    case RTLookupResponse::ThreatInfo::UNWANTED_SOFTWARE:
+      threat_type = "UNWANTED_SOFTWARE";
+      break;
+    case RTLookupResponse::ThreatInfo::UNCLEAR_BILLING:
+      threat_type = "UNCLEAR_BILLING";
+      break;
+  }
+  threat_info_dict.SetKey("threat_type", base::Value(threat_type));
+
+  threat_info_dict.SetKey(
+      "cache_duration_sec",
+      base::Value(static_cast<double>(threat_info.cache_duration_sec())));
+
+  threat_info_dict.SetKey("cache_expression",
+                          base::Value(threat_info.cache_expression()));
+
+  std::string verdict_type;
+  switch (threat_info.verdict_type()) {
+    case RTLookupResponse::ThreatInfo::VERDICT_TYPE_UNSPECIFIED:
+      verdict_type = "VERDICT_TYPE_UNSPECIFIED";
+      break;
+    case RTLookupResponse::ThreatInfo::SAFE:
+      verdict_type = "SAFE";
+      break;
+    case RTLookupResponse::ThreatInfo::DANGEROUS:
+      verdict_type = "DANGEROUS";
+      break;
+  }
+  threat_info_dict.SetKey("verdict_type", base::Value(verdict_type));
+
+  return std::move(threat_info_dict);
+}
+
 base::Value SerializeDomFeatures(const DomFeatures& dom_features) {
   base::DictionaryValue dom_features_dict;
   auto feature_map = std::make_unique<base::ListValue>();
@@ -1053,6 +1131,52 @@ std::string SerializePGResponse(const LoginReputationClientResponse& response) {
   return response_serialized;
 }
 
+std::string SerializeRTLookupPing(const RTLookupRequest& request) {
+  base::DictionaryValue request_dict;
+
+  request_dict.SetKey("url", base::Value(request.url()));
+  request_dict.SetKey("population",
+                      SerializeChromeUserPopulation(request.population()));
+
+  std::string lookupType;
+  switch (request.lookup_type()) {
+    case RTLookupRequest::LOOKUP_TYPE_UNSPECIFIED:
+      lookupType = "LOOKUP_TYPE_UNSPECIFIED";
+      break;
+    case RTLookupRequest::NAVIGATION:
+      lookupType = "NAVIGATION";
+      break;
+    case RTLookupRequest::DOWNLOAD:
+      lookupType = "DOWNLOAD";
+      break;
+  }
+
+  request_dict.SetKey("lookup_type", base::Value(lookupType));
+
+  std::string request_serialized;
+  JSONStringValueSerializer serializer(&request_serialized);
+  serializer.set_pretty_print(true);
+  serializer.Serialize(request_dict);
+  return request_serialized;
+}
+
+std::string SerializeRTLookupResponse(const RTLookupResponse& response) {
+  base::DictionaryValue response_dict;
+
+  base::ListValue threat_info_list;
+  for (const RTLookupResponse::ThreatInfo& threat_info :
+       response.threat_info()) {
+    threat_info_list.Append(SerializeRTThreatInfo(threat_info));
+  }
+  response_dict.SetKey("threat_infos", std::move(threat_info_list));
+
+  std::string response_serialized;
+  JSONStringValueSerializer serializer(&response_serialized);
+  serializer.set_pretty_print(true);
+  serializer.Serialize(response_dict);
+  return response_serialized;
+}
+
 base::Value SerializeLogMessage(const base::Time& timestamp,
                                 const std::string& message) {
   base::DictionaryValue result;
@@ -1324,6 +1448,45 @@ void SafeBrowsingUIHandler::GetPGResponses(const base::ListValue* args) {
   ResolveJavascriptCallback(base::Value(callback_id), responses_sent);
 }
 
+void SafeBrowsingUIHandler::GetRTLookupPings(const base::ListValue* args) {
+  const std::vector<RTLookupRequest> requests =
+      WebUIInfoSingleton::GetInstance()->rt_lookup_pings();
+
+  base::ListValue pings_sent;
+  for (size_t request_index = 0; request_index < requests.size();
+       request_index++) {
+    base::ListValue ping_entry;
+    ping_entry.Append(base::Value(int(request_index)));
+    ping_entry.Append(
+        base::Value(SerializeRTLookupPing(requests[request_index])));
+    pings_sent.Append(std::move(ping_entry));
+  }
+
+  AllowJavascript();
+  std::string callback_id;
+  args->GetString(0, &callback_id);
+  ResolveJavascriptCallback(base::Value(callback_id), pings_sent);
+}
+
+void SafeBrowsingUIHandler::GetRTLookupResponses(const base::ListValue* args) {
+  const std::map<int, RTLookupResponse> responses =
+      WebUIInfoSingleton::GetInstance()->rt_lookup_responses();
+
+  base::ListValue responses_sent;
+  for (const auto& token_and_response : responses) {
+    base::ListValue response_entry;
+    response_entry.Append(base::Value(token_and_response.first));
+    response_entry.Append(
+        base::Value(SerializeRTLookupResponse(token_and_response.second)));
+    responses_sent.Append(std::move(response_entry));
+  }
+
+  AllowJavascript();
+  std::string callback_id;
+  args->GetString(0, &callback_id);
+  ResolveJavascriptCallback(base::Value(callback_id), responses_sent);
+}
+
 void SafeBrowsingUIHandler::GetReferrerChain(const base::ListValue* args) {
   std::string url_string;
   args->GetString(1, &url_string);
@@ -1431,6 +1594,28 @@ void SafeBrowsingUIHandler::NotifyPGResponseJsListener(
   FireWebUIListener("pg-responses-update", response_list);
 }
 
+void SafeBrowsingUIHandler::NotifyRTLookupPingJsListener(
+    int token,
+    const RTLookupRequest& request) {
+  base::ListValue request_list;
+  request_list.Append(base::Value(token));
+  request_list.Append(base::Value(SerializeRTLookupPing(request)));
+
+  AllowJavascript();
+  FireWebUIListener("rt-lookup-pings-update", request_list);
+}
+
+void SafeBrowsingUIHandler::NotifyRTLookupResponseJsListener(
+    int token,
+    const RTLookupResponse& response) {
+  base::ListValue response_list;
+  response_list.Append(base::Value(token));
+  response_list.Append(base::Value(SerializeRTLookupResponse(response)));
+
+  AllowJavascript();
+  FireWebUIListener("rt-lookup-responses-update", response_list);
+}
+
 void SafeBrowsingUIHandler::NotifyLogMessageJsListener(
     const base::Time& timestamp,
     const std::string& message) {
@@ -1485,6 +1670,14 @@ void SafeBrowsingUIHandler::RegisterMessages() {
       "getPGResponses",
       base::BindRepeating(&SafeBrowsingUIHandler::GetPGResponses,
                           base::Unretained(this)));
+  web_ui()->RegisterMessageCallback(
+      "getRTLookupPings",
+      base::BindRepeating(&SafeBrowsingUIHandler::GetRTLookupPings,
+                          base::Unretained(this)));
+  web_ui()->RegisterMessageCallback(
+      "getRTLookupResponses",
+      base::BindRepeating(&SafeBrowsingUIHandler::GetRTLookupResponses,
+                          base::Unretained(this)));
   web_ui()->RegisterMessageCallback(
       "getLogMessages",
       base::BindRepeating(&SafeBrowsingUIHandler::GetLogMessages,

components/safe_browsing/web_ui/safe_browsing_ui.h

@@ -9,6 +9,7 @@
 #include "base/macros.h"
 #include "components/safe_browsing/browser/safe_browsing_network_context.h"
 #include "components/safe_browsing/proto/csd.pb.h"
+#include "components/safe_browsing/proto/realtimeapi.pb.h"
 #include "components/safe_browsing/proto/webui.pb.h"
 #include "components/sync/protocol/user_event_specifics.pb.h"
 #include "content/public/browser/web_ui_controller.h"
@@ -83,6 +84,14 @@ class SafeBrowsingUIHandler : public content::WebUIMessageHandler {
   // currently open chrome://safe-browsing tab was opened.
   void GetPGResponses(const base::ListValue* args);
 
+  // Get the real time lookup pings that have been sent since the oldest
+  // currently open chrome://safe-browsing tab was opened.
+  void GetRTLookupPings(const base::ListValue* args);
+
+  // Get the real time lookup responses that have been received since the oldest
+  // currently open chrome://safe-browsing tab was opened.
+  void GetRTLookupResponses(const base::ListValue* args);
+
   // Get the current referrer chain for a given URL.
   void GetReferrerChain(const base::ListValue* args);
 
@@ -132,6 +141,15 @@ class SafeBrowsingUIHandler : public content::WebUIMessageHandler {
       int token,
       const LoginReputationClientResponse& response);
 
+  // Called when any new real time lookup pings are sent while one or more
+  // WebUI tabs are open.
+  void NotifyRTLookupPingJsListener(int token, const RTLookupRequest& request);
+
+  // Called when any new real time lookup responses are received while one or
+  // more WebUI tabs are open.
+  void NotifyRTLookupResponseJsListener(int token,
+                                        const RTLookupResponse& response);
+
   // Called when any new log messages are received while one or more WebUI tabs
   // are open.
   void NotifyLogMessageJsListener(const base::Time& timestamp,
@@ -218,6 +236,17 @@ class WebUIInfoSingleton {
   // Clear the list of sent PhishGuard pings and responses.
   void ClearPGPings();
 
+  // Add the new ping to |rt_lookup_pings_|. Returns a token that can be used in
+  // |AddToRTLookupResponses| to correlate a ping and response.
+  int AddToRTLookupPings(const RTLookupRequest request);
+
+  // Add the new response to |rt_lookup_responses_| and send it to all the open
+  // chrome://safe-browsing tabs.
+  void AddToRTLookupResponses(int token, const RTLookupResponse response);
+
+  // Clear the list of sent RT Lookup pings and responses.
+  void ClearRTLookupPings();
+
   // Log an arbitrary message. Frequently used for debugging.
   void LogMessage(const std::string& message);
 
@@ -286,6 +315,18 @@ class WebUIInfoSingleton {
     return pg_responses_;
   }
 
+  // Get the list of real time lookup pings since the oldest currently open
+  // chrome://safe-browsing tab was opened.
+  const std::vector<RTLookupRequest>& rt_lookup_pings() const {
+    return rt_lookup_pings_;
+  }
+
+  // Get the list of real time lookup pings since the oldest currently open
+  // chrome://safe-browsing tab was opened.
+  const std::map<int, RTLookupResponse>& rt_lookup_responses() const {
+    return rt_lookup_responses_;
+  }
+
   ReferrerChainProvider* referrer_chain_provider() {
     return referrer_chain_provider_;
   }
@@ -350,6 +391,14 @@ class WebUIInfoSingleton {
   // chrome://safe-browsing tab was opened.
   std::map<int, LoginReputationClientResponse> pg_responses_;
 
+  // List of real time lookup pings sent since the oldest currently open
+  // chrome://safe-browsing tab was opened.
+  std::vector<RTLookupRequest> rt_lookup_pings_;
+
+  // List of real time lookup responses received since the oldest currently open
+  // chrome://safe-browsing tab was opened.
+  std::map<int, RTLookupResponse> rt_lookup_responses_;
+
   // List of WebUI listener objects. "SafeBrowsingUIHandler*" cannot be const,
   // due to being used by functions that call AllowJavascript(), which is not
   // marked const.