[Devtools] Add the ability to import HAR files

HAR files can now be exported and imported from the network panel. This
can be done by dragging a HAR file and dropping it in the Network Log
View.

R=caseq
TBR=dgozman
BUG=374557

Change-Id: Ibb451fc0a2a4d34739e596c320e99fb8866c6cac
Reviewed-on: https://chromium-review.googlesource.com/557995
Commit-Queue: Blaise Bruer <allada@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Cr-Commit-Position: refs/heads/master@{#487419}

third_party/WebKit/LayoutTests/inspector/modules-load-network-expected.txt

@@ -6,6 +6,7 @@ Loaded modules:
     data_grid
     diff
     formatter
+    har_importer
     network
     network_priorities
     object_ui

third_party/WebKit/Source/devtools/BUILD.gn

@@ -253,6 +253,9 @@ all_devtools_files = [
   "front_end/gonzales/gonzales-scss.js",
   "front_end/gonzales/module.json",
   "front_end/gonzales/SCSSParser.js",
+  "front_end/har_importer/HARFormat.js",
+  "front_end/har_importer/HARImporter.js",
+  "front_end/har_importer/module.json",
   "front_end/heap_snapshot_model/HeapSnapshotModel.js",
   "front_end/heap_snapshot_model/module.json",
   "front_end/heap_snapshot_worker.js",
@@ -901,6 +904,7 @@ generated_non_autostart_non_remote_modules = [
   "$resources_out_dir/elements/elements_module.js",
   "$resources_out_dir/event_listeners/event_listeners_module.js",
   "$resources_out_dir/formatter/formatter_module.js",
+  "$resources_out_dir/har_importer/har_importer_module.js",
   "$resources_out_dir/heap_snapshot_model/heap_snapshot_model_module.js",
   "$resources_out_dir/inline_editor/inline_editor_module.js",
   "$resources_out_dir/layer_viewer/layer_viewer_module.js",

third_party/WebKit/Source/devtools/front_end/Runtime.js

@@ -737,6 +737,7 @@ Runtime.Module = class {
       'ui': 'UI',
       'object_ui': 'ObjectUI',
       'perf_ui': 'PerfUI',
+      'har_importer': 'HARImporter',
     };
     var namespace = specialCases[this._name] || this._name.split('_').map(a => a.substring(0, 1).toUpperCase() + a.substring(1)).join('');
     self[namespace] = self[namespace] || {};

third_party/WebKit/Source/devtools/front_end/common/ResourceType.js

@@ -44,6 +44,33 @@ Common.ResourceType = class {
     this._isTextType = isTextType;
   }
 
+  /**
+   * @param {?string} mimeType
+   * @return {!Common.ResourceType}
+   */
+  static fromMimeType(mimeType) {
+    var contentTypeAndTopLevelType = mimeType.match(/(\w*)\/\w*/);
+    if (!contentTypeAndTopLevelType)
+      return Common.resourceTypes.Other;
+
+    var resourceType = Common.ResourceType._resourceTypeByMimeType.get(contentTypeAndTopLevelType[0]);
+    if (resourceType)
+      return resourceType;
+
+    resourceType = Common.ResourceType._resourceTypeByMimeType.get(contentTypeAndTopLevelType[1]);
+    if (resourceType)
+      return resourceType;
+    return Common.resourceTypes.Other;
+  }
+
+  /**
+   * @param {string} url
+   * @return {?Common.ResourceType}
+   */
+  static fromURL(url) {
+    return Common.ResourceType._resourceTypeByExtension.get(Common.ParsedURL.extractExtension(url)) || null;
+  }
+
   /**
    * @param {string} url
    * @return {string|undefined}
@@ -206,6 +233,21 @@ Common.ResourceType._mimeTypeByName = new Map([
   ['Cakefile', 'text/x-coffeescript']
 ]);
 
+Common.ResourceType._resourceTypeByExtension = new Map([
+  ['js', Common.resourceTypes.Script],
+
+  ['css', Common.resourceTypes.Stylesheet], ['xsl', Common.resourceTypes.Stylesheet],
+
+  ['jpeg', Common.resourceTypes.Image], ['jpg', Common.resourceTypes.Image], ['svg', Common.resourceTypes.Image],
+  ['gif', Common.resourceTypes.Image], ['png', Common.resourceTypes.Image], ['ico', Common.resourceTypes.Image],
+  ['tiff', Common.resourceTypes.Image], ['tif', Common.resourceTypes.Image], ['bmp', Common.resourceTypes.Image],
+
+  ['webp', Common.resourceTypes.Media],
+
+  ['ttf', Common.resourceTypes.Font], ['otf', Common.resourceTypes.Font], ['ttc', Common.resourceTypes.Font],
+  ['woff', Common.resourceTypes.Font]
+]);
+
 Common.ResourceType._mimeTypeByExtension = new Map([
   // Web extensions
   ['js', 'text/javascript'], ['css', 'text/css'], ['html', 'text/html'], ['htm', 'text/html'],
@@ -273,3 +315,15 @@ Common.ResourceType._mimeTypeByExtension = new Map([
   // Font
   ['ttf', 'font/opentype'], ['otf', 'font/opentype'], ['ttc', 'font/opentype'], ['woff', 'application/font-woff']
 ]);
+
+Common.ResourceType._resourceTypeByMimeType = new Map([
+  // Web types
+  ['text/javascript', Common.resourceTypes.Script], ['text/css', Common.resourceTypes.Stylesheet],
+  ['text/html', Common.resourceTypes.Document],
+
+  // Image
+  ['image', Common.resourceTypes.Image],
+
+  // Font
+  ['font', Common.resourceTypes.Font], ['application/font-woff', Common.resourceTypes.Font]
+]);
\ No newline at end of file

third_party/WebKit/Source/devtools/front_end/har_importer/HARImporter.js

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+HARImporter.Importer = class {
+  /**
+   * @param {!HARImporter.HARLog} log
+   * @return {!Array<!SDK.NetworkRequest>}
+   */
+  static requestsFromHARLog(log) {
+    /** @type {!Map<string, !HARImporter.HARPage>} */
+    var pages = new Map();
+    for (var page of log.pages)
+      pages.set(page.id, page);
+
+    log.entries.sort((a, b) => a.startedDateTime - b.startedDateTime);
+
+    /** @type {!Map<string, !NetworkLog.PageLoad>} */
+    var pageLoads = new Map();
+    /** @type {!Array<!SDK.NetworkRequest>} */
+    var requests = [];
+    for (var entry of log.entries) {
+      var pageLoad = pageLoads.get(entry.pageref);
+      var documentURL = pageLoad ? pageLoad.mainRequest.url() : entry.request.url;
+      var request = new SDK.NetworkRequest('har-' + requests.length, entry.request.url, documentURL, '', '', null);
+      var page = pages.get(entry.pageref);
+      if (!pageLoad && page) {
+        pageLoad = HARImporter.Importer._buildPageLoad(page, request);
+        pageLoads.set(entry.pageref, pageLoad);
+      }
+      HARImporter.Importer._fillRequestFromHAREntry(request, entry, pageLoad);
+      if (pageLoad)
+        pageLoad.bindRequest(request);
+      requests.push(request);
+    }
+    return requests;
+  }
+
+  /**
+   * @param {!HARImporter.HARPage} page
+   * @param {!SDK.NetworkRequest} mainRequest
+   * @return {!NetworkLog.PageLoad}
+   */
+  static _buildPageLoad(page, mainRequest) {
+    var pageLoad = new NetworkLog.PageLoad(mainRequest);
+    pageLoad.startTime = page.startedDateTime;
+    pageLoad.contentLoadTime = page.pageTimings.onContentLoad * 1000;
+    pageLoad.loadTime = page.pageTimings.onLoad * 1000;
+    return pageLoad;
+  }
+
+  /**
+   * @param {!SDK.NetworkRequest} request
+   * @param {!HARImporter.HAREntry} entry
+   * @param {?NetworkLog.PageLoad} pageLoad
+   */
+  static _fillRequestFromHAREntry(request, entry, pageLoad) {
+    // Request data.
+    if (entry.request.postData)
+      request.requestFormData = entry.request.postData.text;
+    request.connectionId = entry.connection || '';
+    request.requestMethod = entry.request.method;
+    request.setRequestHeaders(entry.request.headers);
+
+    // Response data.
+    if (entry.response.content.mimeType && entry.response.content.mimeType !== 'x-unknown')
+      request.mimeType = entry.response.content.mimeType;
+    request.responseHeaders = entry.response.headers;
+    request.statusCode = entry.response.status;
+    request.statusText = entry.response.statusText;
+    var protocol = entry.response.httpVersion.toLowerCase();
+    if (protocol === 'http/2.0')
+      protocol = 'h2';
+    request.protocol = protocol.replace(/^http\/2\.0?\+quic/, 'http/2+quic');
+
+    // Timing data.
+    var issueTime = entry.startedDateTime.getTime() / 1000;
+    request.setIssueTime(issueTime, issueTime);
+
+    // Content data.
+    var contentSize = entry.response.content.size > 0 ? entry.response.content.size : 0;
+    var headersSize = entry.response.headersSize > 0 ? entry.response.headersSize : 0;
+    var bodySize = entry.response.bodySize > 0 ? entry.response.bodySize : 0;
+    request.resourceSize = contentSize || (headersSize + bodySize);
+    var transferSize = entry.response.customAsNumber('transferSize');
+    if (transferSize === undefined)
+      transferSize = entry.response.headersSize + entry.response.bodySize;
+    request.setTransferSize(transferSize >= 0 ? transferSize : 0);
+
+    var fromCache = entry.customAsString('fromCache');
+    if (fromCache === 'memory')
+      request.setFromMemoryCache();
+    else if (fromCache === 'disk')
+      request.setFromDiskCache();
+
+    var contentData = {error: null, content: null, encoded: entry.response.content.encoding === 'base64'};
+    if (entry.response.content.text !== undefined)
+      contentData.content = entry.response.content.text;
+    request.setContentData(contentData);
+
+    // Timing data.
+    HARImporter.Importer._setupTiming(request, issueTime, entry.time, entry.timings);
+
+    // Meta data.
+    request.setRemoteAddress(entry.serverIPAddress || '', 80);  // Har does not support port numbers.
+    var resourceType = (pageLoad && pageLoad.mainRequest === request) ?
+        Common.resourceTypes.Document :
+        Common.ResourceType.fromMimeType(entry.response.content.mimeType);
+    if (!resourceType)
+      resourceType = Common.ResourceType.fromURL(entry.request.url) || Common.resourceTypes.Other;
+    request.setResourceType(resourceType);
+
+    request.finished = true;
+  }
+
+  /**
+   * @param {!SDK.NetworkRequest} request
+   * @param {number} issueTime
+   * @param {number} entryTotalDuration
+   * @param {!HARImporter.HARTimings} timings
+   */
+  static _setupTiming(request, issueTime, entryTotalDuration, timings) {
+    /**
+     * @param {number|undefined} timing
+     * @return {number}
+     */
+    function accumulateTime(timing) {
+      if (timing === undefined || timing < 0)
+        return -1;
+      lastEntry += timing;
+      return lastEntry;
+    }
+    var lastEntry = timings.blocked >= 0 ? timings.blocked : 0;
+
+    var proxy = timings.customAsNumber('blocked_proxy') || -1;
+    var queueing = timings.customAsNumber('blocked_queueing') || -1;
+
+    // SSL is part of connect for both HAR and Chrome's format so subtract it here.
+    var ssl = timings.ssl >= 0 ? timings.ssl : 0;
+    if (timings.connect > 0)
+      timings.connect -= ssl;
+    var timing = {
+      proxyStart: proxy > 0 ? lastEntry - proxy : -1,
+      proxyEnd: proxy > 0 ? lastEntry : -1,
+      requestTime: issueTime + (queueing > 0 ? queueing : 0) / 1000,
+      dnsStart: timings.dns >= 0 ? lastEntry : -1,
+      dnsEnd: accumulateTime(timings.dns),
+
+      // Add ssl to end time without modifying lastEntry (see comment above).
+      connectStart: timings.connect >= 0 ? lastEntry : -1,
+      connectEnd: accumulateTime(timings.connect) + ssl,
+
+      // Now update lastEntry to add ssl timing back in (see comment above).
+      sslStart: timings.ssl >= 0 ? lastEntry : -1,
+      sslEnd: accumulateTime(timings.ssl),
+
+      workerStart: -1,
+      workerReady: -1,
+      sendStart: timings.send >= 0 ? lastEntry : -1,
+      sendEnd: accumulateTime(timings.send),
+      pushStart: 0,
+      pushEnd: 0,
+      receiveHeadersEnd: accumulateTime(timings.wait)
+    };
+    accumulateTime(timings.receive);
+
+    request.timing = timing;
+    request.endTime = issueTime + Math.max(entryTotalDuration, lastEntry) / 1000;
+  }
+};

third_party/WebKit/Source/devtools/front_end/har_importer/module.json

+{
+    "dependencies": [
+        "common",
+        "network_log",
+        "sdk"
+    ],
+    "scripts": [
+        "HARFormat.js",
+        "HARImporter.js"
+    ]
+}

third_party/WebKit/Source/devtools/front_end/inspector.json

@@ -17,6 +17,7 @@
         { "name": "services", "type": "autostart" },
         { "name": "elements", "condition": "!v8only" },
         { "name": "network", "condition": "!v8only" },
+        { "name": "har_importer", "condition": "!v8only" },
         { "name": "sources" },
         { "name": "timeline", "condition": "!v8only" },
         { "name": "timeline_model", "condition": "!v8only" },

third_party/WebKit/Source/devtools/front_end/integration_test_runner.json

@@ -16,6 +16,7 @@
     { "name": "host", "type": "autostart" },
     { "name": "common", "type": "autostart" },
     { "name": "emulation", "type": "autostart" },
+    { "name": "har_importer", "condition": "!v8only" },
     { "name": "workspace", "type": "autostart" },
     { "name": "bindings", "type": "autostart" },
     { "name": "persistence", "type": "autostart" },

third_party/WebKit/Source/devtools/front_end/network/NetworkLogView.js

@@ -108,6 +108,9 @@ Network.NetworkLogView = class extends UI.VBox {
     this._resetSuggestionBuilder();
     this._initializeView();
 
+    new UI.DropTarget(
+        this.element, [UI.DropTarget.Types.Files], Common.UIString('Drop HAR files here'), this._handleDrop.bind(this));
+
     Common.moduleSetting('networkColorCodeResourceTypes')
         .addChangeListener(this._invalidateAllItems.bind(this, false), this);
 
@@ -381,6 +384,48 @@ Network.NetworkLogView = class extends UI.VBox {
     InspectorFrontendHost.copyText(content || '');
   }
 
+  /**
+   * @param {!DataTransfer} dataTransfer
+   */
+  _handleDrop(dataTransfer) {
+    var items = dataTransfer.items;
+    if (!items.length)
+      return;
+    var entry = items[0].webkitGetAsEntry();
+    if (entry.isDirectory)
+      return;
+
+    entry.file(this._onLoadFromFile.bind(this));
+  }
+
+  /**
+   * @param {!File} file
+   */
+  async _onLoadFromFile(file) {
+    var outputStream = new Common.StringOutputStream();
+    var reader = new Bindings.ChunkedFileReader(file, /* chunkSize */ 10000000);
+    var success = await reader.read(outputStream);
+    if (!success) {
+      this._harLoadFailed(reader.error().message);
+      return;
+    }
+    try {
+      // HARRoot and JSON.parse might throw.
+      var harRoot = new HARImporter.HARRoot(JSON.parse(outputStream.data()));
+    } catch (e) {
+      this._harLoadFailed(e);
+      return;
+    }
+    NetworkLog.networkLog.importRequests(HARImporter.Importer.requestsFromHARLog(harRoot.log));
+  }
+
+  /**
+   * @param {string} message
+   */
+  _harLoadFailed(message) {
+    Common.console.error('Failed to load HAR file with following error: ' + message);
+  }
+
   /**
    * @param {?string} groupKey
    */

third_party/WebKit/Source/devtools/front_end/network/module.json

@@ -122,6 +122,7 @@
         "network_log",
         "product_registry",
         "mobile_throttling",
+        "har_importer",
         "network_priorities"
     ],
     "scripts": [

third_party/WebKit/Source/devtools/front_end/network_log/NetworkLog.js

@@ -322,6 +322,20 @@ NetworkLog.NetworkLog = class extends Common.Object {
       this._pageLoadForManager.set(manager, currentPageLoad);
   }
 
+  /**
+   * @param {!Array<!SDK.NetworkRequest>} requests
+   */
+  importRequests(requests) {
+    this.reset();
+    this._requests = [];
+    this._requestsSet.clear();
+    for (var request of requests) {
+      this._requests.push(request);
+      this._requestsSet.add(request);
+      this.dispatchEventToListeners(NetworkLog.NetworkLog.Events.RequestAdded, request);
+    }
+  }
+
   /**
    * @param {!Common.Event} event
    */

third_party/WebKit/Source/devtools/front_end/sdk/NetworkRequest.js

@@ -901,6 +901,14 @@ SDK.NetworkRequest = class extends Common.Object {
     return this._contentData;
   }
 
+  /**
+   * @param {!SDK.NetworkRequest.ContentData} data
+   */
+  setContentData(data) {
+    console.assert(!this._contentData, 'contentData can only be set once.');
+    this._contentData = Promise.resolve(data);
+  }
+
   /**
    * @override
    * @return {string}

third_party/WebKit/Source/devtools/scripts/special_case_namespaces.json

@@ -3,5 +3,6 @@
   "ui": "UI",
   "object_ui": "ObjectUI",
   "perf_ui": "PerfUI",
-  "css_tracker": "CSSTracker"
+  "css_tracker": "CSSTracker",
+  "har_importer": "HARImporter"
 }
\ No newline at end of file