[Chromecast] Fixed LogcatExtractor urlElide being too sensitive

BUG=internal b/63104369
BUG=internal b/64753989
TEST=run cast_shell_junit_tests

Change-Id: I3d6aea7e5434bdb206d2ba0ab47a1a84d118c218
Reviewed-on: https://chromium-review.googlesource.com/659177
Commit-Queue: Sandeep Vijayasekar <sandv@chromium.org>
Reviewed-by: Luke Halliwell <halliwell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#502945}

chromecast/browser/android/BUILD.gn

@@ -90,7 +90,10 @@ android_library("cast_shell_java") {
 }
 
 junit_binary("cast_shell_junit_tests") {
-  java_files = [ "junit/src/org/chromium/chromecast/shell/CastWebContentsComponentTest.java" ]
+  java_files = [
+    "junit/src/org/chromium/chromecast/shell/CastWebContentsComponentTest.java",
+    "junit/src/org/chromium/chromecast/shell/LogcatExtractorUnitTest.java",
+  ]
 
   srcjar_deps = [ ":cast_shell_build_config_gen" ]
 

chromecast/browser/android/apk/src/org/chromium/chromecast/shell/LogcatExtractor.java

@@ -15,7 +15,9 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+
 /**
  * Extracts logcat out of Android devices and elide PII sensitive info from it.
  *
@@ -31,10 +33,40 @@ import java.util.regex.Pattern;
 class LogcatExtractor {
     private static final String TAG = "LogcatExtractor";
     private static final long HALF_SECOND = 500;
-    protected static final int LOGCAT_SIZE = 256; // Number of lines.
+    protected static final int LOGCAT_SIZE = 512; // Number of lines.
     protected static final String EMAIL_ELISION = "XXX@EMAIL.ELIDED";
     @VisibleForTesting
     protected static final String URL_ELISION = "HTTP://WEBADDRESS.ELIDED";
+
+    private static final String GOOD_IRI_CHAR = "a-zA-Z0-9\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF";
+
+    private static final Pattern IP_ADDRESS = Pattern.compile(
+            "((25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\\.(25[0-5]|2[0-4]"
+            + "[0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\\.(25[0-5]|2[0-4][0-9]|[0-1]"
+            + "[0-9]{2}|[1-9][0-9]|[1-9]|0)\\.(25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}"
+            + "|[1-9][0-9]|[0-9]))");
+
+    private static final String IRI =
+            "[" + GOOD_IRI_CHAR + "]([" + GOOD_IRI_CHAR + "\\-]{0,61}[" + GOOD_IRI_CHAR + "]){0,1}";
+
+    private static final String GOOD_GTLD_CHAR = "a-zA-Z\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF";
+    private static final String GTLD = "[" + GOOD_GTLD_CHAR + "]{2,63}";
+    private static final String HOST_NAME = "(" + IRI + "\\.)+" + GTLD;
+
+    private static final Pattern DOMAIN_NAME =
+            Pattern.compile("(" + HOST_NAME + "|" + IP_ADDRESS + ")");
+
+    private static final Pattern WEB_URL =
+            Pattern.compile("(?:\\b|^)((?:(http|https|Http|Https|rtsp|Rtsp):"
+                    + "\\/\\/(?:(?:[a-zA-Z0-9\\$\\-\\_\\.\\+\\!\\*\\'\\(\\)"
+                    + "\\,\\;\\?\\&\\=]|(?:\\%[a-fA-F0-9]{2})){1,64}(?:\\:(?:[a-zA-Z0-9\\$\\-\\_"
+                    + "\\.\\+\\!\\*\\'\\(\\)\\,\\;\\?\\&\\=]|(?:\\%[a-fA-F0-9]{2})){1,25})?\\@)?)?"
+                    + "(?:" + DOMAIN_NAME + ")"
+                    + "(?:\\:\\d{1,5})?)"
+                    + "(\\/(?:(?:[" + GOOD_IRI_CHAR + "\\;\\/\\?\\:\\@\\&\\=\\#\\~"
+                    + "\\-\\.\\+\\!\\*\\'\\(\\)\\,\\_])|(?:\\%[a-fA-F0-9]{2}))*)?"
+                    + "(?:\\b|$)");
+
     @VisibleForTesting
     protected static final String IP_ELISION = "1.2.3.4";
     @VisibleForTesting
@@ -47,6 +79,26 @@ class LogcatExtractor {
             Pattern.compile("([0-9a-fA-F]{2}[-:]+){5}[0-9a-fA-F]{2}");
     private static final Pattern CONSOLE_MSG = Pattern.compile("\\[\\w*:CONSOLE.*\\].*");
 
+    private static final String[] CHROME_NAMESPACE = new String[] {"org.chromium.", "com.google."};
+
+    private static final String[] CAST_NAMESPACE = new String[] {"Cast.", "CastV2."};
+
+    private static final String[] SYSTEM_NAMESPACE = new String[] {"android.accessibilityservice",
+            "android.accounts", "android.animation", "android.annotation", "android.app",
+            "android.appwidget", "android.bluetooth", "android.content", "android.database",
+            "android.databinding", "android.drm", "android.gesture", "android.graphics",
+            "android.hardware", "android.inputmethodservice", "android.location", "android.media",
+            "android.mtp", "android.net", "android.nfc", "android.opengl", "android.os",
+            "android.preference", "android.print", "android.printservice", "android.provider",
+            "android.renderscript", "android.sax", "android.security", "android.service",
+            "android.speech", "android.support", "android.system", "android.telecom",
+            "android.telephony", "android.test", "android.text", "android.transition",
+            "android.util", "android.view", "android.webkit", "android.widget", "com.android.",
+            "dalvik.", "java.", "javax.", "org.apache.", "org.json.", "org.w3c.dom.", "org.xml.",
+            "org.xmlpull."};
+
+    private static final Pattern JAVA_FILE = Pattern.compile(".java:[0-9]+$");
+
     public static String getElidedLogcat() throws IOException, InterruptedException {
         List<String> rawLogcat = getLogcat();
         List<String> elidedLogcat = Collections.unmodifiableList(processLogcat(rawLogcat));
@@ -123,14 +175,61 @@ class LogcatExtractor {
         return Patterns.EMAIL_ADDRESS.matcher(original).replaceAll(EMAIL_ELISION);
     }
     /**
-     * Elides any URLs in the specified {@link String} with {@link #URL_ELISION}.
+     * Elides any URLs in the specified {@link String} with
+     * {@link #URL_ELISION}.
      *
      * @param original String potentially containing URLs.
      * @return String with elided URLs.
      */
     @VisibleForTesting
     protected static String elideUrl(String original) {
-        return Patterns.WEB_URL.matcher(original).replaceAll(URL_ELISION);
+        StringBuilder buffer = new StringBuilder(original);
+        Matcher matcher = WEB_URL.matcher(buffer);
+        int start = 0;
+        while (matcher.find(start)) {
+            start = matcher.start();
+            int end = matcher.end();
+            String url = buffer.substring(start, end);
+            if (!likelyToBeChromeNamespace(url) && !likelyToBeSystemNamespace(url)
+                    && !likelyToBeCastNamespace(url) && !likelyToBeJavaFile(url)) {
+                buffer.replace(start, end, URL_ELISION);
+                end = start + URL_ELISION.length();
+                matcher = WEB_URL.matcher(buffer);
+            }
+            start = end;
+        }
+        return buffer.toString();
+    }
+
+    public static boolean likelyToBeChromeNamespace(String url) {
+        for (String ns : CHROME_NAMESPACE) {
+            if (url.startsWith(ns)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean likelyToBeSystemNamespace(String url) {
+        for (String ns : SYSTEM_NAMESPACE) {
+            if (url.startsWith(ns)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean likelyToBeCastNamespace(String url) {
+        for (String ns : CAST_NAMESPACE) {
+            if (url.startsWith(ns)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean likelyToBeJavaFile(String url) {
+        return JAVA_FILE.matcher(url).find();
     }
     /**
      * Elides any IP addresses in the specified {@link String} with {@link #IP_ELISION}.

chromecast/browser/android/junit/src/org/chromium/chromecast/shell/LogcatExtractorUnitTest.java

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.chromecast.shell;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+
+import org.chromium.testing.local.LocalRobolectricTestRunner;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * junit tests for {@link LogcatExtractor}.
+ */
+@RunWith(LocalRobolectricTestRunner.class)
+@Config(manifest = Config.NONE)
+public class LogcatExtractorUnitTest {
+    private static final int MAX_LINES = 5;
+
+    @Test
+    public void testElideEmail() {
+        String original = "email me at someguy@mailservice.com";
+        String expected = "email me at XXX@EMAIL.ELIDED";
+        assertEquals(expected, LogcatExtractor.elideEmail(original));
+    }
+
+    @Test
+    public void testElideUrl() {
+        String original = "file bugs at crbug.com";
+        String expected = "file bugs at HTTP://WEBADDRESS.ELIDED";
+        assertEquals(expected, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl2() {
+        String original = "exception at org.chromium.chrome.browser.crash.LogcatExtractorUnitTest";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl3() {
+        String original = "file bugs at crbug.com or code.google.com";
+        String expected = "file bugs at HTTP://WEBADDRESS.ELIDED or HTTP://WEBADDRESS.ELIDED";
+        assertEquals(expected, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl4() {
+        String original = "test shorturl.com !!!";
+        String expected = "test HTTP://WEBADDRESS.ELIDED !!!";
+        assertEquals(expected, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl5() {
+        String original = "test just.the.perfect.len.url !!!";
+        String expected = "test HTTP://WEBADDRESS.ELIDED !!!";
+        assertEquals(expected, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl6() {
+        String original = "test a.very.very.very.very.very.long.url !!!";
+        String expected = "test HTTP://WEBADDRESS.ELIDED !!!";
+        assertEquals(expected, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideUrl7() {
+        String original = " at android.content.Intent \n at java.util.ArrayList";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testDontElideFileSuffixes() {
+        String original = "chromium_android_linker.so";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testElideIp() {
+        String original = "traceroute 127.0.0.1";
+        String expected = "traceroute 1.2.3.4";
+        assertEquals(expected, LogcatExtractor.elideIp(original));
+    }
+
+    @Test
+    public void testElideMac1() {
+        String original = "MAC: AB-AB-AB-AB-AB-AB";
+        String expected = "MAC: 01:23:45:67:89:AB";
+        assertEquals(expected, LogcatExtractor.elideMac(original));
+    }
+
+    @Test
+    public void testElideMac2() {
+        String original = "MAC: AB:AB:AB:AB:AB:AB";
+        String expected = "MAC: 01:23:45:67:89:AB";
+        assertEquals(expected, LogcatExtractor.elideMac(original));
+    }
+
+    @Test
+    public void testElideConsole() {
+        String original = "I/chromium(123): [INFO:CONSOLE(2)] hello!";
+        String expected = "I/chromium(123): [ELIDED:CONSOLE(0)] ELIDED CONSOLE MESSAGE";
+        assertEquals(expected, LogcatExtractor.elideConsole(original));
+    }
+
+    @Test
+    public void testLogTagNotElided() {
+        List<String> original = Arrays.asList(new String[] {"I/cr_FooBar(123): Some message"});
+        assertEquals(original, LogcatExtractor.processLogcat(original));
+    }
+
+    @Test
+    public void testLogCastNotElided() {
+        String original = "09-08 11:52:12.569  4406  4406 I chromium: "
+                + "Cast.Discovery.Mdns.Request.AppId.In=25";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testLogAndroidNotElided() {
+        String original = "09-08 11:54:38.053  4406  4406 D cr_AvSettingsAndroid:"
+                + "[AvSettingsAndroid.java:180] HDMI plug update: "
+                + "action=android.media.action.HDMI_AUDIO_PLUG, plug=1";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testLogJavaNotElided() {
+        String original = "09-08 11:54:38.053  4406  4406 D cr_AvSettingsAndroid: "
+                + " [AvSettingsAndroid.java:187] Max channel count = 8";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+
+    @Test
+    public void testLogCcNotElided() {
+        String original = "09-08 11:59:12.335  4406  4443 I chromium: "
+                + "[4406:4443:INFO:wifi_util.cc(113)]   No peers:";
+        assertEquals(original, LogcatExtractor.elideUrl(original));
+    }
+}
\ No newline at end of file