Android: Update java_deobfuscate to match more lines

Also adds a test, and a --logcat flag for better matching of logcat
lines.

Bug: 713710
Change-Id: Ie21030b3c4e68d02cfb2e0d65d2c4c4f00624a80
Reviewed-on: https://chromium-review.googlesource.com/657265Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#501535}

build/android/pylib/symbols/deobfuscator.py

@@ -118,6 +118,9 @@ class Deobfuscator(object):
 
 
 class DeobfuscatorPool(object):
+  # As of Sep 2017, each instance requires about 500MB of RAM, as measured by:
+  # /usr/bin/time -v out/Release/bin/java_deobfuscate \
+  #     out/Release/apks/ChromePublic.apk.mapping
   def __init__(self, mapping_path, pool_size=4):
     self._mapping_path = mapping_path
     self._pool = [Deobfuscator(mapping_path) for _ in xrange(pool_size)]

build/android/stacktrace/java/org/chromium/build/FlushingReTrace.java

@@ -20,25 +20,59 @@ import proguard.retrace.ReTrace;
  *  2. Disables output buffering
  */
 public class FlushingReTrace {
-    // This regex is based on the one from:
-    // http://proguard.sourceforge.net/manual/retrace/usage.html.
-    // But with the "at" part changed to "(?::|\bat)", to account for lines like:
-    //     06-22 13:58:02.895  4674  4674 E THREAD_STATE:     bLA.a(PG:173)
-    // And .*=%c\s* added as the second subpattern to account for lines like:
-    //     INSTRUMENTATION_STATUS: class=bNs
-    // Normal stack trace lines look like:
-    // java.lang.RuntimeException: Intentional Java Crash
-    //     at org.chromium.chrome.browser.tab.Tab.handleJavaCrash(Tab.java:682)
-    //     at org.chromium.chrome.browser.tab.Tab.loadUrl(Tab.java:644)
+    // E.g.: D/ConnectivityService(18029): Message
+    // E.g.: W/GCM     (15158): Message
+    // E.g.: 09-08 14:22:59.995 18029 18055 I ProcessStatsService: Message
+    // E.g.: 09-08 14:30:59.145 17731 18020 D MDnsDS  : Message
+    private static final String LOGCAT_PREFIX =
+            "(?:[VDIWEF]/.*?\\(\\d+\\): |\\d\\d-\\d\\d [0-9:. ]+[VDIWEF] .*?: )?";
+
+    // Note: Order of these sub-patterns defines their precedence.
+    // Note: Deobfuscation of methods without the presense of line numbers basically never works.
+    // There is a test for these pattern at //build/android/stacktrace/java_deobfuscate_test.py
     private static final String LINE_PARSE_REGEX =
-            "(?:.*?(?::|\\bat)\\s+%c\\.%m\\s*\\(%s(?::%l)?\\)\\s*)|"
-            + "(?:.*=%c\\s*)|"
-            + "(?:(?:.*?[:\"]\\s+)?%c(?::.*)?)";
+            // Eagerly match logcat prefix to avoid conflicting with the patterns below.
+            LOGCAT_PREFIX
+            + "(?:"
+            // Based on default ReTrace regex, but with "at" changed to to allow :
+            // E.g.: 06-22 13:58:02.895  4674  4674 E THREAD_STATE:     bLA.a(PG:173)
+            // Normal stack trace lines look like:
+            // \tat org.chromium.chrome.browser.tab.Tab.handleJavaCrash(Tab.java:682)
+            + "(?:.*?(?::|\\bat)\\s+%c\\.%m\\s*\\(%s(?::%l)?\\))|"
+            // E.g.: VFY: unable to resolve new-instance 3810 (LSome/Framework/Class;) in Lfoo/Bar;
+            + "(?:.*L%C;.*)|"
+            // E.g.: END SomeTestClass#someMethod
+            + "(?:.*?%c#%m.*?)|"
+            // E.g.: The member "Foo.bar"
+            // E.g.: The class "Foobar"
+            + "(?:.*?\"%c\\.%m\".*)|"
+            + "(?:.*?\"%c\".*)|"
+            // E.g.: java.lang.RuntimeException: Intentional Java Crash
+            + "(?:%c:.*)|"
+            // All lines that end with a class / class+method:
+            // E.g.: The class: Foo
+            // E.g.: INSTRUMENTATION_STATUS: class=Foo
+            // E.g.: NoClassDefFoundError: SomeFrameworkClass in isTestClass for Foo
+            // E.g.: Could not find class 'SomeFrameworkClass', referenced from method Foo.bar
+            // E.g.: Could not find method SomeFrameworkMethod, referenced from method Foo.bar
+            + "(?:.*(?:=|:\\s*|\\s+)%c\\.%m)|"
+            + "(?:.*(?:=|:\\s*|\\s+)%c)"
+            + ")";
+
+    private static void usage() {
+        System.err.println("Usage: echo $OBFUSCATED_CLASS | java_deobfuscate Foo.apk.mapping");
+        System.err.println("Usage: java_deobfuscate Foo.apk.mapping < foo.log");
+        System.err.println("Note: Deobfuscation of symbols outside the context of stack "
+                + "traces will work only when lines match the regular expression defined "
+                + "in FlushingReTrace.java.");
+        System.err.println("Also: Deobfuscation of method names without associated line "
+                + "numbers does not seem to work.");
+        System.exit(1);
+    }
 
     public static void main(String[] args) {
-        if (args.length != 1) {
-            System.err.println("Usage: java_deobfuscate Foo.apk.map < foo.log > bar.log");
-            System.exit(1);
+        if (args.length != 1 || args[0].startsWith("-")) {
+            usage();
         }
 
         File mappingFile = new File(args[0]);

build/android/stacktrace/java_deobfuscate_test.py

+#!/usr/bin/env python
+#
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Tests for java_deobfuscate."""
+
+import argparse
+import subprocess
+import sys
+import tempfile
+
+
+LINE_PREFIXES = [
+    '',
+    # logcat -v threadtime
+    '09-08 14:38:35.535 18029 18084 E qcom_sensors_hal: ',
+    # logcat
+    'W/GCM     (15158): ',
+]
+
+TEST_MAP = """\
+this.was.Deobfuscated -> FOO:
+    int[] FontFamily -> a
+    1:3:void someMethod(int,android.os.Bundle):65:65 -> bar
+"""
+
+TEST_DATA = """\
+Here is a FOO
+Here is a FOO baz
+Here is a "FOO" baz
+Here is a "FOO.bar" baz
+Here it is: FOO
+Here it is: FOO.bar
+Here is a FOO.bar
+Here is a FOO.bar baz
+END FOO#bar
+new-instance 3810 (LSome/Framework/Class;) in LFOO;
+FOO: Error message
+\tat FOO.bar(PG:1)
+""".splitlines(True)
+
+EXPECTED_OUTPUT = """\
+Here is a this.was.Deobfuscated
+Here is a FOO baz
+Here is a "this.was.Deobfuscated" baz
+Here is a "this.was.Deobfuscated.someMethod" baz
+Here it is: this.was.Deobfuscated
+Here it is: this.was.Deobfuscated.someMethod
+Here is a this.was.Deobfuscated.someMethod
+Here is a FOO.bar baz
+END this.was.Deobfuscated#someMethod
+new-instance 3810 (LSome/Framework/Class;) in Lthis/was/Deobfuscated;
+this.was.Deobfuscated: Error message
+\tat this.was.Deobfuscated.someMethod(Deobfuscated.java:65)
+""".splitlines(True)
+
+
+def _RunTest(bin_path, map_file, prefix):
+  cmd = [bin_path, map_file]
+  payload = TEST_DATA
+  expected_output = EXPECTED_OUTPUT
+
+  payload = [prefix + x for x in payload]
+  expected_output = [prefix + x for x in expected_output]
+
+  proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  actual_output = proc.communicate(''.join(payload))[0].splitlines(True)
+  any_unexpected_failures = False
+  for actual, expected in zip(actual_output, expected_output):
+    if actual == expected:
+      sys.stdout.write('Good: ' + actual)
+    elif actual.replace('bar', 'someMethod') == expected:
+      # TODO(agrieve): Fix ReTrace's ability to deobfuscated methods.
+      sys.stdout.write('Fine: ' + actual)
+    else:
+      sys.stdout.write('BAD:  ' + actual)
+      any_unexpected_failures = True
+  return not any_unexpected_failures
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('path_to_java_deobfuscate')
+  args = parser.parse_args()
+
+  with tempfile.NamedTemporaryFile() as map_file:
+    map_file.write(TEST_MAP)
+    map_file.flush()
+    passed = True
+    for prefix in LINE_PREFIXES:
+      if not _RunTest(args.path_to_java_deobfuscate, map_file.name, prefix):
+        passed = False
+  print 'Result:', 'PASS' if passed else 'FAIL'
+  sys.exit(int(not passed))
+
+
+if __name__ == '__main__':
+  main()