Generalize path exclusions by supporting --exclude-paths parameter.

This CL removes hardcoded, path-based exclusions from the rewriter's
source code and moves them into a filter file.  This way, the exclusions
can be updated (or experimented with) without having to recompile the
rewriter.

Bug: 1069567
Change-Id: I7f0151fb8fdeb595da44e44c063caef0c154c31b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2337393
Commit-Queue: Łukasz Anforowicz <lukasza@chromium.org>
Reviewed-by: Bartek Nowierski <bartekn@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#796254}

tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt

+# File that lists regular expressions of paths that should be ignored when
+# running the rewrite_raw_ptr_fields tool on Chromium sources.
+#
+# If a source file path contains any of the lines in the filter file below,
+# then such source file will not be rewritten.
+#
+# Note that the rewriter has a hardcoded logic for a handful of path-based
+# exclusions that cannot be expressed as substring matches:
+# - Excluding paths containing "third_party/", but still covering
+#   "third_party/blink/"
+#   (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp).
+# - Excluding paths _starting_ with "gen/" or containing "/gen/"
+#   (i.e. hopefully just the paths under out/.../gen/... directory)
+#   via the isInGeneratedLocation AST matcher in RewriteRawPtrFields.cpp.
+
+# Exclude to prevent PartitionAlloc<->CheckedPtr cyclical dependency.
+base/allocator/
+
+# Exclude - deprecated and contains legacy C++ and pre-C++11 code.
+ppapi/
+
+# Exclude tools that do not ship in the Chrome binary.
+base/android/linker/
+tools/
+net/tools/
+
+# Exclude paths in separate repositories - i.e. in directories that
+# 1. Contain a ".git" subdirectory
+# 2. And hasn't been excluded via "third_party/" substring in their path
+#    (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp).
+#
+# The list below has been generated with:
+#
+#  $ find . -type d -name .git | \
+#      sed -e 's/\.git$//g' | \
+#      sed -e 's/\.\///g' | \
+#      grep -v third_party | \
+#      grep -v '^$' | \
+#      sort | uniq > ~/scratch/git-paths
+buildtools/
+buildtools/clang_format/script/
+chrome/app/theme/default_100_percent/google_chrome/
+chrome/app/theme/default_200_percent/google_chrome/
+chrome/app/theme/google_chrome/
+chrome/app/vector_icons/google_chrome/
+chrome/browser/chromeos/arc/voice_interaction/internal/
+chrome/browser/google/linkdoctor_internal/
+chrome/browser/internal/
+chrome/browser/media/engagement_internal/
+chrome/browser/media/kaleidoscope/internal/
+chrome/browser/resources/chromeos/quickoffice/
+chrome/browser/resources/media_router/extension/src/
+chrome/browser/resources/media_router_internal/
+chrome/browser/resources/settings_internal/
+chrome/browser/spellchecker/internal/
+chrome/browser/ui/media_router/internal/
+chrome/installer/mac/internal/
+chromeos/assistant/internal/
+chromeos/assistant/libassistant/deps/
+chromeos/assistant/libassistant/src/
+chromeos/assistant/libassistant/src/buildtools/
+chromeos/assistant/libassistant/src/buildtools/clang_format/script/
+chromeos/components/media_app_ui/resources/app/
+chrome/services/soda/internal/
+chrome/test/data/firefox3_profile/searchplugins/
+chrome/test/data/firefox3_searchplugins/
+chrome/test/data/gpu/vt/
+chrome/test/data/osdd/
+chrome/test/data/pdf_private/
+chrome/test/data/perf/canvas_bench/
+chrome/test/data/perf/frame_rate/content/
+chrome/test/data/perf/frame_rate/private/
+chrome/test/data/perf/private/
+chrome/test/data/xr/webvr_info/
+chrome/test/media_router/internal/
+chrome/test/python_tests/
+chrome/tools/test/reference_build/chrome_linux/
+clank/
+components/ntp_tiles/resources/internal/
+components/resources/default_100_percent/google_chrome/
+components/resources/default_200_percent/google_chrome/
+components/resources/default_300_percent/google_chrome/
+components/site_isolation/internal/
+content/test/data/plugin/
+data/autodiscovery/
+data/dom_perf/
+data/mach_ports/
+data/memory_test/
+data/mozilla_js_tests/
+data/page_cycler/
+data/selenium_core/
+data/tab_switching/
+google_apis/internal/
+libassistant/communication/
+libassistant/contrib/
+libassistant/internal/
+libassistant/resources/
+libassistant/shared/
+media/cdm/api/
+mojo/internal/
+native_client/
+remoting/android/internal/
+remoting/host/installer/linux/internal/
+remoting/internal/
+remoting/test/internal/
+remoting/tools/internal/
+remoting/webapp/app_remoting/internal/
+skia/tools/clusterfuzz-data/
+tools/histograms/
+tools/page_cycler/acid3/
+tools/perf/data/
+tools/swarming_client/
+ui/file_manager/internal/
+ui/webui/internal/
+v8/
+webkit/data/bmp_decoder/
+webkit/data/ico_decoder/
+webkit/data/test_shell/plugins/

tools/clang/rewrite_raw_ptr_fields/rewrite.sh

@@ -24,6 +24,9 @@ then
   OUT_DIR="$1"
 fi
 
+SCRIPT_PATH=$(realpath $0)
+REWRITER_SRC_DIR=$(dirname $SCRIPT_PATH)
+
 COMPILE_DIRS=.
 EDIT_DIRS=.
 
@@ -49,6 +52,7 @@ time ninja -C $OUT_DIR $GEN_H_TARGETS
 echo "*** Generating the ignore list ***"
 time tools/clang/scripts/run_tool.py \
     --tool rewrite_raw_ptr_fields \
+    --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \
     --generate-compdb \
     -p $OUT_DIR \
     $COMPILE_DIRS > ~/scratch/rewriter.out
@@ -64,6 +68,7 @@ echo "*** Running the main rewrite phase ***"
 time tools/clang/scripts/run_tool.py \
     --tool rewrite_raw_ptr_fields \
     --tool-arg=--exclude-fields=$HOME/scratch/combined-fields-to-ignore.txt \
+    --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \
     -p $OUT_DIR \
     $COMPILE_DIRS > ~/scratch/rewriter.main.out
 
@@ -73,12 +78,6 @@ cat ~/scratch/rewriter.main.out | \
     tools/clang/scripts/extract_edits.py | \
     tools/clang/scripts/apply_edits.py -p $OUT_DIR $EDIT_DIRS
 
-# Revert directories that are known to be troublesome and/or not needed.
-git checkout -- base/allocator/  # prevent cycles; CheckedPtr uses allocator
-git checkout -- ppapi/  # lots of legacy C and pre-C++11 code
-git checkout -- tools/  # not built into Chrome
-git checkout -- net/tools/  # not built into Chrome
-
 # Format sources, as many lines are likely over 80 chars now.
 echo "*** Formatting ***"
 time git cl format

tools/clang/rewrite_raw_ptr_fields/tests/fields-to-ignore.txt

-# File that can be used as an argument of the --blocked-fields cmdline
+# File that can be used as an argument of the --exclude-fields cmdline
 # parameter of rewrite_raw_ptr_fields tool.
 #
 # Each non-whitespace / non-comment line specified a

tools/clang/rewrite_raw_ptr_fields/tests/path-filter-file-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+struct MyStruct {
+  // No rewrite expected - this whole source file is mentioned in the
+  // tests/paths-to-ignore.txt file.
+  SomeClass* ptr_field_;
+};

tools/clang/rewrite_raw_ptr_fields/tests/path-filter-file-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+struct MyStruct {
+  // No rewrite expected - this whole source file is mentioned in the
+  // tests/paths-to-ignore.txt file.
+  SomeClass* ptr_field_;
+};

tools/clang/rewrite_raw_ptr_fields/tests/paths-to-ignore.txt

+# File that can be used as an argument of the --exclude-paths cmdline
+# parameter of rewrite_raw_ptr_fields tool.
+#
+# Each non-whitespace / non-comment line specifies a regex that is
+# matched against paths of the source files.  Paths matching any
+# regex will not be rewritten.
+
+# Next line is non-empty, but contains only whitespace.
+            
+
+# Next line contains a substring of the tests/path-filter-file-...cc file.
+th-filter-fi
+
+# The lines below don't match any test paths - these lines have been included to
+# exercise a bit further the regex # building code in
+# FilterFile::ContainsSubstringOf.
+#
+# Just a random substring that doesn't match any test paths.
+no-such-substring-any-where-in-test-files
+# Random regex characters that might need to be escaped.
+\( ( \) ) \| | [0-9] \[0-9\] .{}
+# A dot shouldn't match *any* character.
+.n  # anything other than .c and .cc should not match any test paths

tools/clang/rewrite_raw_ptr_fields/tests/run_tool.args

 --tool-arg=--exclude-fields=fields-to-ignore.txt
+--tool-arg=--exclude-paths=paths-to-ignore.txt