Implement a tool to annotate methods with tracing

Create a tool to enable developers to annotate large amount of code with
tracing. This is an MVP which just adds tracing of the method name.

Bug: 1111787
Change-Id: I7a6fd7f51b9a3887074b1f53f005f1c0c8671a4b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2332401
Commit-Queue: Karel Král <karelkral@google.com>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Łukasz Anforowicz <lukasza@chromium.org>
Cr-Commit-Position: refs/heads/master@{#800515}

tools/clang/trace_annotator/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  BitReader
+  MCParser
+  Option
+  )
+
+add_llvm_executable(trace_annotator
+  TraceAnnotator.cpp
+  )
+
+target_link_libraries(trace_annotator
+  clangAST
+  clangASTMatchers
+  clangAnalysis
+  clangBasic
+  clangDriver
+  clangEdit
+  clangFrontend
+  clangLex
+  clangParse
+  clangSema
+  clangSerialization
+  clangTooling
+  )
+
+cr_install(TARGETS trace_annotator RUNTIME DESTINATION bin)

tools/clang/trace_annotator/README.md

+# Trace Annotator Tool
+
+## Introduction
+
+A tool to annotate functions with ```c++ TRACE_EVENT```. This tool serves two
+workflows:
+
+*  Debugging: bulk add traces to all functions in a directory and after finding
+   the bug remove the traces.
+*  Adding traces to code: bulk add traces to all functions in a directory,
+   review the changes carefully and create a patch.
+
+The goal of this tool is to transfer a function:
+```c++
+int foo(int bar, int baz) {
+  return 42;
+}
+```
+into:
+```c++
+int foo(int bar, int baz) {
+  TRACE_EVENT0("test", "foo");
+  return 42;
+}
+```
+
+In future also argument tracing is to be supported.
+
+This document is based on //docs/clang_tool_refactoring.md
+
+## Building
+
+The following might take approx. 2 hours depending on your computer.
+
+*  Make a new checkout of chromium (suggested, but optional).
+*  From chromium/src:
+*  ```shell cr build all``` To make sure all files have been generated.
+*  ```shell cp -R third_party/llvm-build ~```
+*  ```shell tools/clang/scripts/build.py --bootstrap --without-android
+   --without-fuchsia --extra-tools trace_annotator```
+   *  TODO how to build with plugin 'find-bad-constructs'?
+*  ```shell cp -R ~/llvm-build third_party``` This should enable goma support
+   again.
+
+### Rebuild just the tool:
+
+*  ```shell cd third_party/llvm-build/Release+Asserts```
+*  ```shell ninja trace_annotator```
+
+Beware that running ```shell gclient sync``` might overwrite the build and
+another full build might be necessary. A backup of the binary from
+//third_party/llvm-build/Release+Asserts/bin/trace_annotator might be useful.
+
+## Testing
+
+*  ```shell tools/clang/scripts/test_tool.py --apply-edits trace_annotator```
+
+## Running
+
+*  Chrome plugins are not supported yet, run: ```shell gn args out/Debug/``` and
+   add: ```clang_use_chrome_plugins = false``` option.
+*  Make sure you have up to date compilation database:
+   *  To generate it run: ```shell tools/clang/scripts/generate_compdb.py -p
+      out/Debug/ > out/Debug/compile_commands.json```
+   *  These are the compiler options for individual files (needed to use the
+      right version of C++, right library paths...).
+
+*  ```shell DIR="net"; \
+      git checkout $DIR && tools/clang/scripts/run_tool.py --tool trace_annotator -p out/Debug/ $DIR \
+      | tools/clang/scripts/extract_edits.py \
+      | tools/clang/scripts/apply_edits.py -p out/Debug $DIR \
+      && git cl format $DIR```
+
+*  Consult documentation of ```//tools/clang/scripts/run_tool.py``` for more
+   options.
+
+### Suggestion:
+
+Do not run the tool on //base or anything that has to do with tracing or
+synchronization. Or at least do not submit the resulting patch.
+
+### Debugging workflow suggestion:
+
+*  Do some changes.
+*  ```shell git add . ; git commit```
+*  Run the tool.
+*  ```shell git add . ; git commit```
+*  Do some more changes (including fixing a bug).
+*  ```shell git add . ; git commit```
+*  ```shell git rebase -i``` and follow the help.
+
+### Creating tracing patch suggestion:
+
+*  Run the tool.
+*  Double check all generated code.
+*  Add method annotations for methods that are hidden by compiler options (e.g.,
+   if you are on unix then the code in ```c++ #ifdef OS_WIN``` will not be
+   annotated.
+
+## TODO
+
+*  Add options:
+   *  Whether to add "do not submit" comment (in upper case).
+   *  Function name formatting (without namespace(s) / getQualifiedNameAsString /
+      with namespaces but without template tags).
+   *  Category name.
+   *  Make tracing of function arguments.
+*  Standalone build of the tool (outside of //third_party to avoid overwriting
+   by ```shell gclient sync```).

tools/clang/trace_annotator/TraceAnnotator.cpp

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// This implements a Clang tool to annotate methods with tracing. It should be
+// run using the tools/clang/scripts/run_tool.py helper as described in
+// README.md
+
+#include <string>
+#include <vector>
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FormatVariadic.h"
+
+using namespace clang::ast_matchers;
+using clang::tooling::CommonOptionsParser;
+using clang::tooling::Replacement;
+using clang::tooling::Replacements;
+
+namespace {
+
+class FunctionDefCallback : public MatchFinder::MatchCallback {
+ public:
+  explicit FunctionDefCallback(std::vector<Replacement>* replacements)
+      : replacements_(replacements) {}
+
+  void run(const MatchFinder::MatchResult& result) override;
+
+ private:
+  std::vector<Replacement>* const replacements_;
+};
+
+class TraceAnnotator {
+ public:
+  explicit TraceAnnotator(std::vector<Replacement>* replacements)
+      : function_def_callback_(replacements) {}
+
+  void SetupMatchers(MatchFinder* match_finder);
+
+ private:
+  FunctionDefCallback function_def_callback_;
+};
+
+// Given:
+//   template <typename T, typename T2> void foo(T t, T2 t2) {};  // N1 and N4
+//   template <typename T2> void foo<int, T2>(int t, T2 t) {};    // N2
+//   template <> void foo<int, char>(int t, char t2) {};          // N3
+//   void foo() {
+//     // This creates implicit template specialization (N4) out of the
+//     // explicit template definition (N1).
+//     foo<bool, double>(true, 1.23);
+//   }
+// with the following AST nodes:
+//   FunctionTemplateDecl foo
+//   |-FunctionDecl 0x191da68 foo 'void (T, T2)'         // N1
+//   `-FunctionDecl 0x194bf08 foo 'void (bool, double)'  // N4
+//   FunctionTemplateDecl foo
+//   `-FunctionDecl foo 'void (int, T2)'                 // N2
+//   FunctionDecl foo 'void (int, char)'                 // N3
+//
+// Matches AST node N4, but not AST nodes N1, N2 nor N3.
+AST_MATCHER(clang::FunctionDecl, isImplicitFunctionTemplateSpecialization) {
+  switch (Node.getTemplateSpecializationKind()) {
+    case clang::TSK_ImplicitInstantiation:
+      return true;
+    case clang::TSK_Undeclared:
+    case clang::TSK_ExplicitSpecialization:
+    case clang::TSK_ExplicitInstantiationDeclaration:
+    case clang::TSK_ExplicitInstantiationDefinition:
+      return false;
+  }
+}
+
+AST_POLYMORPHIC_MATCHER(isInMacroLocation,
+                        AST_POLYMORPHIC_SUPPORTED_TYPES(clang::Decl,
+                                                        clang::Stmt,
+                                                        clang::TypeLoc)) {
+  return Node.getBeginLoc().isMacroID();
+}
+
+void TraceAnnotator::SetupMatchers(MatchFinder* match_finder) {
+  const clang::ast_matchers::DeclarationMatcher function_call =
+      functionDecl(
+          has(compoundStmt().bind("function body")),
+          /* Avoid matching the following cases: */
+          unless(anyOf(
+              /* Do not match implicit function template specializations to
+                 avoid conflicting edits. */
+              isImplicitFunctionTemplateSpecialization(),
+              /* Do not match constexpr functions. */
+              isConstexpr(), isDefaulted(),
+              /* Do not match ctor/dtor. */
+              cxxConstructorDecl(), cxxDestructorDecl(),
+              /* Tracing macros can be tricky (e.g., QuicUint128Impl comparison
+                 operators). */
+              isInMacroLocation(), has(compoundStmt(isInMacroLocation())),
+              /* Do not trace lambdas (no name, possbly tracking more parameters
+                 than intended because of [&]). */
+              hasParent(cxxRecordDecl(isLambda())))))
+          .bind("function");
+  match_finder->addMatcher(function_call, &function_def_callback_);
+}
+
+// Returns a string containing the qualified name of the function. Does not
+// output template parameters of the function or in case of methods of the
+// associated class (as opposed to |function->getQualifiedNameAsString|).
+std::string getFunctionName(const clang::FunctionDecl* function) {
+  std::string qualified_name;
+  // Add namespace(s) to the name.
+  if (auto* name_space = llvm::dyn_cast<clang::NamespaceDecl>(
+          function->getEnclosingNamespaceContext())) {
+    qualified_name += name_space->getQualifiedNameAsString();
+    qualified_name += "::";
+  }
+  // If the function is a method, add class name (without templates).
+  if (auto* method = llvm::dyn_cast<clang::CXXMethodDecl>(function)) {
+    qualified_name += method->getParent()->getNameAsString();
+    qualified_name += "::";
+  }
+  // Add function name (without templates).
+  qualified_name += function->getNameAsString();
+  return qualified_name;
+}
+
+void FunctionDefCallback::run(const MatchFinder::MatchResult& result) {
+  const clang::FunctionDecl* function =
+      result.Nodes.getNodeAs<clang::FunctionDecl>("function");
+  // Using this instead of |function->getBody| prevents conflicts with parameter
+  // names in headers and implementations.
+  const clang::CompoundStmt* function_body =
+      result.Nodes.getNodeAs<clang::CompoundStmt>("function body");
+  clang::CharSourceRange range =
+      clang::CharSourceRange::getTokenRange(function_body->getBeginLoc());
+
+  const char kReplacementTextTemplate[] = R"( TRACE_EVENT0("test", "{0}"); )";
+  std::string function_name = getFunctionName(function);
+  std::string replacement_text =
+      llvm::formatv(kReplacementTextTemplate, function_name).str();
+
+  const char kAnnotationTemplate[] = " { {0}";
+  std::string annotation =
+      llvm::formatv(kAnnotationTemplate, replacement_text).str();
+
+  replacements_->push_back(
+      Replacement(*result.SourceManager, range, annotation));
+}
+
+}  // namespace
+
+static llvm::cl::extrahelp common_help(CommonOptionsParser::HelpMessage);
+
+int main(int argc, const char* argv[]) {
+  llvm::cl::OptionCategory category("TraceAnnotator Tool");
+  CommonOptionsParser options(argc, argv, category);
+  clang::tooling::ClangTool tool(options.getCompilations(),
+                                 options.getSourcePathList());
+
+  std::vector<Replacement> replacements;
+  TraceAnnotator converter(&replacements);
+  MatchFinder match_finder;
+  converter.SetupMatchers(&match_finder);
+
+  std::unique_ptr<clang::tooling::FrontendActionFactory> frontend_factory =
+      clang::tooling::newFrontendActionFactory(&match_finder);
+  int result = tool.run(frontend_factory.get());
+  if (result != 0)
+    return result;
+
+  if (replacements.empty())
+    return 0;
+
+  // Each replacement line should have the following format:
+  // r:<file path>:<offset>:<length>:<replacement text>
+  // Only the <replacement text> field can contain embedded ":" characters.
+  // TODO(dcheng): Use a more clever serialization. Ideally we'd use the YAML
+  // serialization and then use clang-apply-replacements, but that would require
+  // copying and pasting a larger amount of boilerplate for all Chrome clang
+  // tools.
+
+  // Keep a set of files where we have already added base_tracing include.
+  std::set<std::string> include_added_to;
+
+  llvm::outs() << "==== BEGIN EDITS ====\n";
+  for (const auto& r : replacements) {
+    // Add base_tracing import if necessary.
+    if (include_added_to.find(r.getFilePath().str()) ==
+        include_added_to.end()) {
+      include_added_to.insert(r.getFilePath().str());
+      // Add also copyright so that |test-expected.cc| passes presubmit.
+      llvm::outs() << "include-user-header:::" << r.getFilePath()
+                   << ":::-1:::-1:::base/trace_event/base_tracing.h"
+                   << "\n";
+    }
+    // Add the actual replacement.
+    llvm::outs() << "r:::" << r.getFilePath() << ":::" << r.getOffset()
+                 << ":::" << r.getLength() << ":::" << r.getReplacementText()
+                 << "\n";
+  }
+  llvm::outs() << "==== END EDITS ====\n";
+
+  return 0;
+}

tools/clang/trace_annotator/tests/test-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <algorithm>
+#include <vector>
+
+#include "base/trace_event/base_tracing.h"
+
+int no_body(int);  // No annotation
+
+int foo(int, char) {
+  TRACE_EVENT0("test", "foo");
+  return 13;
+}
+
+namespace testnamespace {
+
+namespace nestednamespace {
+
+int fo0(int bar, char faz) {
+  TRACE_EVENT0("test", "testnamespace::nestednamespace::fo0");
+  int baz = bar + 10;
+  return baz;
+}
+
+}  // namespace nestednamespace
+
+template <typename T>
+T twice(T x) {
+  TRACE_EVENT0("test", "testnamespace::twice");
+  return 2 * x;
+}
+
+}  // namespace testnamespace
+
+class Aclass {
+ public:
+  Aclass() {}   // Constructor should not be annotated.
+  ~Aclass() {}  // Destructor should not be annotated.
+  int furt(int par1, char par2) {
+    TRACE_EVENT0("test", "Aclass::furt");
+    return par1 + par2;
+  }
+
+  // Should not be annotated (or =default exchanged for a body)
+  bool operator==(const Aclass&) const = default;
+};
+
+template <typename T>
+class TemplatedClass {
+ public:
+  int fun();
+};
+
+template <typename T>
+class Specialized {
+ public:
+  int f() {
+    TRACE_EVENT0("test", "Specialized::f");
+    return 1;
+  }
+};
+
+template <>
+class Specialized<double> {
+ public:
+  int f() {
+    TRACE_EVENT0("test", "Specialized::f");
+    return 1;
+  }
+};
+
+namespace double_fun {
+
+template <typename T, typename S>
+class DoubleTemplate {
+ public:
+  int fun() {
+    TRACE_EVENT0("test", "double_fun::DoubleTemplate::fun");
+    return 0;
+  }
+};
+
+template <typename S>
+class DoubleTemplate<int, S> {
+ public:
+  int fun() {
+    TRACE_EVENT0("test", "double_fun::DoubleTemplate::fun");
+    return 0;
+  }
+};
+
+template <typename T>
+class DoubleTemplate<T, int> {
+ public:
+  int fun() {
+    TRACE_EVENT0("test", "double_fun::DoubleTemplate::fun");
+    return 0;
+  }
+};
+
+template <>
+class DoubleTemplate<int, int> {
+ public:
+  int fun() {
+    TRACE_EVENT0("test", "double_fun::DoubleTemplate::fun");
+    return 0;
+  }
+};
+
+}  // namespace double_fun
+
+int main(int argc, char* argv[]) {
+  TRACE_EVENT0("test", "main");
+  int four = testnamespace::twice<int>(1);
+  double two = testnamespace::twice<double>(1.0);
+
+  TemplatedClass<int> itc;
+  TemplatedClass<double> dtc;
+  int zero = itc.fun() + dtc.fun();
+  foo(1, 'a');
+
+  Specialized<int> si;
+  zero += si.f();
+
+  double_fun::DoubleTemplate<char, char> dtcc;
+  double_fun::DoubleTemplate<char, int> dtci;
+  double_fun::DoubleTemplate<int, char> dtic;
+  double_fun::DoubleTemplate<int, int> dtii;
+  int funny_zero = dtcc.fun() + dtci.fun() + dtic.fun() + dtii.fun();
+
+  std::vector<int> v = {3, 1, 4, 1, 5, 9};
+  std::sort(v.begin(), v.end(), [](int a, int b) { return a < b; });
+
+  return 0;
+}
+
+template <typename T>
+int TemplatedClass<T>::fun() {
+  TRACE_EVENT0("test", "TemplatedClass::fun");
+  return 0;
+}

tools/clang/trace_annotator/tests/test-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <algorithm>
+#include <vector>
+
+int no_body(int);  // No annotation
+
+int foo(int, char) {
+  return 13;
+}
+
+namespace testnamespace {
+
+namespace nestednamespace {
+
+int fo0(int bar, char faz) {
+  int baz = bar + 10;
+  return baz;
+}
+
+}  // namespace nestednamespace
+
+template <typename T>
+T twice(T x) {
+  return 2 * x;
+}
+
+}  // namespace testnamespace
+
+class Aclass {
+ public:
+  Aclass() {}   // Constructor should not be annotated.
+  ~Aclass() {}  // Destructor should not be annotated.
+  int furt(int par1, char par2) { return par1 + par2; }
+
+  // Should not be annotated (or =default exchanged for a body)
+  bool operator==(const Aclass&) const = default;
+};
+
+template <typename T>
+class TemplatedClass {
+ public:
+  int fun();
+};
+
+template <typename T>
+class Specialized {
+ public:
+  int f() { return 1; }
+};
+
+template <>
+class Specialized<double> {
+ public:
+  int f() { return 1; }
+};
+
+namespace double_fun {
+
+template <typename T, typename S>
+class DoubleTemplate {
+ public:
+  int fun() { return 0; }
+};
+
+template <typename S>
+class DoubleTemplate<int, S> {
+ public:
+  int fun() { return 0; }
+};
+
+template <typename T>
+class DoubleTemplate<T, int> {
+ public:
+  int fun() { return 0; }
+};
+
+template <>
+class DoubleTemplate<int, int> {
+ public:
+  int fun() { return 0; }
+};
+
+}  // namespace double_fun
+
+int main(int argc, char* argv[]) {
+  int four = testnamespace::twice<int>(1);
+  double two = testnamespace::twice<double>(1.0);
+
+  TemplatedClass<int> itc;
+  TemplatedClass<double> dtc;
+  int zero = itc.fun() + dtc.fun();
+  foo(1, 'a');
+
+  Specialized<int> si;
+  zero += si.f();
+
+  double_fun::DoubleTemplate<char, char> dtcc;
+  double_fun::DoubleTemplate<char, int> dtci;
+  double_fun::DoubleTemplate<int, char> dtic;
+  double_fun::DoubleTemplate<int, int> dtii;
+  int funny_zero = dtcc.fun() + dtci.fun() + dtic.fun() + dtii.fun();
+
+  std::vector<int> v = {3, 1, 4, 1, 5, 9};
+  std::sort(v.begin(), v.end(), [](int a, int b) { return a < b; });
+
+  return 0;
+}
+
+template <typename T>
+int TemplatedClass<T>::fun() {
+  return 0;
+}