Handling of basics: |Pointee* field| -> |CheckedPtr<Pointee> field|.

Bug: 1069567
Change-Id: Ie83b99f25e5d8d13f466c91f8d5118bfd039e856
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2144418Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Commit-Queue: Łukasz Anforowicz <lukasza@chromium.org>
Cr-Commit-Position: refs/heads/master@{#759921}

tools/clang/rewrite_raw_ptr_fields/CMakeLists.txt

+set(LLVM_LINK_COMPONENTS
+  BitReader
+  MCParser
+  Option
+  X86AsmParser
+  X86CodeGen
+  )
+
+add_llvm_executable(rewrite_raw_ptr_fields
+  RewriteRawPtrFields.cpp
+  )
+
+target_link_libraries(rewrite_raw_ptr_fields
+  clangAST
+  clangASTMatchers
+  clangAnalysis
+  clangBasic
+  clangDriver
+  clangEdit
+  clangFrontend
+  clangLex
+  clangParse
+  clangSema
+  clangSerialization
+  clangTooling
+  )
+
+cr_install(TARGETS rewrite_raw_ptr_fields RUNTIME DESTINATION bin)

tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This is implementation of a clang tool that rewrites raw pointer fields into
+// CheckedPtr<T>:
+//     Pointee* field_
+// becomes:
+//     CheckedPtr<Pointee> field_
+//
+// For more details, see the doc here:
+// https://docs.google.com/document/d/1chTvr3fSofQNV_PDPEHRyUgcJCQBgTDOOBriW9gIm9M
+
+#include <assert.h>
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/ASTMatchers/ASTMatchersMacros.h"
+#include "clang/Basic/CharInfo.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/MacroArgs.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TargetSelect.h"
+
+using namespace clang::ast_matchers;
+using clang::tooling::CommonOptionsParser;
+using clang::tooling::Replacement;
+
+namespace {
+
+class FieldDeclRewriter : public MatchFinder::MatchCallback {
+ public:
+  explicit FieldDeclRewriter(std::vector<Replacement>* replacements)
+      : replacements_(replacements) {}
+
+  void run(const MatchFinder::MatchResult& result) override {
+    const clang::SourceManager& source_manager = *result.SourceManager;
+    const clang::FieldDecl* field_decl =
+        result.Nodes.getNodeAs<clang::FieldDecl>("fieldDecl");
+    const clang::TypeSourceInfo* type_source_info =
+        field_decl->getTypeSourceInfo();
+
+    clang::QualType pointer_type = type_source_info->getType();
+    assert(type_source_info->getType()->isPointerType() &&
+           "matcher should only match pointer types");
+
+    // Calculate the |replacement_range|.
+    //
+    // Consider the following example:
+    //      const Pointee* const field_name_;
+    //      ^-------------------^   = |replacement_range|
+    //                           ^  = |field_decl->getLocation()|
+    //      ^                       = |field_decl->getBeginLoc()|
+    //                   ^          = PointerTypeLoc::getStarLoc
+    //            ^------^          = TypeLoc::getSourceRange
+    //
+    // We get the |replacement_range| in a bit clumsy way, because clang docs
+    // for QualifiedTypeLoc explicitly say that these objects "intentionally do
+    // not provide source location for type qualifiers".
+    clang::SourceRange replacement_range(
+        field_decl->getBeginLoc(),
+        field_decl->getLocation().getLocWithOffset(-1));
+
+    // Generate and add a replacement.
+    replacements_->emplace_back(
+        source_manager, clang::CharSourceRange::getCharRange(replacement_range),
+        GenerateNewText(pointer_type));
+  }
+
+ private:
+  std::string GenerateNewText(const clang::QualType& pointer_type) {
+    assert(pointer_type->isPointerType() && "caller must pass a pointer type!");
+    clang::QualType pointee_type = pointer_type->getPointeeType();
+
+    // Convert pointee type to string.
+    clang::LangOptions lang_options;
+    clang::PrintingPolicy printing_policy(lang_options);
+    printing_policy.SuppressTagKeyword = 1;  // s/class Pointee/Pointee/
+    std::string pointee_type_as_string =
+        pointee_type.getAsString(printing_policy);
+
+    // TODO(lukasza): Preserve qualifiers from |pointer_type| by generating
+    // results from fresh AST (rather than via string concatenation).
+    return std::string("CheckedPtr<") + pointee_type_as_string + ">";
+  }
+
+  std::vector<Replacement>* const replacements_;
+};
+
+}  // namespace
+
+int main(int argc, const char* argv[]) {
+  // TODO(dcheng): Clang tooling should do this itself.
+  // http://llvm.org/bugs/show_bug.cgi?id=21627
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmParser();
+  llvm::cl::OptionCategory category(
+      "rewrite_raw_ptr_fields: changes |T* field_| to |CheckedPtr<T> field_|.");
+  CommonOptionsParser options(argc, argv, category);
+  clang::tooling::ClangTool tool(options.getCompilations(),
+                                 options.getSourcePathList());
+
+  MatchFinder match_finder;
+  std::vector<Replacement> replacements;
+
+  // Field declarations =========
+  // Given
+  //   struct S {
+  //     int* y;
+  //   };
+  // matches |int* y|.
+  auto field_decl_matcher = fieldDecl(hasType(pointerType())).bind("fieldDecl");
+  FieldDeclRewriter field_decl_rewriter(&replacements);
+  match_finder.addMatcher(field_decl_matcher, &field_decl_rewriter);
+
+  // Prepare and run the tool.
+  std::unique_ptr<clang::tooling::FrontendActionFactory> factory =
+      clang::tooling::newFrontendActionFactory(&match_finder);
+  int result = tool.run(factory.get());
+  if (result != 0)
+    return result;
+
+  // Serialization format is documented in tools/clang/scripts/run_tool.py
+  llvm::outs() << "==== BEGIN EDITS ====\n";
+  for (const auto& r : replacements) {
+    std::string replacement_text = r.getReplacementText().str();
+    std::replace(replacement_text.begin(), replacement_text.end(), '\n', '\0');
+    llvm::outs() << "r:::" << r.getFilePath() << ":::" << r.getOffset()
+                 << ":::" << r.getLength() << ":::" << replacement_text << "\n";
+  }
+  llvm::outs() << "==== END EDITS ====\n";
+
+  return 0;
+}

tools/clang/rewrite_raw_ptr_fields/tests/attributes-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+// Based on Chromium's //base/thread_annotations.h
+#define GUARDED_BY(lock) __attribute__((guarded_by(lock)))
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> field GUARDED_BY(lock);
+  CheckedPtr<SomeClass> field GUARDED_BY(lock);
+  int lock;
+};

tools/clang/rewrite_raw_ptr_fields/tests/attributes-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+// Based on Chromium's //base/thread_annotations.h
+#define GUARDED_BY(lock) __attribute__((guarded_by(lock)))
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> field GUARDED_BY(lock);
+  SomeClass* field GUARDED_BY(lock);
+  int lock;
+};

tools/clang/rewrite_raw_ptr_fields/tests/basics-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field;
+  CheckedPtr<SomeClass> raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+struct MyStruct {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field;
+  CheckedPtr<SomeClass> raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+template <typename T>
+class MyTemplate {
+  // Expected rewrite: CheckedPtr<T> raw_ptr_field;
+  CheckedPtr<T> raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+// The field below won't compile without the |typename| keyword (because
+// at this point we don't know if MaybeProvidesType<T>::Type is a type,
+// value or something else).  Let's see if the rewriter will preserve
+// preserve the |typename| keyword.
+template <typename T>
+struct MaybeProvidesType;
+template <typename T>
+struct DependentNameTest {
+  // Expected rewrite: CheckedPtr<typename MaybeProvidesType<T>::Type> field;
+  CheckedPtr<typename MaybeProvidesType<T>::Type> field;
+};

tools/clang/rewrite_raw_ptr_fields/tests/basics-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field;
+  SomeClass* raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+struct MyStruct {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field;
+  SomeClass* raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+template <typename T>
+class MyTemplate {
+  // Expected rewrite: CheckedPtr<T> raw_ptr_field;
+  T* raw_ptr_field;
+
+  // No rewrite expected.
+  int int_field;
+};
+
+// The field below won't compile without the |typename| keyword (because
+// at this point we don't know if MaybeProvidesType<T>::Type is a type,
+// value or something else).  Let's see if the rewriter will preserve
+// preserve the |typename| keyword.
+template <typename T>
+struct MaybeProvidesType;
+template <typename T>
+struct DependentNameTest {
+  // Expected rewrite: CheckedPtr<typename MaybeProvidesType<T>::Type> field;
+  typename MaybeProvidesType<T>::Type* field;
+};

tools/clang/rewrite_raw_ptr_fields/tests/initializers-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+SomeClass* GetPointer();
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field = GetPointer();
+  CheckedPtr<SomeClass> raw_ptr_field = GetPointer();
+};

tools/clang/rewrite_raw_ptr_fields/tests/initializers-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+SomeClass* GetPointer();
+
+class MyClass {
+  // Expected rewrite: CheckedPtr<SomeClass> raw_ptr_field = GetPointer();
+  SomeClass* raw_ptr_field = GetPointer();
+};

tools/clang/rewrite_raw_ptr_fields/tests/qualifiers-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+class MyClass {
+  MyClass() : ptr_field3_(nullptr), ptr_field6_(nullptr) {}
+
+  // Expected rewrite: CheckedPtr<const SomeClass> ptr_field1_;
+  CheckedPtr<const SomeClass> ptr_field1_;
+
+  // Expected rewrite: CheckedPtr<volatile SomeClass> ptr_field2_;
+  CheckedPtr<volatile SomeClass> ptr_field2_;
+
+  // Expected rewrite: const CheckedPtr<SomeClass> ptr_field3_;
+  //
+  // TODO(lukasza): Fix this by using |qualType.getAsString|.
+  // Currently the "outer" |const| is dropped.
+  CheckedPtr<SomeClass> ptr_field3_;
+
+  // Expected rewrite: mutable CheckedPtr<SomeClass> ptr_field4_;
+  //
+  // TODO(lukasza): Fix this by looking at |field_decl->isMutable()|.
+  // Currently the |mutable| specifier is dropped.
+  CheckedPtr<SomeClass> ptr_field4_;
+
+  // Expected rewrite: CheckedPtr<const SomeClass> ptr_field5_;
+  CheckedPtr<const SomeClass> ptr_field5_;
+
+  // Expected rewrite: volatile CheckedPtr<const SomeClass> ptr_field6_;
+  //
+  // TODO(lukasza): Fix this by using |qualType.getAsString|.
+  // Currently the "outer" qualifiers (like |volatile| below) are dropped.
+  CheckedPtr<const SomeClass> ptr_field6_;
+};

tools/clang/rewrite_raw_ptr_fields/tests/qualifiers-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+class MyClass {
+  MyClass() : ptr_field3_(nullptr), ptr_field6_(nullptr) {}
+
+  // Expected rewrite: CheckedPtr<const SomeClass> ptr_field1_;
+  const SomeClass* ptr_field1_;
+
+  // Expected rewrite: CheckedPtr<volatile SomeClass> ptr_field2_;
+  volatile SomeClass* ptr_field2_;
+
+  // Expected rewrite: const CheckedPtr<SomeClass> ptr_field3_;
+  //
+  // TODO(lukasza): Fix this by using |qualType.getAsString|.
+  // Currently the "outer" |const| is dropped.
+  SomeClass* const ptr_field3_;
+
+  // Expected rewrite: mutable CheckedPtr<SomeClass> ptr_field4_;
+  //
+  // TODO(lukasza): Fix this by looking at |field_decl->isMutable()|.
+  // Currently the |mutable| specifier is dropped.
+  mutable SomeClass* ptr_field4_;
+
+  // Expected rewrite: CheckedPtr<const SomeClass> ptr_field5_;
+  SomeClass const* ptr_field5_;
+
+  // Expected rewrite: volatile CheckedPtr<const SomeClass> ptr_field6_;
+  //
+  // TODO(lukasza): Fix this by using |qualType.getAsString|.
+  // Currently the "outer" qualifiers (like |volatile| below) are dropped.
+  const SomeClass* volatile ptr_field6_;
+};

tools/clang/rewrite_raw_ptr_fields/tests/typedefs-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+// Expected rewrite: typedef CheckedPtr<SomeClass> SomeClassPtrTypedef.
+// TODO(lukasza): Handle rewriting typedefs.
+typedef SomeClass* SomeClassPtrTypedef;
+
+// Expected rewrite: using SomeClassPtrTypeAlias = CheckedPtr<SomeClass>;
+// TODO(lukasza): Handle rewriting type aliases.
+using SomeClassPtrTypeAlias = SomeClass*;
+
+struct MyStruct {
+  // No rewrite expected here.
+  SomeClassPtrTypedef field1;
+  SomeClassPtrTypeAlias field2;
+
+  // Only "shallow" rewrite expected here (without unsugaring/inlining the type
+  // aliases).  So:
+  // Expected rewrite: CheckedPtr<SomeClassPtrTypedef> field3;
+  CheckedPtr<SomeClassPtrTypedef> field3;
+  // Expected rewrite: CheckedPtr<SomeClassPtrTypeAlias> field4;
+  CheckedPtr<SomeClassPtrTypeAlias> field4;
+};

tools/clang/rewrite_raw_ptr_fields/tests/typedefs-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+// Expected rewrite: typedef CheckedPtr<SomeClass> SomeClassPtrTypedef.
+// TODO(lukasza): Handle rewriting typedefs.
+typedef SomeClass* SomeClassPtrTypedef;
+
+// Expected rewrite: using SomeClassPtrTypeAlias = CheckedPtr<SomeClass>;
+// TODO(lukasza): Handle rewriting type aliases.
+using SomeClassPtrTypeAlias = SomeClass*;
+
+struct MyStruct {
+  // No rewrite expected here.
+  SomeClassPtrTypedef field1;
+  SomeClassPtrTypeAlias field2;
+
+  // Only "shallow" rewrite expected here (without unsugaring/inlining the type
+  // aliases).  So:
+  // Expected rewrite: CheckedPtr<SomeClassPtrTypedef> field3;
+  SomeClassPtrTypedef* field3;
+  // Expected rewrite: CheckedPtr<SomeClassPtrTypeAlias> field4;
+  SomeClassPtrTypeAlias* field4;
+};

tools/clang/rewrite_raw_ptr_fields/tests/various-types-expected.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+struct MyStruct {
+  // Expected rewrite: CheckedPtr<CheckedPtr<SomeClass>> double_ptr;
+  // TODO(lukasza): Handle recursion/nesting.
+  CheckedPtr<SomeClass*> double_ptr;
+
+  // Expected rewrite: CheckedPtr<void> void_ptr;
+  CheckedPtr<void> void_ptr;
+
+  // No rewrite expected (non-supported-type [1]).
+  // TODO(lukasza): Skip function pointers.
+  CheckedPtr<int (void)>*func_ptr)();
+  int (MyStruct::*member_func_ptr)(char);
+
+  // No rewrite expected (non-supported-type [1]).  Even with the indirection
+  // via typedef or nesting inside another pointer type.
+  // TODO(lukasza): Skip function pointers (in presence of typedefs).
+  typedef void (*func_ptr_typedef)(char);
+  func_ptr_typedef func_ptr_typedef_field1;
+  CheckedPtr<MyStruct::func_ptr_typedef> func_ptr_typedef_field2;
+};
+
+// [1] non-supported-type - type that won't ever be either
+// A) allocated by PartitionAllc or B) derived from CheckedPtrSupport.

tools/clang/rewrite_raw_ptr_fields/tests/various-types-original.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+class SomeClass;
+
+struct MyStruct {
+  // Expected rewrite: CheckedPtr<CheckedPtr<SomeClass>> double_ptr;
+  // TODO(lukasza): Handle recursion/nesting.
+  SomeClass** double_ptr;
+
+  // Expected rewrite: CheckedPtr<void> void_ptr;
+  void* void_ptr;
+
+  // No rewrite expected (non-supported-type [1]).
+  // TODO(lukasza): Skip function pointers.
+  int (*func_ptr)();
+  int (MyStruct::*member_func_ptr)(char);
+
+  // No rewrite expected (non-supported-type [1]).  Even with the indirection
+  // via typedef or nesting inside another pointer type.
+  // TODO(lukasza): Skip function pointers (in presence of typedefs).
+  typedef void (*func_ptr_typedef)(char);
+  func_ptr_typedef func_ptr_typedef_field1;
+  func_ptr_typedef* func_ptr_typedef_field2;
+};
+
+// [1] non-supported-type - type that won't ever be either
+// A) allocated by PartitionAllc or B) derived from CheckedPtrSupport.

tools/clang/scripts/run_tool.py

@@ -357,6 +357,9 @@ def main():
     tool_path = os.path.abspath(os.path.join(
           os.path.dirname(__file__),
           '../../../third_party/llvm-build/Release+Asserts/bin'))
+  if not os.path.exists(tool_path):
+    sys.stderr.write('tool not found: %s\n' % tool_path)
+    return -1
 
   if args.all:
     # Reading source files is postponed to after possible regeneration of

tools/clang/scripts/test_tool.py

@@ -142,6 +142,8 @@ def main(argv):
   parser.add_argument('tool_name',
                       nargs=1,
                       help='Clang tool to be tested.')
+  parser.add_argument(
+      '--test-filter', default='*', help='optional glob filter for test names')
   args = parser.parse_args(argv)
   tool_to_test = args.tool_name[0]
   print('\nTesting %s\n' % tool_to_test)
@@ -151,8 +153,9 @@ def main(argv):
       tools_clang_directory, tool_to_test, 'tests')
   compile_database = os.path.join(test_directory_for_tool,
                                   'compile_commands.json')
-  source_files = glob.glob(os.path.join(test_directory_for_tool,
-                                        '*-original.cc'))
+  source_files = glob.glob(
+      os.path.join(test_directory_for_tool,
+                   '%s-original.cc' % args.test_filter))
   ext = 'cc' if args.apply_edits else 'txt'
   actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc'])
                   for source_file in source_files]