sqlite: Create an sqlite_dev target for use in developer tools

We'd like to use some SQLite features currently disabled in Chrome
in a developer trace-analyzing tool called trace_processor_shell.
As discussed on the thread [1], it's undesirable to enable them
Chrome-wide.

This CL adds an "sqlite_dev" build target and third_party/sqlite/dev
include directory with a separate sqlite library that has most of the
features enabled.

[1] https://groups.google.com/a/chromium.org/g/storage-dev/c/OkwAfpYWAa0

Bug: b/150127983, b/150125501
Change-Id: If328bb737679b6d352904bcb543bc6ed1c522b90
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2082389Reviewed-by: Victor Costan <pwnall@chromium.org>
Commit-Queue: Mikhail Khokhlov <khokhlov@google.com>
Cr-Commit-Position: refs/heads/master@{#749640}

third_party/sqlite/README.chromium

@@ -9,6 +9,7 @@ License: Public domain
 2) Making changes to Chromium SQLite.
 3) Import new release of SQLite.
 4) Running SQLite's test suite within the Chromium checkout.
+5) Two versions of SQLite library.
 
 ---
 
@@ -332,3 +333,12 @@ mkdir $TMPFS
 
 This is faster, but it is plausible that different things are being tested than
 real-world use.
+
+--------------------------------------------
+
+5) Two versions of SQLite library.
+
+There are two targets defined in the BUILD.gn file: sqlite and sqlite_full.
+scripts/generate_amalgamation.sh generates two separate source files with
+different set of build flags for them. The full version is intended for use
+in developer tools only, for all other uses please depend on sqlite target.

third_party/sqlite/dev/sqlite3.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_SQLITE_DEV_SQLITE3_H_
+#define THIRD_PARTY_SQLITE_DEV_SQLITE3_H_
+
+// This is a shim header to include the right sqlite3 headers.
+// Use this instead of referencing sqlite3 headers directly.
+
+// We prefix chrome_ to SQLite's exported symbols, so that we don't clash with
+// other SQLite libraries loaded by the system libraries. This only matters when
+// using the component build, where our SQLite's symbols are visible to the
+// dynamic library loader.
+#include "third_party/sqlite/amalgamation_dev/rename_exports.h"
+#include "third_party/sqlite/amalgamation_dev/sqlite3.h"
+
+#endif  // THIRD_PARTY_SQLITE_DEV_SQLITE3_H_

third_party/sqlite/dev/sqlite3_shim.c

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This is a shim that injects Chrome-specific defitions into sqlite3.c
+// BUILD.gn uses this instead of building the sqlite3 amalgamation directly.
+
+// We prefix chrome_ to SQLite's exported symbols, so that we don't clash with
+// other SQLite libraries loaded by the system libraries. This only matters when
+// using the component build, where our SQLite's symbols are visible to the
+// dynamic library loader.
+#include "third_party/sqlite/amalgamation_dev/rename_exports.h"
+
+#include "third_party/sqlite/sqlite3_shim_fixups.h"
+
+#include "third_party/sqlite/amalgamation_dev/sqlite3.c"

third_party/sqlite/scripts/generate_amalgamation.sh

@@ -4,16 +4,19 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# This script generates two amalgamated sources for SQLite.
+# One is for Chromium, it has some features disabled. The other one is for
+# developer tools, with a fuller set of features.
+# The build flag sets should be in sync with the defines in BUILD.gn,
+# "chromium_sqlite3_compile_options" and "common_sqlite3_compile_options"
+# configs.
+
 set -o errexit  # Stop the script on the first error.
 set -o nounset  # Catch un-initialized variables.
 
 cd patched
 
-mkdir build
-cd build
-
-# The compile flags here should match the defines in BUILD.gn.
-BUILD_GN_FLAGS="\
+BUILD_GN_FLAGS_COMMON="\
     -DSQLITE_DISABLE_FTS3_UNICODE \
     -DSQLITE_DISABLE_FTS4_DEFERRED \
     -DSQLITE_ENABLE_FTS3 \
@@ -27,47 +30,66 @@ BUILD_GN_FLAGS="\
     -DSQLITE_HAVE_ISNAN \
     -DSQLITE_MAX_WORKER_THREADS=0 \
     -DSQLITE_MAX_MMAP_SIZE=268435456 \
+    -DSQLITE_OMIT_DECLTYPE \
+    -DSQLITE_OMIT_DEPRECATED \
+    -DSQLITE_OMIT_LOAD_EXTENSION \
+    -DSQLITE_OMIT_PROGRESS_CALLBACK \
+    -DSQLITE_OMIT_SHARED_CACHE \
+    -DSQLITE_SECURE_DELETE \
+    -DSQLITE_THREADSAFE=1 \
+    -DSQLITE_USE_ALLOCA \
+"
+
+BUILD_GN_FLAGS_CHROMIUM="\
     -DSQLITE_OMIT_ANALYZE \
     -DSQLITE_OMIT_AUTOINIT \
     -DSQLITE_OMIT_AUTORESET \
     -DSQLITE_OMIT_COMPILEOPTION_DIAGS \
     -DSQLITE_OMIT_COMPLETE \
-    -DSQLITE_OMIT_DECLTYPE \
-    -DSQLITE_OMIT_DEPRECATED \
     -DSQLITE_OMIT_EXPLAIN \
     -DSQLITE_OMIT_GET_TABLE \
-    -DSQLITE_OMIT_LOAD_EXTENSION \
     -DSQLITE_OMIT_LOOKASIDE \
     -DSQLITE_OMIT_TCL_VARIABLE \
-    -DSQLITE_OMIT_PROGRESS_CALLBACK \
     -DSQLITE_OMIT_REINDEX \
-    -DSQLITE_OMIT_SHARED_CACHE \
     -DSQLITE_OMIT_TRACE \
     -DSQLITE_OMIT_UPSERT \
     -DSQLITE_OMIT_WINDOWFUNC \
-    -DSQLITE_SECURE_DELETE \
-    -DSQLITE_THREADSAFE=1 \
-    -DSQLITE_USE_ALLOCA \
 "
 
-../configure \
-    CFLAGS="-Os $BUILD_GN_FLAGS $(icu-config --cppflags)" \
-    LDFLAGS="$(icu-config --ldflags)" \
-    --disable-load-extension \
-    --enable-amalgamation \
-    --enable-threadsafe
+BUILD_GN_FLAGS_DEV="\
+    -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+"
+
+generate_amalgamation() {
+  local build_flags=$1
+  local amalgamation_path=$2
+
+  mkdir build
+  cd build
+
+  ../configure \
+      CFLAGS="-Os $build_flags $(icu-config --cppflags)" \
+      LDFLAGS="$(icu-config --ldflags)" \
+      --disable-load-extension \
+      --enable-amalgamation \
+      --enable-threadsafe
+
+  make shell.c sqlite3.h sqlite3.c
+  cp -f sqlite3.h sqlite3.c ../$amalgamation_path/
+
+  # shell.c must be placed in a different directory from sqlite3.h, because it
+  # contains an '#include "sqlite3.h"' that we want to resolve to our custom
+  # //third_party/sqlite/sqlite3.h, not to the sqlite3.h produced here.
+  mkdir -p ../$amalgamation_path/shell/
+  cp -f shell.c ../$amalgamation_path/shell/
 
-make shell.c sqlite3.h sqlite3.c
-cp -f sqlite3.h sqlite3.c ../../amalgamation
+  cd ..
+  rm -rf build
 
-# shell.c must be placed in a different directory from sqlite3.h, because it
-# contains an '#include "sqlite3.h"' that we want to resolve to our custom
-# //third_party/sqlite/sqlite3.h, not to the sqlite3.h produced here.
-mkdir -p ../../amalgamation/shell/
-cp -f shell.c ../../amalgamation/shell/
+  ../scripts/extract_sqlite_api.py $amalgamation_path/sqlite3.h \
+                                   $amalgamation_path/rename_exports.h
+}
 
-cd ..
-rm -rf build
+generate_amalgamation "$BUILD_GN_FLAGS_COMMON $BUILD_GN_FLAGS_CHROMIUM" ../amalgamation
+generate_amalgamation "$BUILD_GN_FLAGS_COMMON $BUILD_GN_FLAGS_DEV" ../amalgamation_dev
 
-../scripts/extract_sqlite_api.py ../amalgamation/sqlite3.h \
-                                 ../amalgamation/rename_exports.h

third_party/sqlite/sqlite3_shim.c

@@ -11,60 +11,6 @@
 // dynamic library loader.
 #include "third_party/sqlite/amalgamation/rename_exports.h"
 
-#if defined(SQLITE_OMIT_COMPILEOPTION_DIAGS)
-// When SQLITE_OMIT_COMPILEOPTION_DIAGS is defined, sqlite3.h emits macros
-// instead of declarations for sqlite3_compileoption_{get,used}().
-//
-// In order to avoid a macro redefinition warning, we must undo the #define in
-// rename_exports.h.
-#if defined(sqlite3_compileoption_get)
-#undef sqlite3_compileoption_get
-#else
-#error "This workaround is no longer needed."
-#endif  // !defined(sqlite3_compileoption_get)
-#if defined(sqlite3_compileoption_used)
-#undef sqlite3_compileoption_used
-#else
-#error "This workaround is no longer needed."
-#endif  // !defined(sqlite3_compileoption_used)
-#endif  // defined(SQLITE_OMIT_COMPILEOPTION_DIAGS)
-
-// Linux-specific configuration fixups.
-#if defined(__linux__)
-
-// features.h, included below, indirectly includes sys/mman.h. The latter header
-// only defines mremap if _GNU_SOURCE is defined. Depending on the order of the
-// files in the amalgamation, removing the define below may result in a build
-// error on Linux.
-#if defined(__GNUC__) && !defined(_GNU_SOURCE)
-#define _GNU_SOURCE
-#endif
-#include <features.h>
-
-// SQLite wants to track malloc sizes. On OSX it uses malloc_size(), on Windows
-// _msize(), elsewhere it handles it manually by enlarging the malloc and
-// injecting a field. Enable malloc_usable_size() for Linux.
-//
-// malloc_usable_size() is not exported by the Android NDK. It is not
-// implemented by uclibc.
-#if !defined(__UCLIBC__) && !defined(__ANDROID__)
-#define HAVE_MALLOC_H 1
-#define HAVE_MALLOC_USABLE_SIZE 1
-#endif
-
-#endif  // defined(__linux__)
-
-// For unfortunately complex reasons, Chrome has release builds where
-// DCHECK_IS_ON() (so we want SQLITE_DEBUG to be on) but NDEBUG is also defined.
-// This causes declarations for mutex-checking functions used by SQLITE_DEBUG
-// code (sqlite3_mutex_held, sqlite3_mutex_notheld) to be omitted, resulting in
-// warnings.
-//
-// The easiest solution for now is to undefine NDEBUG when SQLITE_DEBUG is
-// defined. The #undef only takes effect for the SQLite implementation (included
-// below), and does not impact any dependency.
-#if defined(SQLITE_DEBUG) && defined(NDEBUG)
-#undef NDEBUG
-#endif  // defined(SQLITE_DEBUG) && defined(NDEBUG)
+#include "third_party/sqlite/sqlite3_shim_fixups.h"
 
 #include "third_party/sqlite/amalgamation/sqlite3.c"

third_party/sqlite/sqlite3_shim_fixups.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_SQLITE_SQLITE3_SHIM_FIXUPS_H_
+#define THIRD_PARTY_SQLITE_SQLITE3_SHIM_FIXUPS_H_
+
+// This file contains various fixups for the amalgamated SQLite code.
+// It is intended to be included in sqlite3_shim.c only.
+
+#if defined(SQLITE_OMIT_COMPILEOPTION_DIAGS)
+// When SQLITE_OMIT_COMPILEOPTION_DIAGS is defined, sqlite3.h emits macros
+// instead of declarations for sqlite3_compileoption_{get,used}().
+//
+// In order to avoid a macro redefinition warning, we must undo the #define in
+// rename_exports.h.
+#if defined(sqlite3_compileoption_get)
+#undef sqlite3_compileoption_get
+#else
+#error "This workaround is no longer needed."
+#endif  // !defined(sqlite3_compileoption_get)
+#if defined(sqlite3_compileoption_used)
+#undef sqlite3_compileoption_used
+#else
+#error "This workaround is no longer needed."
+#endif  // !defined(sqlite3_compileoption_used)
+#endif  // defined(SQLITE_OMIT_COMPILEOPTION_DIAGS)
+
+// Linux-specific configuration fixups.
+#if defined(__linux__)
+
+// features.h, included below, indirectly includes sys/mman.h. The latter header
+// only defines mremap if _GNU_SOURCE is defined. Depending on the order of the
+// files in the amalgamation, removing the define below may result in a build
+// error on Linux.
+#if defined(__GNUC__) && !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif
+#include <features.h>
+
+// SQLite wants to track malloc sizes. On OSX it uses malloc_size(), on Windows
+// _msize(), elsewhere it handles it manually by enlarging the malloc and
+// injecting a field. Enable malloc_usable_size() for Linux.
+//
+// malloc_usable_size() is not exported by the Android NDK. It is not
+// implemented by uclibc.
+#if !defined(__UCLIBC__) && !defined(__ANDROID__)
+#define HAVE_MALLOC_H 1
+#define HAVE_MALLOC_USABLE_SIZE 1
+#endif
+
+#endif  // defined(__linux__)
+
+// For unfortunately complex reasons, Chrome has release builds where
+// DCHECK_IS_ON() (so we want SQLITE_DEBUG to be on) but NDEBUG is also defined.
+// This causes declarations for mutex-checking functions used by SQLITE_DEBUG
+// code (sqlite3_mutex_held, sqlite3_mutex_notheld) to be omitted, resulting in
+// warnings.
+//
+// The easiest solution for now is to undefine NDEBUG when SQLITE_DEBUG is
+// defined. The #undef only takes effect for the SQLite implementation (included
+// below), and does not impact any dependency.
+#if defined(SQLITE_DEBUG) && defined(NDEBUG)
+#undef NDEBUG
+#endif  // defined(SQLITE_DEBUG) && defined(NDEBUG)
+
+#endif  // THIRD_PARTY_SQLITE_SQLITE3_SHIM_FIXUPS_H_