Use chain-loading for Linux nacl_helper

This replaces the nacl_helper_bootstrap program, dynamically-linked against
nacl_helper.so, with a standalone, statically-linked nacl_helper_bootstrap
program that loads the dynamic linker, instructing it in turn to load the
nacl_helper program (now a PIE rather than a DSO).

This avoids two problems with the old scheme:
1. The nacl_helper_bootstrap program remained in the dynamic linker's
   list of loaded objects, as the main executable, even though the
   memory where its .dynamic section had been was overwritten with
   the NaCl untrusted address space.  Code that traverses the list of
   all loaded objects could thus attempt to look at pointers into this
   part of memory, and be led astray.
2. nacl_helper_bootstrap's large (~1G) bss segment could cause the kernel
   to refuse to load the program because it didn't think there was enough
   free memory in the system for so large an allocation of anonymous memory.

The bootstrap program is kept very small by avoiding all use of libc
(except for memset and integer division routines needed on ARM).  It has
its own custom start-up code hand-written in assembly and its own custom
system call stubs done with hand-written GCC inline asm statements.

To avoid the second problem, the bootstrap program no longer has a large
bss.  Instead, it has a special ELF segment (i.e. PT_LOAD header) that
specifies no memory access, and a large (~1G) mapping size from the file.
This mapping is way off the end of the file, but the kernel doesn't mind
that, and since it's all a file mapping, the kernel does not do its normal
memory accounting for consuming a large amount of anonymous memory.

Unfortunately, it's impossible to get the linker to produce exactly the
right PT_LOAD header by itself.  Using a custom linker script, we get the
layout exactly how we want it and a PT_LOAD header that is almost right.
We then use a build-time helper program to munge one field of the PT_LOAD
to make it exactly what we need.

BUG= http://code.google.com/p/chromium/issues/detail?id=94147
TEST= hand-tested chromium build, invoked with --nacl-linux-helper

R=bradchen@google.com,mseaborn@chromium.org


Review URL: http://codereview.chromium.org/7795010

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@98909 0039d316-1c4b-4281-b951-d872f2087c98

build/install-build-deps.sh

@@ -103,7 +103,7 @@ chromeos_dev_list="libpulse-dev"
 # Packages need for development
 dev_list="apache2.2-bin bison fakeroot flex g++ gperf language-pack-fr
           libapache2-mod-php5 libasound2-dev libbz2-dev libcairo2-dev
-          libcups2-dev libdbus-glib-1-dev libgconf2-dev
+          libcups2-dev libdbus-glib-1-dev libelf-dev libgconf2-dev
           libgl1-mesa-dev libglu1-mesa-dev libglib2.0-dev libgnome-keyring-dev
           libgtk2.0-dev libjpeg62-dev libkrb5-dev libnspr4-dev libnss3-dev
           libpam0g-dev libsctp-dev libsqlite3-dev libxslt1-dev libxss-dev

chrome/common/chrome_paths.cc

@@ -53,8 +53,10 @@ const FilePath::CharType kInternalNaClPluginFileName[] =
 #endif
 
 #if defined(OS_POSIX) && !defined(OS_MACOSX)
-// File name of the nacl_helper, Linux only.
+// File name of the nacl_helper and nacl_helper_bootstrap, Linux only.
 const FilePath::CharType kInternalNaClHelperFileName[] =
+    FILE_PATH_LITERAL("nacl_helper");
+const FilePath::CharType kInternalNaClHelperBootstrapFileName[] =
     FILE_PATH_LITERAL("nacl_helper_bootstrap");
 #endif
 
@@ -246,6 +248,11 @@ bool PathProvider(int key, FilePath* result) {
         return false;
       cur = cur.Append(kInternalNaClHelperFileName);
       break;
+    case chrome::FILE_NACL_HELPER_BOOTSTRAP:
+      if (!PathService::Get(base::DIR_MODULE, &cur))
+        return false;
+      cur = cur.Append(kInternalNaClHelperBootstrapFileName);
+      break;
 #endif
     case chrome::FILE_RESOURCES_PACK:
 #if defined(OS_MACOSX)

chrome/common/chrome_paths.h

@@ -71,6 +71,7 @@ enum {
 
 #if defined(OS_POSIX) && !defined(OS_MACOSX)
   FILE_NACL_HELPER,             // Full path to Linux nacl_helper executable.
+  FILE_NACL_HELPER_BOOTSTRAP,   // ... and nacl_helper_bootstrap executable.
 #endif
   FILE_NACL_PLUGIN,             // Full path to the internal NaCl plugin file.
   FILE_LIBAVCODEC,              // Full path to libavcodec media decoding

chrome/nacl.gypi

@@ -183,9 +183,7 @@
     ['OS=="linux"', {
       'targets': [
         {
-          'target_name': 'nacl_helper.so',
-          # 'executable' will be overridden below when we add the -shared
-          # flag; here it prevents gyp from using the --whole-archive flag
+          'target_name': 'nacl_helper',
           'type': 'executable',
           'include_dirs': [
             '..',
@@ -194,7 +192,7 @@
             'nacl',
           ],
           'sources': [
-            '../chrome/nacl/nacl_helper_linux.cc',
+            'nacl/nacl_helper_linux.cc',
           ],
           'conditions': [
             ['toolkit_uses_gtk == 1', {
@@ -203,36 +201,108 @@
               ],
             }],
           ],
+          'cflags': ['-fPIE'],
           'link_settings': {
-            # NOTE: '-shared' overrides 'executable' above
-            'ldflags': ['-shared',
-                        '-Wl,--version-script=chrome/nacl/nacl_helper_exports.txt',
-                      ],
+            'ldflags': ['-pie'],
           },
         },
         {
-          'target_name': 'nacl_helper_bootstrap',
+          'target_name': 'nacl_helper_bootstrap_munge_phdr',
           'type': 'executable',
-          'dependencies': [
-            'nacl_helper.so',
+          'toolsets': ['host'],
+          'sources': [
+            'nacl/nacl_helper_bootstrap_munge_phdr.c',
+          ],
+          'libraries': [
+            '-lelf',
+          ],
+          # This is an ugly kludge because gyp doesn't actually treat
+          # host_arch=x64 target_arch=ia32 as proper cross compilation.
+          # It still wants to compile the "host" program with -m32 et
+          # al.  Though a program built that way can indeed run on the
+          # x86-64 host, we cannot reliably build this program on such a
+          # host because Ubuntu does not provide the full suite of
+          # x86-32 libraries in packages that can be installed on an
+          # x86-64 host; in particular, libelf is missing.  So here we
+          # use the hack of eliding all the -m* flags from the
+          # compilation lines, getting the command close to what they
+          # would be if gyp were to really build properly for the host.
+          # TODO(bradnelson): Clean up with proper cross support.
+          'conditions': [
+            ['host_arch=="x64"', {
+              'cflags/': [['exclude', '-m.*']],
+              'ldflags/': [['exclude', '-m.*']],
+            }],
+          ],
+        },
+        {
+          'target_name': 'nacl_helper_bootstrap_raw',
+          'type': 'executable',
+          'include_dirs': [
+            '..',
           ],
           'sources': [
-            '../chrome/nacl/nacl_helper_bootstrap_linux.c',
+            'nacl/nacl_helper_bootstrap_linux.c',
+            # We list the linker script here for documentation purposes.
+            # But even this doesn't make gyp treat it as a dependency,
+            # so incremental builds won't relink when the script changes.
+            # TODO(bradnelson): Fix the dependency handling.
+            'nacl/nacl_helper_bootstrap_linux.x',
+          ],
+          'cflags': [
+            # The tiny standalone bootstrap program is incompatible with
+            # -fstack-protector, which might be on by default.  That switch
+            # requires using the standard libc startup code, which we do not.
+            '-fno-stack-protector',
+            # We don't want to compile it PIC (or its cousin PIE), because
+            # it goes at an absolute address anyway, and because any kind
+            # of PIC complicates life for the x86-32 assembly code.  We
+            # append -fno-* flags here instead of using a 'cflags!' stanza
+            # to remove -f* flags, just in case some system's compiler
+            # defaults to using PIC for everything.
+            '-fno-pic', '-fno-PIC',
+            '-fno-pie', '-fno-PIE',
           ],
-          # TODO(bradchen): Delete the -B argument when Gold supports
-          # -Ttext properly. Until then use ld.bfd.
           'link_settings': {
-            'ldflags': ['-B', 'tools/ld_bfd',
-                        # Force text segment at 0x10000 (64KB)
-                        # The max-page-size option is needed on x86-64 linux
-                        # where 4K pages are not the default in the BFD linker.
-                        '-Wl,-Ttext-segment,10000,-z,max-page-size=0x1000',
-                        # reference nacl_helper as a shared library
-                        '<(PRODUCT_DIR)/nacl_helper.so',
-                        '-Wl,-rpath,<(SHARED_LIB_DIR)',
-                      ],
+            'ldflags': [
+              # TODO(bradchen): Delete the -B argument when Gold is verified
+              # to produce good results with our custom linker script.
+              # Until then use ld.bfd.
+              '-B', 'tools/ld_bfd',
+              # This programs is (almost) entirely standalone.  It has
+              # its own startup code, so no crt1.o for it.  It is
+              # statically linked, and on x86 it actually does not use
+              # libc at all.  However, on ARM it needs a few (safe)
+              # things from libc, so we don't use '-nostdlib' here.
+              '-static', '-nostartfiles',
+              # Link with our custom linker script to get out special layout.
+              # TODO(bradnelson): Use some <(foo) instead of chrome/ here.
+              '-Wl,--script=chrome/nacl/nacl_helper_bootstrap_linux.x',
+              # On x86-64, the default page size with some
+              # linkers is 2M rather than the real Linux page
+              # size of 4K.  A larger page size is incompatible
+              # with our custom linker script's special layout.
+              '-Wl,-z,max-page-size=0x1000',
+            ],
           },
         },
+        {
+          'target_name': 'nacl_helper_bootstrap',
+          'dependencies': [
+            'nacl_helper_bootstrap_raw',
+            'nacl_helper_bootstrap_munge_phdr#host',
+          ],
+          'type': 'none',
+          'actions': [{
+              'action_name': 'munge_phdr',
+              'inputs': ['nacl/nacl_helper_bootstrap_munge_phdr.py',
+                         '<(PRODUCT_DIR)/nacl_helper_bootstrap_munge_phdr',
+                         '<(PRODUCT_DIR)/nacl_helper_bootstrap_raw'],
+              'outputs': ['<(PRODUCT_DIR)/nacl_helper_bootstrap'],
+              'message': 'Munging ELF program header',
+              'action': ['python', '<@(_inputs)', '<@(_outputs)']
+            }],
+          }
       ],
     }],
   ],

chrome/nacl/nacl_fork_delegate_linux.cc

@@ -26,6 +26,8 @@ NaClForkDelegate::NaClForkDelegate()
       sandboxed_(false),
       fd_(-1) {}
 
+const char kNaClHelperAtZero[] = "--at-zero";
+
 void NaClForkDelegate::Init(const bool sandboxed,
                             const int browserdesc,
                             const int sandboxdesc) {
@@ -48,18 +50,18 @@ void NaClForkDelegate::Init(const bool sandboxed,
   const bool use_helper = CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kNaClLinuxHelper);
   FilePath helper_exe;
-  if (use_helper && PathService::Get(chrome::FILE_NACL_HELPER, &helper_exe)) {
+  FilePath helper_bootstrap_exe;
+  if (use_helper &&
+      PathService::Get(chrome::FILE_NACL_HELPER, &helper_exe) &&
+      PathService::Get(chrome::FILE_NACL_HELPER_BOOTSTRAP,
+                       &helper_bootstrap_exe)) {
     CommandLine::StringVector argv = CommandLine::ForCurrentProcess()->argv();
-    argv[0] = helper_exe.value();
+    argv[0] = helper_bootstrap_exe.value();
+    argv[1] = helper_exe.value();
+    argv[2] = kNaClHelperAtZero;
     base::LaunchOptions options;
     options.fds_to_remap = &fds_to_map;
     options.clone_flags = CLONE_FS | SIGCHLD;
-    // LD_BIND_NOW forces non-lazy binding in the dynamic linker, to
-    // prevent the linker from trying to look at the text of the nacl_helper
-    // program after it has been replaced by the nacl module.
-    base::environment_vector env;
-    env.push_back(std::make_pair("LD_BIND_NOW", "1"));
-    options.environ = &env;
     ready_ = base::LaunchProcess(argv, options, NULL);
     // parent and error cases are handled below
   }

chrome/nacl/nacl_helper_bootstrap_linux.x

+/* Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ *
+ * This is a custom linker script used to build nacl_helper_bootstrap.
+ * It has a very special layout.  This script will only work with input
+ * that is kept extremely minimal.  If there are unexpected input sections
+ * not named here, the result will not be correct.
+ *
+ * We need to use a standalone loader program rather than just using a
+ * dynamically-linked program here because its entire address space will be
+ * taken over for the NaCl untrusted address space.  A normal program would
+ * cause dynamic linker data structures to point to its .dynamic section,
+ * which is no longer available after startup.
+ *
+ * We need this special layout (and the nacl_helper_bootstrap_munge_phdr
+ * step) because simply having bss space large enough to reserve the
+ * address space would cause the kernel loader to think we're using that
+ * much anonymous memory and refuse to execute the program on a machine
+ * with not much memory available.
+ */
+
+/*
+ * Set the entry point to the symbol called _start, which we define in assembly.
+ */
+ENTRY(_start)
+
+/*
+ * This is the address where the program text starts.
+ * We set this as low as we think we can get away with.
+ * The common settings for sysctl vm.mmap_min_addr range from 4k to 64k.
+ */
+TEXT_START = 0x10000;
+
+/*
+ * This is the top of the range we are trying to reserve, which is 1G
+ * for x86-32 and ARM.  For an x86-64 zero-based sandbox, this really
+ * needs to be 36G.
+ */
+RESERVE_TOP = 1 << 30;
+
+/*
+ * We specify the program headers we want explicitly, to get the layout
+ * exactly right and to give the "reserve" segment p_flags of zero, so
+ * that it gets mapped as PROT_NONE.
+ */
+PHDRS {
+  text PT_LOAD FILEHDR PHDRS;
+  reserve PT_LOAD FLAGS(0);
+  stack PT_GNU_STACK FLAGS(6);	/* RW, no E */
+}
+
+/*
+ * Now we lay out the sections across those segments.
+ */
+SECTIONS {
+  /*
+   * Here is the program itself.
+   */
+  .text TEXT_START + SIZEOF_HEADERS : {
+    *(.note.gnu.build-id)
+    *(.text*)
+    *(.rodata*)
+    *(.eh_frame*)
+  } :text
+  etext = .;
+
+  /*
+   * Now we move up to the next p_align increment, and place the dummy
+   * segment there.  The linker emits this segment with the p_vaddr and
+   * p_memsz we want, which reserves the address space.  But the linker
+   * gives it a p_filesz of zero.  We have to edit the phdr after link
+   * time to give it a p_filesz matching its p_memsz.  That way, the
+   * kernel doesn't think we are preallocating a huge amount of memory.
+   * It just maps it from the file, i.e. way off the end of the file,
+   * which is perfect for reserving the address space.
+   */
+  . = ALIGN(CONSTANT(COMMONPAGESIZE));
+  RESERVE_START = .;
+  .reserve : {
+    . = RESERVE_TOP - RESERVE_START;
+  } :reserve
+
+  /*
+   * These are empty input sections the linker generates.
+   * If we don't discard them, they pollute the flags in the output segment.
+   */
+  /DISCARD/ : {
+    *(.iplt)
+    *(.rel*)
+    *(.igot.plt)
+  }
+}

chrome/nacl/nacl_helper_bootstrap_munge_phdr.c

+/* Copyright (c) 2011 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ *
+ * This is a trivial program to edit an ELF file in place, making
+ * one crucial modification to a program header.  It's invoked:
+ *      bootstrap_phdr_hacker FILENAME SEGMENT_NUMBER
+ * where SEGMENT_NUMBER is the zero-origin index of the program header
+ * we'll touch.  This is a PT_LOAD with p_filesz of zero.  We change its
+ * p_filesz to match its p_memsz value.
+ */
+
+#include <errno.h>
+#include <error.h>
+#include <fcntl.h>
+#include <gelf.h>
+#include <libelf.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+int main(int argc, char **argv) {
+  if (argc != 3)
+    error(1, 0, "Usage: %s FILENAME SEGMENT_NUMBER", argv[0]);
+
+  const char *const file = argv[1];
+  const int segment = atoi(argv[2]);
+
+  int fd = open(file, O_RDWR);
+  if (fd < 0)
+    error(2, errno, "Cannot open %s for read/write", file);
+
+  if (elf_version(EV_CURRENT) == EV_NONE)
+    error(2, 0, "elf_version: %s", elf_errmsg(-1));
+
+  Elf *elf = elf_begin(fd, ELF_C_RDWR, NULL);
+  if (elf == NULL)
+    error(2, 0, "elf_begin: %s", elf_errmsg(-1));
+
+  if (elf_flagelf(elf, ELF_C_SET, ELF_F_LAYOUT) == 0)
+    error(2, 0, "elf_flagelf: %s", elf_errmsg(-1));
+
+  GElf_Phdr phdr;
+  GElf_Phdr *ph = gelf_getphdr(elf, segment, &phdr);
+  if (ph == NULL)
+    error(2, 0, "gelf_getphdr: %s", elf_errmsg(-1));
+
+  if (ph->p_type != PT_LOAD)
+    error(3, 0, "Program header %d is %u, not PT_LOAD",
+          segment, (unsigned int) ph->p_type);
+  if (ph->p_filesz != 0)
+    error(3, 0, "Program header %d has nonzero p_filesz", segment);
+
+  ph->p_filesz = ph->p_memsz;
+  if (gelf_update_phdr(elf, segment, ph) == 0)
+    error(2, 0, "gelf_update_phdr: %s", elf_errmsg(-1));
+
+  if (elf_flagphdr(elf, ELF_C_SET, ELF_F_DIRTY) == 0)
+    error(2, 0, "elf_flagphdr: %s", elf_errmsg(-1));
+
+  if (elf_update(elf, ELF_C_WRITE) < 0)
+    error(2, 0, "elf_update: %s", elf_errmsg(-1));
+
+  close(fd);
+
+  return 0;
+}

chrome/nacl/nacl_helper_bootstrap_munge_phdr.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# This takes three command-line arguments:
+#       MUNGE-PHDR-PROGRAM      file name of program built from
+#                               nacl_helper_bootstrap_munge_phdr.c
+#       INFILE                  raw linked ELF file name
+#       OUTFILE                 output file name
+#
+# We just run the MUNGE-PHDR-PROGRAM on a copy of INFILE.
+# That modifies the file in place.  Then we move it to OUTFILE.
+#
+# We only have this wrapper script because nacl_helper_bootstrap_munge_phdr.c
+# wants to modify a file in place (and it would be a much longer and more
+# fragile program if it created a fresh ELF output file instead).
+
+import shutil
+import subprocess
+import sys
+
+
+def Main(argv):
+  if len(argv) != 4:
+    print 'Usage: %s MUNGE-PHDR-PROGRAM INFILE OUTFILE' % argv[0]
+    sys.exit(1)
+  [prog, munger, infile, outfile] = argv
+  tmpfile = outfile + '.tmp'
+  shutil.copy(infile, tmpfile)
+  segment_num = '1'
+  subprocess.check_call([munger, tmpfile, segment_num])
+  shutil.move(tmpfile, outfile)
+
+if __name__ == '__main__':
+  Main(sys.argv)

chrome/nacl/nacl_helper_exports.txt

-# gnu-ld version script for exporting desired symbols from nacl_helper
-#
-
-NACL_HELPER_1_0 {
-	global:
-		nacl_helper_init;
-		nacl_helper_get_1G_address;
-	local:
-		*;
-};

chrome/nacl/nacl_helper_linux.cc

@@ -23,6 +23,7 @@
 #include "content/common/main_function_params.h"
 #include "content/common/unix_domain_socket_posix.h"
 #include "ipc/ipc_switches.h"
+#include "native_client/src/trusted/service_runtime/sel_memory.h"
 
 namespace {
 
@@ -116,33 +117,20 @@ void HandleForkRequest(const std::vector<int>& child_fds) {
 
 }  // namespace
 
-static const void* g_nacl_reserved_space = NULL;
-extern "C" __attribute__((visibility("default")))
-const void* nacl_helper_get_1G_address() {
-  return g_nacl_reserved_space;
-}
+static const char kNaClHelperAtZero[] = "at-zero";
 
-// nacl_helper_init does the real work of this module. It is invoked as
-// a static constructor and never returns, preventing main() from the
-// nacl_helper_bootstrap program from being called.
-//
-// NOTE This routine must not return.
-extern "C" __attribute__((visibility("default")))
-void nacl_helper_init(int argc, char *argv[],
-                      const char *nacl_reserved_space) {
+int main(int argc, char *argv[]) {
   CommandLine::Init(argc, argv);
   base::AtExitManager exit_manager;
   base::RandUint64();  // acquire /dev/urandom fd before sandbox is raised
   std::vector<int> empty; // for SendMsg() calls
 
   g_suid_sandbox_active = (NULL != getenv("SBX_D"));
-  g_nacl_reserved_space = nacl_reserved_space;
-  if (!nacl_reserved_space) {
-    VLOG(1) << "nacl_reserved_space is NULL";
-  } else {
-    VLOG(1) << "nacl_reserved_space is at "
-            << (void *)nacl_reserved_space;
+
+  if (CommandLine::ForCurrentProcess()->HasSwitch(kNaClHelperAtZero)) {
+    g_nacl_prereserved_sandbox_addr = (void *) (uintptr_t) 0x10000;
   }
+
   // Send the zygote a message to let it know we are ready to help
   if (!UnixDomainSocket::SendMsg(kNaClZygoteDescriptor,
                                  kNaClHelperStartupAck,