Implement elf_headers library for ELF file parsing and modification.

None of the existing python packages in third_party/ can modify ELF
structure and as such not fit for the script. This library allows the
script to parse the library and read useful information from it.

Bug: 998082
Change-Id: I7d3197b93cf1a3b9c1797249a77d7ed8f130b519
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1777928
Commit-Queue: Denis Solonkov <solonkovda@google.com>
Reviewed-by: Alex Ilin <alexilin@chromium.org>
Reviewed-by: Boris Sazonov <bsazonov@chromium.org>
Cr-Commit-Position: refs/heads/master@{#695089}

tools/android/elf_compression/README.md

@@ -37,6 +37,5 @@ unusual (but legal) changes made to the library.
 ## Testing
 To run tests:
 
-    test/compression_script_test.py
+    test/run_tests.py
 
-The test simply verifies that the sample library is not broken after applying the script.

tools/android/elf_compression/elf_headers.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""This library contains 64 bit ELF headers access and modification methods.
+
+This library implements classes representing various ELF structures.
+For more detailed description of ELF headers and fields refer to the ELF
+standard: http://www.skyfree.org/linux/references/ELF_Format.pdf and to the
+64 bit update: https://uclibc.org/docs/elf-64-gen.pdf.
+
+This library was created because the script required precise manipulations of
+file offsets. I.e: move the headers segment to the end of the file.
+No package, capable of doing this kind of manipulations was found, so the
+creation of this library was deemed necessary.
+
+The point of entry is ElfHeader class that provides methods for accessing
+additional parts of the ELF file.
+"""
+
+import enum
+import logging
+
+
+class ElfEntry(object):
+  """Base class for ELF headers.
+
+  Provides methods for populating fields.
+  """
+
+  def __init__(self, byte_order, fields=None):
+    """ElfEntry constructor.
+
+    Args:
+      byte_order: str. Either 'little' for little endian or 'big' for big
+        endian.
+      fields: List[Tuple[str, int]]. An ordered list of pairs of
+        (attribute name, size in bytes). This list will be used for parsing
+        data and automatically setting up those fields.
+    """
+    if fields is None:
+      self._fields = []
+    else:
+      self._fields = fields
+    self.byte_order = byte_order
+
+  def ParseBytes(self, data, offset):
+    """Parses Entry fields from data starting at offset using _fields.
+
+    Args:
+        data: bytes.
+        offset: int. The start point of parsing.
+    """
+    current_offset = offset
+    for field_name, field_size in self._fields:
+      value = int.from_bytes(
+          data[current_offset:current_offset + field_size],
+          byteorder=self.byte_order)
+      setattr(self, field_name, value)
+      current_offset += field_size
+
+  def ApplyKwargs(self, **kwargs):
+    """Set the fields from kwargs matching the _fields array entries."""
+    for field_name, _ in self._fields:
+      if field_name not in kwargs:
+        logging.error('field_name %s not found in kwargs', field_name)
+        continue
+      setattr(self, field_name, kwargs[field_name])
+
+  @classmethod
+  def FromBytes(cls, byte_order, data, offset):
+    """Static wrapper around ParseBytes method.
+
+    Args:
+      byte_order: str. Either 'little' for little endian or 'big' for big
+        endian.
+      data: bytes.
+      offset: int. The start point of parsing.
+    """
+    obj = cls(byte_order)
+    obj.ParseBytes(data, offset)
+    return obj
+
+
+class ProgramHeader(ElfEntry):
+  """This class represent PhdrEntry from ELF standard."""
+
+  class Type(enum.IntEnum):
+    PT_NULL = 0
+    PT_LOAD = 1
+    PT_DYNAMIC = 2
+    PT_INTERP = 3
+    PT_NOTE = 4
+    PT_SHLIB = 5
+    PT_PHDR = 6
+
+  def __init__(self, byte_order):
+    """ProgramHeader constructor.
+
+    Args:
+      byte_order: str.
+    """
+    # We have to set them here to avoid attribute-error from pylint.
+    self.p_type = None
+    self.p_flags = None
+    self.p_offset = None
+    self.p_vaddr = None
+    self.p_paddr = None
+    self.p_filesz = None
+    self.p_memsz = None
+    self.p_align = None
+    fields = [
+        ('p_type', 4),
+        ('p_flags', 4),
+        ('p_offset', 8),
+        ('p_vaddr', 8),
+        ('p_paddr', 8),
+        ('p_filesz', 8),
+        ('p_memsz', 8),
+        ('p_align', 8),
+    ]
+    super(ProgramHeader, self).__init__(byte_order, fields)
+
+
+class ElfHeader(ElfEntry):
+  """This class represents ELFHdr from the ELF standard.
+
+  On its initialization it determines the bitness and endianness of the binary.
+  """
+
+  class EiClass(enum.IntEnum):
+    ELFCLASS32 = 1
+    ELFCLASS64 = 2
+
+  class EiData(enum.IntEnum):
+    ELFDATALSB = 1
+    ELFDATAMSB = 2
+
+  class EType(enum.IntEnum):
+    ET_NONE = 0
+    ET_REL = 1
+    ET_EXEC = 2
+    ET_DYN = 3
+    ET_CORE = 4
+
+  _EI_CLASS_OFFSET = 4
+  _EI_DATA_OFFSET = 5
+
+  def _GetEiClass(self, data):
+    """Returns the value of ei_class."""
+    return data[self._EI_CLASS_OFFSET]
+
+  def _GetEiData(self, data):
+    """Returns the value of ei_data."""
+    return data[self._EI_DATA_OFFSET]
+
+  def _ValidateBitness(self, data):
+    """Verifies that library supports file's bitness."""
+    if self._GetEiClass(data) != ElfHeader.EiClass.ELFCLASS64:
+      raise RuntimeError('only 64 bit objects are supported')
+
+  def _ReadByteOrder(self, data):
+    """Reads and returns the file's byte order."""
+    ei_data = data[self._EI_DATA_OFFSET]
+    if ei_data == ElfHeader.EiData.ELFDATALSB:
+      return 'little'
+    elif ei_data == ElfHeader.EiData.ELFDATAMSB:
+      return 'big'
+    raise RuntimeError('Failed to parse ei_data')
+
+  def _ParsePhdrs(self, data):
+    current_offset = self.e_phoff
+    for _ in range(0, self.e_phnum):
+      self.phdrs.append(
+          ProgramHeader.FromBytes(self.byte_order, data, current_offset))
+      current_offset += self.e_phentsize
+
+  def __init__(self, data):
+    """ElfHeader constructor.
+
+    Args:
+      data: bytearray.
+    """
+    # We have to set them here to avoid attribute-error from pylint.
+    self.ei_magic = None
+    self.ei_class = None
+    self.ei_data = None
+    self.ei_version = None
+    self.ei_osabi = None
+    self.ei_abiversion = None
+    self.ei_pad = None
+    self.e_type = None
+    self.e_machine = None
+    self.e_version = None
+    self.e_entry = None
+    self.e_phoff = None
+    self.e_shoff = None
+    self.e_flags = None
+    self.e_ehsize = None
+    self.e_phentsize = None
+    self.e_phnum = None
+    self.e_shentsize = None
+    self.e_shnum = None
+    self.e_shstrndx = None
+    fields = [
+        ('ei_magic', 4),
+        ('ei_class', 1),
+        ('ei_data', 1),
+        ('ei_version', 1),
+        ('ei_osabi', 1),
+        ('ei_abiversion', 1),
+        ('ei_pad', 7),
+        ('e_type', 2),
+        ('e_machine', 2),
+        ('e_version', 4),
+        ('e_entry', 8),
+        ('e_phoff', 8),
+        ('e_shoff', 8),
+        ('e_flags', 4),
+        ('e_ehsize', 2),
+        ('e_phentsize', 2),
+        ('e_phnum', 2),
+        ('e_shentsize', 2),
+        ('e_shnum', 2),
+        ('e_shstrndx', 2),
+    ]
+
+    self._ValidateBitness(data)
+    byte_order = self._ReadByteOrder(data)
+    super(ElfHeader, self).__init__(byte_order, fields)
+
+    self.ParseBytes(data, 0)
+    if self.e_type != ElfHeader.EType.ET_DYN:
+      raise RuntimeError('Only shared libraries are supported')
+
+    self.phdrs = []
+    self._ParsePhdrs(data)
+
+  def GetPhdrs(self):
+    """Returns the list of file's program headers."""
+    return self.phdrs

tools/android/elf_compression/test/elf_headers_test.py

+#!/usr/bin/env python3
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import os
+import pathlib
+import sys
+import unittest
+
+sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
+import elf_headers
+
+ELF_HEADER_TEST_LIBRARY = 'testdata/lib.so'
+
+
+class ElfHeaderTest(unittest.TestCase):
+
+  def setUp(self):
+    super(ElfHeaderTest, self).setUp()
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    self.library_path = os.path.join(script_dir, ELF_HEADER_TEST_LIBRARY)
+
+  def testElfHeaderParsing(self):
+    with open(self.library_path, 'rb') as f:
+      data = f.read()
+    elf = elf_headers.ElfHeader(data)
+    # Validating all of the ELF header fields.
+    self.assertEqual(elf.e_type, elf_headers.ElfHeader.EType.ET_DYN)
+    self.assertEqual(elf.e_entry, 0x1040)
+    self.assertEqual(elf.e_phoff, 64)
+    self.assertEqual(elf.e_shoff, 14136)
+    self.assertEqual(elf.e_flags, 0)
+    self.assertEqual(elf.e_ehsize, 64)
+    self.assertEqual(elf.e_phentsize, 56)
+    self.assertEqual(elf.e_phnum, 8)
+    self.assertEqual(elf.e_shentsize, 64)
+    self.assertEqual(elf.e_shnum, 26)
+    self.assertEqual(elf.e_shstrndx, 25)
+    # Validating types and amounts of all segments excluding GNU specific ones.
+    phdrs = elf.GetPhdrs()
+    self.assertEqual(len(phdrs), 8)
+
+    phdr_types = [
+        elf_headers.ProgramHeader.Type.PT_LOAD,
+        elf_headers.ProgramHeader.Type.PT_LOAD,
+        elf_headers.ProgramHeader.Type.PT_LOAD,
+        elf_headers.ProgramHeader.Type.PT_LOAD,
+        elf_headers.ProgramHeader.Type.PT_DYNAMIC,
+        None,
+        None,
+        None,
+    ]
+    for i in range(0, len(phdrs)):
+      if phdr_types[i] is not None:
+        self.assertEqual(phdrs[i].p_type, phdr_types[i])
+
+    # Validating all of the fields of the first segment.
+    load_phdr = phdrs[0]
+    self.assertEqual(load_phdr.p_offset, 0x0)
+    self.assertEqual(load_phdr.p_vaddr, 0x0)
+    self.assertEqual(load_phdr.p_paddr, 0x0)
+    self.assertEqual(load_phdr.p_filesz, 0x468)
+    self.assertEqual(load_phdr.p_memsz, 0x468)
+    self.assertEqual(load_phdr.p_flags, 0b100)
+    self.assertEqual(load_phdr.p_align, 0x1000)
+    # Validating offsets of the second segment
+    load_phdr = phdrs[1]
+    self.assertEqual(load_phdr.p_offset, 0x1000)
+    self.assertEqual(load_phdr.p_vaddr, 0x1000)
+    self.assertEqual(load_phdr.p_paddr, 0x1000)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/android/elf_compression/test/run_tests.py

+#!/usr/bin/env python3
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import unittest
+
+TESTS = [
+    'compression_script_test',
+    'elf_headers_test',
+]
+
+if __name__ == '__main__':
+  suite = unittest.TestSuite()
+  suite.addTests(unittest.defaultTestLoader.loadTestsFromNames(TESTS))
+  unittest.TextTestRunner().run(suite)

tools/android/elf_compression/test/testdata/lib.c

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This is a source code of lib.so file located in this directory. This file
+// serves as a testdata for testing elf_headers.py. We need to store .so
+// precompiled and not build it on demand since we can't rely on resulting ELF
+// structure not changing between different compilations or due to change in
+// clang version.
+//
+// The library was build with the following command:
+//    clang lib.c -shared -fPIC -O2 -o lib.so
+//
+// This library is intentionally very small to both ensure the speed of the test
+// and to not add huge binary file into source control system.
+
+int array[3] = {1, 2, 3};
+
+int GetSum() {
+  return array[0] + array[1] + array[2];
+}