Add custom malloc with max limit to prevent OOM

This adds the custom malloc/free functions from the old
libpng_read_fuzzer to the upstream fuzzer to prevent clusterfuzz running
into OOM.

Bug: 904054
Change-Id: Ibb824beb191cb5657687c55ee2db8c7783547bad
Reviewed-on: https://chromium-review.googlesource.com/c/1330936
Commit-Queue: Christopher Thompson <cthomp@chromium.org>
Reviewed-by: Leon Scroggins <scroggo@chromium.org>
Reviewed-by: Max Moroz <mmoroz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#607622}

third_party/libpng/README.chromium

@@ -19,5 +19,7 @@ Updated to 1.6.35, stripped all unneeded files.
   png_check_chunk_length to call png_benign_error instead of png_chunk_error --
   see crbug.com/827754).
 - Keeps the fuzz target in contrib/oss-fuzz/ for running on clusterfuzz.
+- Applies the patch in patches/0002-fuzzeroom.patch to prevent clusterfuzz
+  running into OOM errors.
 
 [1] https://github.com/glennrp/libpng/pull/203
\ No newline at end of file

third_party/libpng/contrib/oss-fuzz/libpng_read_fuzzer.cc

@@ -78,6 +78,22 @@ void user_read_data(png_structp png_ptr, png_bytep data, size_t length) {
   buf_state->data += length;
 }
 
+void* limited_malloc(png_structp, png_alloc_size_t size) {
+  // libpng may allocate large amounts of memory that the fuzzer reports as
+  // an error. In order to silence these errors, make libpng fail when trying
+  // to allocate a large amount. This allocator used to be in the Chromium
+  // version of this fuzzer.
+  // This number is chosen to match the default png_user_chunk_malloc_max.
+  if (size > 8000000)
+    return nullptr;
+
+  return malloc(size);
+}
+
+void default_free(png_structp, png_voidp ptr) {
+  return free(ptr);
+}
+
 static const int kPngHeaderSize = 8;
 
 // Entry point for LibFuzzer.
@@ -118,6 +134,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
     return 0;
   }
 
+  // Use a custom allocator that fails for large allocations to avoid OOM.
+  png_set_mem_fn(png_handler.png_ptr, nullptr, limited_malloc, default_free);
+
   png_set_crc_action(png_handler.png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE);
 #ifdef PNG_IGNORE_ADLER32
   png_set_option(png_handler.png_ptr, PNG_IGNORE_ADLER32, PNG_OPTION_ON);

third_party/libpng/patches/0002-fuzzeroom.patch

+From bd38fe17b88d63245832978286f2fe12c9ed6de7 Mon Sep 17 00:00:00 2001
+From: Christopher Thompson <cthomp@chromium.org>
+Date: Mon, 12 Nov 2018 09:47:32 -0800
+Subject: [PATCH] Add custom malloc with max limit to prevent OOM
+
+This adds the custom malloc/free functions from the old
+libpng_read_fuzzer to the upstream fuzzer to prevent clusterfuzz running
+into OOM.
+
+Bug: 904054
+Change-Id: Ibb824beb191cb5657687c55ee2db8c7783547bad
+---
+
+diff --git a/third_party/libpng/contrib/oss-fuzz/libpng_read_fuzzer.cc b/third_party/libpng/contrib/oss-fuzz/libpng_read_fuzzer.cc
+index 3a8ecab..ea27d20 100644
+--- a/third_party/libpng/contrib/oss-fuzz/libpng_read_fuzzer.cc
++++ b/third_party/libpng/contrib/oss-fuzz/libpng_read_fuzzer.cc
+@@ -78,6 +78,21 @@
+   buf_state->data += length;
+ }
+ 
++void* limited_malloc(png_structp, png_alloc_size_t size) {
++  // libpng may allocate large amounts of memory that the fuzzer reports as
++  // an error. In order to silence these errors, make libpng fail when trying
++  // to allocate a large amount. This allocator used to be in the Chromium
++  // version of this fuzzer.
++  // This number is chosen to match the default png_user_chunk_malloc_max.
++  if (size > 8000000)
++    return nullptr;
++
++  return malloc(size);
++}
++
++void default_free(png_structp, png_voidp ptr) {
++  return free(ptr);
++}
++
+ static const int kPngHeaderSize = 8;
+ 
+ // Entry point for LibFuzzer.
+@@ -118,6 +133,9 @@
+     return 0;
+   }
+ 
++  // Use a custom allocator that fails for large allocations to avoid OOM.
++  png_set_mem_fn(png_handler.png_ptr, nullptr, limited_malloc, default_free);
++
+   png_set_crc_action(png_handler.png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE);
+ #ifdef PNG_IGNORE_ADLER32
+   png_set_option(png_handler.png_ptr, PNG_IGNORE_ADLER32, PNG_OPTION_ON);

third_party/libpng/patches/README

@@ -8,3 +8,5 @@ upstreamed or released to the canonical libpng repository [1] yet.
  - 0000-plte.patch: ARM NEON optimizations not yet released upstream.
  - 0001-chunkerror.patch: Change chunk errors into benign errors
    (https://crrev.com/c/1014027).
+ - 0002-fuzzeroom.patch: Add custom malloc/free to limit too-large
+   allocations to prevent clusterfuzz OOMs (https://crrev.com/c/1330936).
\ No newline at end of file