oss-fuzz initial integration

.gitignore

@@ -13,6 +13,7 @@ doc/qpdf.1
 doc/zlib-flate.1
 examples/build/
 external-libs
+fuzz/build/
 libqpdf.map
 libqpdf.pc
 libqpdf/build/

ChangeLog

@@ -1,3 +1,10 @@
+2019-06-13  Jay Berkenbilt  <ejb@ql.org>
+
+	* Perform initial integration of Google's oss-fuzz project by
+	copying the fuzzer someone from Google already did into the qpdf
+	repository and adding build support. This shift in control is in
+	preparation for an ideal integration with oss-fuzz.
+
 2019-06-09  Jay Berkenbilt  <ejb@ql.org>
 
 	* When /DecodeParms is an empty list, ignore it on read and delete

Makefile

@@ -35,7 +35,7 @@
 # install to install in a separate location.  This is useful for
 # packagers.
 
-BUILD_ITEMS := manual libqpdf zlib-flate libtests qpdf examples
+BUILD_ITEMS := manual libqpdf zlib-flate libtests qpdf fuzz examples
 OUTPUT_DIR = build
 ALL_TARGETS =
 

NOTICE.md

@@ -10,6 +10,8 @@ Versions of qpdf prior to version 7 were released under the terms of version 2.0
 
 The qpdf distribution includes a copy of [qtest](http://qtest.qbilt.org), which is released under the terms of the [version 2.0 of the Artistic license](https://opensource.org/licenses/Artistic-2.0), which can be found at https://opensource.org/licenses/Artistic-2.0.
 
+The standalone fuzz target runner (fuzz/standalone_fuzz_target_runner.cc) is copyright 2017 by Google and is also released under the Apache license, Version 2.0.
+
 The Rijndael encryption implementation used as the basis for AES encryption and decryption support comes from Philip J. Erdelsky's public domain implementation.  The files `libqpdf/rijndael.cc` and `libqpdf/qpdf/rijndael.h` remain in the public domain.  They were obtained from
 * http://www.efgh.com/software/rijndael.htm
 * http://www.efgh.com/software/rijndael.txt

README-maintainer

@@ -17,6 +17,19 @@ Memory checks:
    LDFLAGS="-fsanitize=address -fsanitize=undefined" \
    --enable-werror --disable-shared
 
+GOOGLE OSS-FUZZ
+
+* https://github.com/google/oss-fuzz/tree/master/projects/qpdf
+* To test locally, see https://github.com/google/oss-fuzz/tree/master/docs/,
+  especially new_project_guide.md
+
+Clone the oss-fuzz project. From the root directory of the repository:
+
+  python infra/helper.py build_image --pull qpdf
+  python infra/helper.py build_fuzzers qpdf
+  python infra/helper.py check_build qpdf
+  python infra/helper.py build_fuzzers --sanitizer coverage qpdf
+  python infra/helper.py coverage qpdf
 
 CODING RULES
 

autoconf.mk.in

@@ -37,6 +37,7 @@ XMLLINT=@XMLLINT@
 BUILD_HTML=@BUILD_HTML@
 BUILD_PDF=@BUILD_PDF@
 VALIDATE_DOC=@VALIDATE_DOC@
+OSS_FUZZ=@OSS_FUZZ@
 QPDF_SKIP_TEST_COMPARE_IMAGES=@QPDF_SKIP_TEST_COMPARE_IMAGES@
 BUILDRULES=@BUILDRULES@
 HAVE_LD_VERSION_SCRIPT=@HAVE_LD_VERSION_SCRIPT@

autofiles.sums

@@ -1,4 +1,4 @@
-ba2adf968b787efe32cd4396a5cfeceeb52d2c48686bdc21a3b03edae169632c  configure.ac
+f0057d67ba676a48d07264f6c9a947c59c36dee48dbd6c41903d5f03c586a9cf  configure.ac
 35bc5c645dc42d47f2daeea06f8f3e767c8a1aee6a35eb2b4854fd2ce66c3413  m4/ax_random_device.m4
 37f8897d5f68d7d484e5457832a8f190ddb7507fa2a467cb7ee2be40a4364643  m4/libtool.m4
 e77ebba8361b36f14b4d0927173a034b98c5d05049697a9ded84d85eb99a7990  m4/ltoptions.m4

azure-pipelines.yml

@@ -100,3 +100,11 @@ jobs:
       buildPlatform: AppImage
   dependsOn: Linux
   condition: succeeded()
+- job: Fuzzers
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+  - script: azure-pipelines/build-fuzzer
+    displayName: 'Build Fuzzer'
+  dependsOn: Linux
+  condition: succeeded()

azure-pipelines/build-fuzzer

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -ex
+export WORK=$PWD/work
+export OUT=$PWD/out
+mkdir -p $WORK $OUT
+sudo apt-get update
+sudo apt-get -y install \
+   autoconf build-essential zlib1g-dev libjpeg-dev
+./fuzz/oss-fuzz-build
+ls -l out/qpdf*fuzzer
+ls -l out/

configure

@@ -630,6 +630,7 @@ ac_includes_default="\
 
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
+OSS_FUZZ
 VALIDATE_DOC
 BUILD_PDF
 BUILD_HTML
@@ -774,6 +775,7 @@ enable_doc_maintenance
 enable_html_doc
 enable_pdf_doc
 enable_validate_doc
+enable_oss_fuzz
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1449,6 +1451,8 @@ Optional Features:
   --enable-html-doc       whether to build HTML documents
   --enable-pdf-doc        whether to build PDF documents
   --enable-validate-doc   whether to validate xml document source
+  --enable-doc-maintenance
+                          if set, build static fuzzers for oss-fuzz
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -16924,6 +16928,19 @@ else
 fi
 
 
+
+# Check whether --enable-oss-fuzz was given.
+if test "${enable_oss_fuzz+set}" = set; then :
+  enableval=$enable_oss_fuzz; if test "$enableval" = "yes"; then
+      OSS_FUZZ=1;
+    else
+      OSS_FUZZ=0;
+    fi
+else
+  OSS_FUZZ=0
+fi
+
+
 if test "$VALIDATE_DOC" = "1"; then
    if test "$XMLLINT" = ""; then
       MISSING_XMLLINT=1

configure.ac

@@ -501,6 +501,17 @@ AC_ARG_ENABLE(validate-doc,
     fi],
    [VALIDATE_DOC=$doc_default])
 
+AC_SUBST(OSS_FUZZ)
+AC_ARG_ENABLE(oss-fuzz,
+   AS_HELP_STRING([--enable-doc-maintenance],
+		  [if set, build static fuzzers for oss-fuzz]),
+   [if test "$enableval" = "yes"; then
+      OSS_FUZZ=1;
+    else
+      OSS_FUZZ=0;
+    fi],
+   [OSS_FUZZ=0])
+
 if test "$VALIDATE_DOC" = "1"; then
    if test "$XMLLINT" = ""; then
       MISSING_XMLLINT=1

fuzz/Makefile

@@ -0,0 +1 @@
+include ../make/proxy.mk

fuzz/README.md

@@ -0,0 +1 @@
+pdf.dict was copied from https://raw.githubusercontent.com/rc0r/afl-fuzz/master/dictionaries/pdf.dict

fuzz/build.mk

@@ -0,0 +1,82 @@
+# This directory contains support for Google's oss-fuzz project. See
+# https://github.com/google/oss-fuzz/tree/master/projects/qpdf
+
+FUZZERS = \
+    qpdf_read_memory_fuzzer
+
+DEFAULT_FUZZ_RUNNER := standalone_fuzz_target_runner
+OBJ_DEFAULT_FUZZ := fuzz/$(OUTPUT_DIR)/$(DEFAULT_FUZZ_RUNNER).$(OBJ)
+
+BINS_fuzz = $(foreach B,$(FUZZERS),fuzz/$(OUTPUT_DIR)/$(call binname,$(B)))
+TARGETS_fuzz = $(OBJ_DEFAULT_FUZZ) $(BINS_fuzz)
+
+INCLUDES_fuzz = include
+
+# LIB_FUZZING_ENGINE is overridden by oss-fuzz
+LIB_FUZZING_ENGINE ?= $(OBJ_DEFAULT_FUZZ)
+
+# Depend on OBJ_DEFAULT_FUZZ to ensure that it is always compiled.
+# Don't depend on LIB_FUZZING_ENGINE, which we can't build. When used
+# by oss-fuzz, it will be there.
+$(BINS_fuzz): $(TARGETS_libqpdf) $(OBJ_DEFAULT_FUZZ)
+
+# -----
+
+$(foreach B,$(FUZZERS),$(eval \
+  OBJS_$(B) = $(call src_to_obj,fuzz/$(B).cc)))
+
+ifeq ($(GENDEPS),1)
+-include $(foreach B,$(FUZZERS),$(call obj_to_dep,$(OBJS_$(B))))
+endif
+
+$(foreach B,$(DEFAULT_FUZZ_RUNNER),$(eval \
+  fuzz/$(OUTPUT_DIR)/%.$(OBJ): fuzz/$(B).cc ; \
+	$(call compile,fuzz/$(B).cc,$(INCLUDES_fuzz))))
+
+$(foreach B,$(FUZZERS),$(eval \
+  $(OBJS_$(B)): fuzz/$(OUTPUT_DIR)/%.$(OBJ): fuzz/$(B).cc ; \
+	$(call compile,fuzz/$(B).cc,$(INCLUDES_fuzz))))
+
+ifeq ($(suffix $(LIB_FUZZING_ENGINE)),.$(OBJ))
+  FUZZ_as_obj := $(LIB_FUZZING_ENGINE)
+  FUZZ_as_lib :=
+else
+  FUZZ_as_obj :=
+  FUZZ_as_lib := $(LIB_FUZZING_ENGINE)
+endif
+
+$(foreach B,$(FUZZERS),$(eval \
+  fuzz/$(OUTPUT_DIR)/$(call binname,$(B)): $(OBJS_$(B)) ; \
+	$(call makebin,$(OBJS_$(B)) $(FUZZ_as_obj),$$@,$(LDFLAGS_libqpdf) $(LDFLAGS),$(FUZZ_as_lib) $(LIBS_libqpdf) $(LIBS))))
+
+ifeq ($(OSS_FUZZ),1)
+
+# Build fuzzers linked with static libraries and installed into a
+# location provided by oss-fuzz. This is specifically to support the
+# oss-fuzz project. These rules won't on systems that don't allow main
+# to be in a library or don't name their libraries libsomething.a.
+
+STATIC_BINS_fuzz := $(foreach B,$(FUZZERS),fuzz/$(OUTPUT_DIR)/static/$(call binname,$(B)))
+$(STATIC_BINS_fuzz): $(TARGETS_libqpdf) $(OBJ_DEFAULT_FUZZ)
+
+# OUT is provided in the oss-fuzz environment
+OUT ?= $(CURDIR)/fuzz/$(OUTPUT_DIR)/fuzz-install
+
+# These are not fully static, but they statically link with qpdf and
+# our external dependencies other than system libraries.
+$(foreach B,$(FUZZERS),$(eval \
+  fuzz/$(OUTPUT_DIR)/static/$(call binname,$(B)): $(OBJS_$(B)) ; \
+	$(call makebin,$(OBJS_$(B)),$$@,$(LDFLAGS_libqpdf) $(LDFLAGS),$(LIB_FUZZING_ENGINE) $(patsubst -l%,-l:lib%.a,$(LIBS_libqpdf) $(LIBS)))))
+
+# The install_fuzz target is used by build.sh in oss-fuzz's qpdf project.
+install_fuzz: $(STATIC_BINS_fuzz)
+	mkdir -p $(OUT)
+	cp fuzz/pdf.dict $(STATIC_BINS_fuzz) $(OUT)/
+	for B in $(FUZZERS); do \
+	  cp fuzz/options $(OUT)/$${B}.options; \
+	  if test -d fuzz/$${B}_seed_corpus; then \
+	    (cd fuzz/$${B}_seed_corpus; zip -q -r $(OUT)/$${B}_seed_corpus.zip .); \
+	  fi; \
+	done
+
+endif # OSS_FUZZ

fuzz/options

@@ -0,0 +1,2 @@
+[libfuzzer]
+dict = pdf.dict

fuzz/oss-fuzz-build

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+# This is used invoked from
+# https://github.com/google/oss-fuzz/blob/master/projects/qpdf/build.sh
+
+# It should be run from the top level directory of a clean checkout of
+# qpdf. It is also exercised in ../azure-pipelines/build-fuzzer
+
+./configure \
+  --enable-oss-fuzz \
+  --enable-static \
+  --disable-shared \
+  --prefix="$WORK" \
+  LDFLAGS="-L$WORK/lib" \
+  CPPFLAGS="-I$WORK/include" \
+  LIBS="-pthread"
+make -j$(nproc) install
+make install_fuzz

fuzz/qpdf_read_memory_fuzzer.cc

@@ -0,0 +1,15 @@
+#include "qpdf/qpdf-c.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdlib>
+
+extern "C" int LLVMFuzzerTestOneInput(const unsigned char* data, size_t size) {
+  const size_t kMaxSize = 64 * 1024; // 64 KiB
+  size = std::min(size, kMaxSize);
+  _qpdf_data* qpdf = qpdf_init();
+  const char* buffer = reinterpret_cast<const char*>(data);
+  qpdf_read_memory(qpdf, /*description=*/"", buffer, size, /*password=*/"");
+  qpdf_cleanup(&qpdf);
+  return 0;
+}

fuzz/standalone_fuzz_target_runner.cc

@@ -0,0 +1,36 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+
+// Except for formatting, comments, and portability, this was copied
+// from projects/example/my-api-repo/standalone_fuzz_target_runner.cpp
+// in https://github.com/oss-fuzz
+
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+
+extern "C" int LLVMFuzzerTestOneInput(unsigned char const* data, size_t size);
+
+int main(int argc, char **argv)
+{
+    for (int i = 1; i < argc; i++)
+    {
+        std::ifstream in(argv[i]);
+        in.seekg(0, in.end);
+        size_t length = in.tellg();
+        in.seekg (0, in.beg);
+        std::cout << "Reading " << length << " bytes from " << argv[i]
+                  << std::endl;
+        // Allocate exactly length bytes so that we reliably catch
+        // buffer overflows.
+        std::vector<char> bytes(length);
+        in.read(bytes.data(), bytes.size());
+        assert(in);
+        LLVMFuzzerTestOneInput(
+            reinterpret_cast<unsigned char const*>(bytes.data()),
+            bytes.size());
+        std::cout << "Execution successful" << std::endl;
+    }
+    return 0;
+}