From 34729e37e001a92e5c7da250bd3118e052c79e64 Mon Sep 17 00:00:00 2001
From: m-holger <m.holger@qpdf.org>
Date: Thu, 18 Jul 2024 12:28:26 +0100
Subject: [PATCH] Limit memory used by Pl_PNGFilter and Pl_TIFFPredictor during
 fuzzing

---
 fuzz/qpdf_fuzzer.cc              |  5 +++++
 libqpdf/Pl_PNGFilter.cc          | 14 ++++++++++++++
 libqpdf/Pl_TIFFPredictor.cc      | 14 ++++++++++++++
 libqpdf/qpdf/Pl_PNGFilter.hh     |  4 ++++
 libqpdf/qpdf/Pl_TIFFPredictor.hh |  4 ++++
 5 files changed, 41 insertions(+)

diff --git a/fuzz/qpdf_fuzzer.cc b/fuzz/qpdf_fuzzer.cc
index db33b2ce..3486c1be 100644
--- a/fuzz/qpdf_fuzzer.cc
+++ b/fuzz/qpdf_fuzzer.cc
@@ -2,6 +2,8 @@
 #include <qpdf/BufferInputSource.hh>
 #include <qpdf/Pl_DCT.hh>
 #include <qpdf/Pl_Discard.hh>
+#include <qpdf/Pl_PNGFilter.hh>
+#include <qpdf/Pl_TIFFPredictor.hh>
 #include <qpdf/QPDF.hh>
 #include <qpdf/QPDFAcroFormDocumentHelper.hh>
 #include <qpdf/QPDFOutlineDocumentHelper.hh>
@@ -179,6 +181,9 @@ FuzzHelper::doChecks()
     // occur legitimately and therefore must be allowed during normal operations.
     Pl_DCT::setMemoryLimit(1'000'000'000);
 
+    Pl_PNGFilter::setMemoryLimit(1'000'000'000);
+    Pl_TIFFPredictor::setMemoryLimit(1'000'000'000);
+
     // Do not decompress corrupt data. This may cause extended runtime within jpeglib without
     // exercising additional code paths in qpdf, and potentially causing counterproductive timeouts.
     Pl_DCT::setThrowOnCorruptData(true);
diff --git a/libqpdf/Pl_PNGFilter.cc b/libqpdf/Pl_PNGFilter.cc
index 4c2dd623..4d2bc71b 100644
--- a/libqpdf/Pl_PNGFilter.cc
+++ b/libqpdf/Pl_PNGFilter.cc
@@ -7,6 +7,11 @@
 #include <cstring>
 #include <stdexcept>
 
+namespace
+{
+    unsigned long long memory_limit{0};
+} // namespace
+
 static int
 abs_diff(int a, int b)
 {
@@ -41,6 +46,9 @@ Pl_PNGFilter::Pl_PNGFilter(
     if ((bpr == 0) || (bpr > (UINT_MAX - 1))) {
         throw std::runtime_error("PNGFilter created with invalid columns value");
     }
+    if (memory_limit > 0 && bpr > (memory_limit / 2U)) {
+        throw std::runtime_error("PNGFilter memory limit exceeded");
+    }
     this->bytes_per_row = bpr & UINT_MAX;
     this->buf1 = QUtil::make_shared_array<unsigned char>(this->bytes_per_row + 1);
     this->buf2 = QUtil::make_shared_array<unsigned char>(this->bytes_per_row + 1);
@@ -53,6 +61,12 @@ Pl_PNGFilter::Pl_PNGFilter(
     this->incoming = (action == a_encode ? this->bytes_per_row : this->bytes_per_row + 1);
 }
 
+void
+Pl_PNGFilter::setMemoryLimit(unsigned long long limit)
+{
+    memory_limit = limit;
+}
+
 void
 Pl_PNGFilter::write(unsigned char const* data, size_t len)
 {
diff --git a/libqpdf/Pl_TIFFPredictor.cc b/libqpdf/Pl_TIFFPredictor.cc
index ec477049..c2cc8561 100644
--- a/libqpdf/Pl_TIFFPredictor.cc
+++ b/libqpdf/Pl_TIFFPredictor.cc
@@ -7,6 +7,11 @@
 #include <climits>
 #include <stdexcept>
 
+namespace
+{
+    unsigned long long memory_limit{0};
+} // namespace
+
 Pl_TIFFPredictor::Pl_TIFFPredictor(
     char const* identifier,
     Pipeline* next,
@@ -31,9 +36,18 @@ Pl_TIFFPredictor::Pl_TIFFPredictor(
     if ((bpr == 0) || (bpr > (UINT_MAX - 1))) {
         throw std::runtime_error("TIFFPredictor created with invalid columns value");
     }
+    if (memory_limit > 0 && bpr > (memory_limit / 2U)) {
+        throw std::runtime_error("TIFFPredictor memory limit exceeded");
+    }
     this->bytes_per_row = bpr & UINT_MAX;
 }
 
+void
+Pl_TIFFPredictor::setMemoryLimit(unsigned long long limit)
+{
+    memory_limit = limit;
+}
+
 void
 Pl_TIFFPredictor::write(unsigned char const* data, size_t len)
 {
diff --git a/libqpdf/qpdf/Pl_PNGFilter.hh b/libqpdf/qpdf/Pl_PNGFilter.hh
index ed8e1e9c..9f1950e9 100644
--- a/libqpdf/qpdf/Pl_PNGFilter.hh
+++ b/libqpdf/qpdf/Pl_PNGFilter.hh
@@ -24,6 +24,10 @@ class Pl_PNGFilter: public Pipeline
         unsigned int bits_per_sample = 8);
     ~Pl_PNGFilter() override = default;
 
+    // Limit the memory used.
+    // NB This is a static option affecting all Pl_PNGFilter instances.
+    static void setMemoryLimit(unsigned long long limit);
+
     void write(unsigned char const* data, size_t len) override;
     void finish() override;
 
diff --git a/libqpdf/qpdf/Pl_TIFFPredictor.hh b/libqpdf/qpdf/Pl_TIFFPredictor.hh
index 3f448f16..4e32936f 100644
--- a/libqpdf/qpdf/Pl_TIFFPredictor.hh
+++ b/libqpdf/qpdf/Pl_TIFFPredictor.hh
@@ -22,6 +22,10 @@ class Pl_TIFFPredictor: public Pipeline
         unsigned int bits_per_sample = 8);
     ~Pl_TIFFPredictor() override = default;
 
+    // Limit the memory used.
+    // NB This is a static option affecting all Pl_TIFFPredictor instances.
+    static void setMemoryLimit(unsigned long long limit);
+
     void write(unsigned char const* data, size_t len) override;
     void finish() override;