Merge pull request #1221 from m-holger/fuzz

Refine handling of severely damaged files

fuzz/CMakeLists.txt

@@ -118,6 +118,7 @@ set(CORPUS_OTHER
   68377.fuzz
   68668.fuzz
   68915.fuzz
+  69857.fuzz
 )
 
 set(CORPUS_DIR ${CMAKE_CURRENT_BINARY_DIR}/qpdf_corpus)

fuzz/qtest/fuzz.test

@@ -21,7 +21,7 @@ my @fuzzers = (
     ['pngpredictor' => 1],
     ['runlength' => 6],
     ['tiffpredictor' => 2],
-    ['qpdf' => 60],             # increment when adding new files
+    ['qpdf' => 61],             # increment when adding new files
     );
 
 my $n_tests = 0;

libqpdf/QPDF.cc

@@ -471,6 +471,10 @@ QPDF::parse(char const* password)
 
     initializeEncryption();
     m->parsed = true;
+    if (m->xref_table.size() > 0 && !getRoot().getKey("/Pages").isDictionary()) {
+        // QPDFs created from JSON have an empty xref table and no root object yet.
+        throw damagedPDF("", 0, "unable to find page tree");
+    }
 }
 
 void
@@ -543,6 +547,9 @@ QPDF::reconstruct_xref(QPDFExc& e)
 
     m->file->seek(0, SEEK_END);
     qpdf_offset_t eof = m->file->tell();
+    // Sanity check on object ids. All objects must appear in xref table / stream. In all realistic
+    // scenarios at leat 3 bytes are required.
+    auto max_obj_id = eof / 3;
     m->file->seek(0, SEEK_SET);
     qpdf_offset_t line_start = 0;
     // Don't allow very long tokens here during recovery.
@@ -560,7 +567,12 @@ QPDF::reconstruct_xref(QPDFExc& e)
             if ((t2.isInteger()) && (readToken(m->file, MAX_LEN).isWord("obj"))) {
                 int obj = QUtil::string_to_int(t1.getValue().c_str());
                 int gen = QUtil::string_to_int(t2.getValue().c_str());
-                insertReconstructedXrefEntry(obj, token_start, gen);
+                if (obj <= max_obj_id) {
+                    insertReconstructedXrefEntry(obj, token_start, gen);
+                } else {
+                    warn(damagedPDF(
+                        "", 0, "ignoring object with impossibly large id " + std::to_string(obj)));
+                }
             }
         } else if (!m->trailer.isInitialized() && t1.isWord("trailer")) {
             QPDFObjectHandle t = readTrailer();

libqpdf/QPDFWriter.cc

@@ -1121,7 +1121,6 @@ QPDFWriter::enqueueObject(QPDFObjectHandle object)
         } else if (obj.renumber == -1) {
             // This can happen if a specially constructed file indicates that an object stream is
             // inside itself.
-            QTC::TC("qpdf", "QPDFWriter ignore self-referential object stream");
         }
         return;
     } else if (!m->linearized) {

qpdf/qpdf.testcov

@@ -277,7 +277,6 @@ QPDF ignore first extra space in xref entry 0
 QPDF ignore second extra space in xref entry 0
 QPDF ignore length error xref entry 0
 QPDF_encryption pad short parameter 0
-QPDFWriter ignore self-referential object stream 0
 QPDFObjectHandle found old angle 1
 QPDF_Stream special filters 3
 QPDFTokenizer block long token 0

qpdf/qtest/invalid-objects.test

@@ -19,7 +19,7 @@ my $n_tests = 4;
 $td->runtest("closed input source",
              {$td->COMMAND => "test_driver 73 minimal.pdf"},
              {$td->FILE => "test73.out",
-              $td->EXIT_STATUS => 2},
+              $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 
 $td->runtest("empty object",

qpdf/qtest/qpdf/bad11-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad11.pdf: file is damaged
 WARNING: bad11.pdf (trailer, offset 905): /Prev key in trailer dictionary is not an integer
 WARNING: bad11.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad11.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad12-recover.out

@@ -1,4 +1,5 @@
 WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7)
+WARNING: bad12.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad12.out

@@ -1,4 +1,5 @@
 WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7)
+WARNING: bad12.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad2-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad2.pdf: file is damaged
 WARNING: bad2.pdf: can't find startxref
 WARNING: bad2.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad2.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad3-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad3.pdf: file is damaged
 WARNING: bad3.pdf (offset 542): xref not found
 WARNING: bad3.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad3.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad4-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad4.pdf: file is damaged
 WARNING: bad4.pdf (xref table, offset 547): xref syntax invalid
 WARNING: bad4.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad4.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad5-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad5.pdf: file is damaged
 WARNING: bad5.pdf (xref table, offset 591): invalid xref entry (obj=2)
 WARNING: bad5.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad5.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad6-recover.out

@@ -1,3 +1,4 @@
+WARNING: bad6.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad6.out

@@ -1,3 +1,4 @@
+WARNING: bad6.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/bad8-recover.out

@@ -1,6 +1,7 @@
 WARNING: bad8.pdf: file is damaged
 WARNING: bad8.pdf (offset 543): xref not found
 WARNING: bad8.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad8.pdf (object 2 0, offset 128): expected endobj
 /QTest is implicit
 /QTest is direct and has type null (2)
 /QTest is null

qpdf/qtest/qpdf/fuzz-16214.out

@@ -11,8 +11,8 @@ WARNING: fuzz-16214.pdf (object 1 0, offset 7189): expected n n obj
 WARNING: fuzz-16214.pdf: Attempting to reconstruct cross-reference table
 WARNING: fuzz-16214.pdf (offset 7207): error decoding stream data for object 2 0: stream inflate: inflate: data: invalid code lengths set
 WARNING: fuzz-16214.pdf (offset 7207): getStreamData called on unfilterable stream
-WARNING: fuzz-16214.pdf (object 11 0, offset 11551): supposed object stream 5 has wrong type
+WARNING: fuzz-16214.pdf (object 8 0, offset 7207): supposed object stream 5 has wrong type
-WARNING: fuzz-16214.pdf (object 11 0, offset 11551): object stream 5 has incorrect keys
+WARNING: fuzz-16214.pdf (object 8 0, offset 7207): object stream 5 has incorrect keys
 WARNING: fuzz-16214.pdf (object 21 0, offset 3639): expected endstream
 WARNING: fuzz-16214.pdf (object 21 0, offset 3112): attempting to recover stream length
 WARNING: fuzz-16214.pdf (object 21 0, offset 3112): recovered stream length: 340

qpdf/qtest/qpdf/issue-119.out

@@ -1,3 +1 @@
-WARNING: issue-119.pdf (object 4 0, offset 298): expected dictionary key but found non-name object; inserting key /QPDFFake1
+qpdf: issue-119.pdf: unable to find page tree
-WARNING: issue-119.pdf (object 4 0, offset 298): expected dictionary key but found non-name object; inserting key /QPDFFake2
-qpdf: operation succeeded with warnings; resulting file may have some problems

qpdf/qtest/qpdf/issue-120.out

@@ -1,7 +1 @@
-WARNING: issue-120.pdf (offset 85): loop detected resolving object 3 0
+qpdf: issue-120.pdf: unable to find page tree
-WARNING: issue-120.pdf (object 6 0, offset 85): supposed object stream 3 is not a stream
-WARNING: issue-120.pdf: file is damaged
-WARNING: issue-120.pdf (object 8 10, offset 26880): expected n n obj
-WARNING: issue-120.pdf: Attempting to reconstruct cross-reference table
-WARNING: issue-120.pdf: object 8 10 not found in file after regenerating cross reference table
-qpdf: operation succeeded with warnings; resulting file may have some problems

qpdf/qtest/qpdf/issue-143.out

@@ -14,6 +14,4 @@ WARNING: issue-143.pdf (object 1 0, offset 21): stream dictionary lacks /Length
 WARNING: issue-143.pdf (object 1 0, offset 84): attempting to recover stream length
 WARNING: issue-143.pdf (object 1 0, offset 84): recovered stream length: 606
 WARNING: issue-143.pdf object stream 1 (object 2 0, offset 33): expected dictionary key but found non-name object; inserting key /QPDFFake1
-WARNING: issue-143.pdf: object 0/0 has unexpected xref entry type
+qpdf: issue-143.pdf: unable to find page tree
-WARNING: issue-143.pdf (object 2 0, offset 84): supposed object stream 12336 is not a stream
-qpdf: operation succeeded with warnings; resulting file may have some problems

qpdf/qtest/qpdf/issue-147.out

@@ -3,6 +3,5 @@ WARNING: issue-147.pdf: file is damaged
 WARNING: issue-147.pdf: can't find startxref
 WARNING: issue-147.pdf: Attempting to reconstruct cross-reference table
 WARNING: issue-147.pdf (trailer, offset 9): expected dictionary key but found non-name object; inserting key /QPDFFake1
-WARNING: issue-147.pdf (object 62 0, offset 88): expected endobj
+WARNING: issue-147.pdf: ignoring object with impossibly large id 62
-WARNING: issue-147.pdf (trailer, offset 90): invalid /ID in trailer dictionary
+qpdf: issue-147.pdf: unable to find /Root dictionary
-qpdf: issue-147.pdf: invalid password

qpdf/qtest/qpdf/issue-51.out

@@ -2,15 +2,4 @@ WARNING: issue-51.pdf: can't find PDF header
 WARNING: issue-51.pdf: reported number of objects (0) is not one plus the highest object number (8)
 WARNING: issue-51.pdf (object 7 0, offset 476): dictionary has duplicated key /0000; last occurrence overrides earlier ones
 WARNING: issue-51.pdf (object 7 0, offset 553): expected endobj
-WARNING: issue-51.pdf (object 1 0, offset 236): dictionary has duplicated key /00000000; last occurrence overrides earlier ones
+issue-51.pdf: unable to find page tree
-WARNING: issue-51.pdf (object 1 0, offset 359): expected endobj
-WARNING: issue-51.pdf (offset 70): loop detected resolving object 2 0
-WARNING: issue-51.pdf (object 2 0, offset 26): stream dictionary lacks /Length key
-WARNING: issue-51.pdf (object 2 0, offset 71): attempting to recover stream length
-WARNING: issue-51.pdf (object 2 0, offset 71): unable to recover stream data; treating stream as empty
-WARNING: issue-51.pdf (object 2 0, offset 977): expected endobj
-WARNING: issue-51.pdf (object 3 0): object has offset 0
-WARNING: issue-51.pdf (object 4 0): object has offset 0
-WARNING: issue-51.pdf (object 5 0): object has offset 0
-WARNING: issue-51.pdf (object 6 0): object has offset 0
-WARNING: issue-51.pdf (object 8 0): object has offset 0

qpdf/qtest/qpdf/job-api.out

@@ -21,6 +21,7 @@ captured stderr
 WARNING: bad2.pdf: file is damaged
 WARNING: bad2.pdf: can't find startxref
 WARNING: bad2.pdf: Attempting to reconstruct cross-reference table
+WARNING: bad2.pdf (object 2 0, offset 128): expected endobj
 WARNING: bad2.pdf (object 4 0, offset 389): expected endobj
 qpdf: operation succeeded with warnings
 test 84 done

qpdf/qtest/qpdf/test73.out

@@ -1,3 +1,3 @@
 getRoot: attempted to dereference an uninitialized QPDFObjectHandle
-WARNING: closed input source: object 1/0: error reading object: QPDF operation attempted on a QPDF object with no input source. QPDF operations are invalid before processFile (or another process method) or after closeInputSource
+WARNING: closed input source: object 4/0: error reading object: QPDF operation attempted on a QPDF object with no input source. QPDF operations are invalid before processFile (or another process method) or after closeInputSource
-closed input source: unable to find /Root dictionary
+test 73 done

qpdf/qtest/specific-bugs.test

@@ -16,19 +16,19 @@ my $td = new TestDriver('specific-bugs');
 
 # The number is the github issue number in which the bug was reported.
 my @bug_tests = (
-    ["51", "resolve loop", 3],
+    ["51", "resolve loop", 2],
     ["99", "object 0", 2],
     ["99b", "object 0", 2],
     ["100", "xref reconstruction loop", 2],
     ["101", "resolve for exception text", 2],
     ["117", "other infinite loop", 3],
     ["118", "other infinite loop", 2],
-    ["119", "other infinite loop", 3],
+    ["119", "other infinite loop", 2],
-    ["120", "other infinite loop", 3],
+    ["120", "other infinite loop", 2],
     ["106", "zlib data error", 3],
     ["141a", "/W entry size 0", 2],
     ["141b", "/W entry size 0", 2],
-    ["143", "self-referential ostream", 3, "--preserve-unreferenced"],
+    ["143", "self-referential ostream", 2, "--preserve-unreferenced"],
     ["146", "very deeply nested array", 2],
     ["147", "previously caused memory error", 2],
     ["148", "free memory on bad flate", 2],

qpdf/test_driver.cc

@@ -2496,7 +2496,7 @@ test_73(QPDF& pdf, char const* arg2)
     }
 
     pdf.closeInputSource();
-    pdf.getRoot().getKey("/Pages").unparseResolved();
+    pdf.getObject(4, 0).unparseResolved();
 }
 
 static void