Merge pull request #1340 from m-holger/i1286

Change QPDFWriter stream_decode_level default to qpdf_dl_generalized (fixes #1286)

ChangeLog

@@ -9,6 +9,11 @@
 	* Add C API function qpdf_oh_free_buffer to release memory allocated
 	by stream data functions.
 
+2024-09-19  M Holger  <m.holger@qpdf.org>
+
+	* Bug fix: QPDFWriter stream DecodeLevel incorrectly defaulted to
+	none instead of generalied. Fixes #1286.
+
 2024-08-25  M Holger  <m.holger@qpdf.org>
 
 	* Add new command-line arguments --remove-metadata and --remove-info

libqpdf/QPDFWriter.cc

@@ -26,6 +26,8 @@
 #include <cstdlib>
 #include <stdexcept>
 
+using namespace std::literals;
+
 QPDFWriter::ProgressReporter::~ProgressReporter() // NOLINT (modernize-use-equals-default)
 {
     // Must be explicit and not inline -- see QPDF_DLL_CLASS in README-maintainer
@@ -1249,7 +1251,7 @@ QPDFWriter::willFilterStream(
     if (stream_dict.isDictionaryOfType("/Metadata")) {
         is_metadata = true;
     }
-    bool filter = (stream.isDataModified() || m->compress_streams || m->stream_decode_level);
+    bool filter = stream.isDataModified() || m->compress_streams || m->stream_decode_level;
     bool filter_on_write = stream.getFilterOnWrite();
     if (!filter_on_write) {
         QTC::TC("qpdf", "QPDFWriter getFilterOnWrite false");
@@ -1261,15 +1263,15 @@ QPDFWriter::willFilterStream(
         // CPU cycles uncompressing and recompressing stuff. This can be overridden with
         // setRecompressFlate(true).
         QPDFObjectHandle filter_obj = stream_dict.getKey("/Filter");
-        if ((!m->recompress_flate) && (!stream.isDataModified()) && filter_obj.isName() &&
+        if (!m->recompress_flate && !stream.isDataModified() && filter_obj.isName() &&
-            ((filter_obj.getName() == "/FlateDecode") || (filter_obj.getName() == "/Fl"))) {
+            (filter_obj.getName() == "/FlateDecode" || filter_obj.getName() == "/Fl")) {
             QTC::TC("qpdf", "QPDFWriter not recompressing /FlateDecode");
             filter = false;
         }
     }
     bool normalize = false;
     bool uncompress = false;
-    if (filter_on_write && is_metadata && ((!m->encrypted) || (m->encrypt_metadata == false))) {
+    if (filter_on_write && is_metadata && (!m->encrypted || !m->encrypt_metadata)) {
         QTC::TC("qpdf", "QPDFWriter not compressing metadata");
         filter = true;
         compress_stream = false;
@@ -1283,29 +1285,37 @@ QPDFWriter::willFilterStream(
     }
 
     bool filtered = false;
-    for (int attempt = 1; attempt <= 2; ++attempt) {
+    for (bool first_attempt: {true, false}) {
         pushPipeline(new Pl_Buffer("stream data"));
         PipelinePopper pp_stream_data(this, stream_data);
         activatePipelineStack(pp_stream_data);
         try {
             filtered = stream.pipeStreamData(
                 m->pipeline,
-                (((filter && normalize) ? qpdf_ef_normalize : 0) |
+                !filter ? 0
-                 ((filter && compress_stream) ? qpdf_ef_compress : 0)),
+                        : ((normalize ? qpdf_ef_normalize : 0) |
-                (filter ? (uncompress ? qpdf_dl_all : m->stream_decode_level) : qpdf_dl_none),
+                           (compress_stream ? qpdf_ef_compress : 0)),
+                !filter ? qpdf_dl_none : (uncompress ? qpdf_dl_all : m->stream_decode_level),
                 false,
-                (attempt == 1));
+                first_attempt);
+            if (filter && !filtered) {
+                // Try again
+                filter = false;
+                stream.setFilterOnWrite(false);
+            } else {
+                break;
+            }
         } catch (std::runtime_error& e) {
+            if (filter && first_attempt) {
+                stream.warnIfPossible("error while getting stream data: "s + e.what());
+                stream.warnIfPossible("qpdf will attempt to write the damaged stream unchanged");
+                filter = false;
+                stream.setFilterOnWrite(false);
+                continue;
+            }
             throw std::runtime_error(
                 "error while getting stream data for " + stream.unparse() + ": " + e.what());
         }
-        if (filter && !filtered) {
-            // Try again
-            filter = false;
-            stream.setFilterOnWrite(false);
-        } else {
-            break;
-        }
     }
     if (!filtered) {
         compress_stream = false;
@@ -1866,7 +1876,7 @@ QPDFWriter::generateID()
         if (m->deterministic_id) {
             if (m->deterministic_id_data.empty()) {
                 QTC::TC("qpdf", "QPDFWriter deterministic with no data");
-                throw std::logic_error("INTERNAL ERROR: QPDFWriter::generateID has no data for "
+                throw std::runtime_error("INTERNAL ERROR: QPDFWriter::generateID has no data for "
                                        "deterministic ID.  This may happen if deterministic ID and "
                                        "file encryption are requested together.");
             }
@@ -2116,7 +2126,7 @@ QPDFWriter::doWriteSetup()
     if (m->encrypted) {
         // Encryption has been explicitly set
         m->preserve_encryption = false;
-    } else if (m->normalize_content || m->stream_decode_level || m->pclm || m->qdf_mode) {
+    } else if (m->normalize_content || !m->compress_streams || m->pclm || m->qdf_mode) {
         // Encryption makes looking at contents pretty useless.  If the user explicitly encrypted
         // though, we still obey that.
         m->preserve_encryption = false;

libqpdf/qpdf/QPDFWriter_private.hh

@@ -65,7 +65,7 @@ class QPDFWriter::Members
     bool normalize_content{false};
     bool compress_streams{true};
     bool compress_streams_set{false};
-    qpdf_stream_decode_level_e stream_decode_level{qpdf_dl_none};
+    qpdf_stream_decode_level_e stream_decode_level{qpdf_dl_generalized};
     bool stream_decode_level_set{false};
     bool recompress_flate{false};
     bool qdf_mode{false};

qpdf/qtest/encryption.test

@@ -723,7 +723,7 @@ $td->runtest("check file's validity",
 $td->runtest("handle missing/invalid Length",
              {$td->COMMAND => "qpdf --check bad-encryption-length.pdf"},
              {$td->FILE => "bad-encryption-length.out",
-                  $td->EXIT_STATUS => 0},
+                  $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 
 cleanup();

qpdf/qtest/inline-images.test

@@ -56,7 +56,7 @@ $td->runtest("externalize damaged image",
                   "qpdf --externalize-inline-images" .
                   " --compress-streams=n --static-id" .
                   " damaged-inline-image.pdf a.pdf"},
-             {$td->STRING => "", $td->EXIT_STATUS => 0},
+             {$td->FILE => "damaged-inline-image.out", $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
              {$td->FILE => "a.pdf"},

qpdf/qtest/qpdf/bad-encryption-length.out

@@ -14,5 +14,5 @@ modify annotations: allowed
 modify other: allowed
 modify anything: allowed
 File is not linearized
-No syntax or stream encoding errors found; the file may still contain
+WARNING: bad-encryption-length.pdf, object 7 0 at offset 531 -> dictionary key /Length: operation for integer attempted on object of type null: returning 0
-errors that qpdf cannot detect
+qpdf: operation succeeded with warnings

qpdf/qtest/qpdf/damaged-inline-image.out

@@ -0,0 +1,3 @@
+WARNING: damaged-inline-image.pdf, stream object 8 0: error while getting stream data: stream inflate: inflate: data: invalid distance too far back
+WARNING: damaged-inline-image.pdf, stream object 8 0: qpdf will attempt to write the damaged stream unchanged
+qpdf: operation succeeded with warnings; resulting file may have some problems

qpdf/qtest/qpdf/fuzz-16214.out

@@ -6,6 +6,11 @@ WARNING: fuzz-16214.pdf (xref stream, offset 116): Cross-reference stream data h
 WARNING: fuzz-16214.pdf: reported number of objects (6) is not one plus the highest object number (35)
 WARNING: fuzz-16214.pdf (object 14 0, offset 652): expected dictionary key but found non-name object; inserting key /QPDFFake1
 WARNING: fuzz-16214.pdf (object 14 0, offset 734): expected endobj
+WARNING: fuzz-16214.pdf (object 15 0, offset 869): unknown token while reading object; treating as string
+WARNING: fuzz-16214.pdf (object 15 0, offset 745): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: fuzz-16214.pdf (object 15 0, offset 894): expected endobj
+WARNING: fuzz-16214.pdf, object 15 0 at offset 745: kid 0 (from 0) MediaBox is undefined; setting to letter / ANSI A
+WARNING: fuzz-16214.pdf, object 15 0 at offset 745: /Type key should be /Page but is not; overriding
 WARNING: fuzz-16214.pdf: file is damaged
 WARNING: fuzz-16214.pdf (object 1 0, offset 7189): expected n n obj
 WARNING: fuzz-16214.pdf: Attempting to reconstruct cross-reference table

qpdf/qtest/qpdf/invalid-id-xref.out

@@ -15,4 +15,6 @@ modify annotations: allowed
 modify other: not allowed
 modify anything: not allowed
 File is not linearized
+WARNING: invalid-id-xref.pdf, trailer at offset 910 -> dictionary key /ID: operation for array attempted on object of type null: returning null
+WARNING: invalid-id-xref.pdf, trailer at offset 910 -> dictionary key /ID -> null returned from invalid array access: operation for string attempted on object of type null: returning empty string
 qpdf: operation succeeded with warnings

qpdf/qtest/qpdf/qjson-bad-data.out

@@ -1 +1,3 @@
+WARNING: qjson-bad-data.json, obj:4 0 R at offset 0: error while getting stream data: base64-decode: base64 decode: invalid input
+WARNING: qjson-bad-data.json, obj:4 0 R at offset 0: qpdf will attempt to write the damaged stream unchanged
 qpdf: error while getting stream data for 4 0 R: base64-decode: base64 decode: invalid input

qpdf/qtest/qpdf/qjson-bad-datafile.out

@@ -1 +1,3 @@
+WARNING: qjson-bad-datafile.json, obj:4 0 R at offset 0: error while getting stream data: open file does not exist: No such file or directory
+WARNING: qjson-bad-datafile.json, obj:4 0 R at offset 0: qpdf will attempt to write the damaged stream unchanged
 qpdf: error while getting stream data for 4 0 R: open file does not exist: No such file or directory