Fix tests when running comparison tool is adequate

Fix all tests where the only required fix is to run qpdf-test-compare
instead of doing a simple file comparison.

examples/CMakeLists.txt

@@ -45,6 +45,7 @@ add_test(
   --top ${qpdf_SOURCE_DIR}
   --bin $<TARGET_FILE_DIR:pdf-create>
   --bin $<TARGET_FILE_DIR:qpdf>
+  --bin $<TARGET_FILE_DIR:qpdf-test-compare>
   --bin $<TARGET_FILE_DIR:libqpdf>  # for Windows to find DLL
   --code ${qpdf_SOURCE_DIR}/examples
   --color ${QTEST_COLOR}

examples/qtest/c-objects.test

@@ -17,8 +17,8 @@ foreach my $i (qw(1 2))
                  {$td->COMMAND => "pdf-c-objects $i.pdf '' a.pdf"},
                  {$td->STRING => "", $td->EXIT_STATUS => 0});
     $td->runtest("check output",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf $i-out.pdf"},
-                 {$td->FILE => "$i-out.pdf"});
+                 {$td->FILE => "$i-out.pdf", $td->EXIT_STATUS => 0});
 }
 
 cleanup();

examples/qtest/custom-filter.test

@@ -24,8 +24,8 @@ $td->runtest("custom filter, decode generalized",
                   $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf generalized.pdf"},
-             {$td->FILE => "generalized.pdf"});
+             {$td->FILE => "generalized.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("custom filter, decode specialized",
              {$td->COMMAND =>
@@ -34,8 +34,8 @@ $td->runtest("custom filter, decode specialized",
                   $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf specialized.pdf"},
-             {$td->FILE => "specialized.pdf"});
+             {$td->FILE => "specialized.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 

examples/qtest/invert-images.test

@@ -20,8 +20,8 @@ $td->runtest("invert images",
              $td->NORMALIZE_NEWLINES);
 
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf out.pdf"},
-             {$td->FILE => "out.pdf"});
+             {$td->FILE => "out.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 

examples/qtest/overlay-page.test

@@ -15,15 +15,15 @@ $td->runtest("overlay-page",
              {$td->COMMAND => "pdf-overlay-page in.pdf stamp.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("compare files",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf out.pdf"},
-             {$td->FILE => "out.pdf"});
+             {$td->FILE => "out.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("overlay-page with fields/ annotations",
              {$td->COMMAND => "pdf-overlay-page in.pdf annotations.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("compare files",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf annotations-out.pdf"},
-             {$td->FILE => "annotations-out.pdf"});
+             {$td->FILE => "annotations-out.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 

examples/qtest/qpdfjob-remove-annotations.test

@@ -15,8 +15,8 @@ $td->runtest("remove-annotations",
              {$td->COMMAND => "qpdfjob-remove-annotations --static-id annotations.pdf out.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("compare files",
-             {$td->FILE => "out.pdf"},
+             {$td->COMMAND => "qpdf-test-compare out.pdf annotations-out.pdf"},
-             {$td->FILE => "annotations-out.pdf"});
+             {$td->FILE => "annotations-out.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 

examples/qtest/set-form-values.test

@@ -15,8 +15,8 @@ $td->runtest("set form values",
              {$td->COMMAND => "pdf-set-form-values form-in.pdf a.pdf soup"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("compare files",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf form-out.pdf"},
-             {$td->FILE => "form-out.pdf"});
+             {$td->FILE => "form-out.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 

qpdf/CMakeLists.txt

@@ -55,6 +55,7 @@ add_test(
   --top ${qpdf_SOURCE_DIR}
   --bin $<TARGET_FILE_DIR:qpdf>
   --bin $<TARGET_FILE_DIR:libqpdf>  # for Windows to find DLL
+  --bin $<TARGET_FILE_DIR:qpdf-test-compare>
   --code ${qpdf_SOURCE_DIR}/qpdf
   --color ${QTEST_COLOR}
   --show-on-failure ${SHOW_FAILED_TEST_OUTPUT}

qpdf/qtest/attachments.test

@@ -207,8 +207,8 @@ $td->runtest("remove multiple attachments",
                   $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf remove-multiple-attachments.pdf"},
-             {$td->FILE => "remove-multiple-attachments.pdf"});
+             {$td->FILE => "remove-multiple-attachments.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("remove multiple attachments (json)",
              {$td->COMMAND =>
                   "qpdf --job-json-file=remove-multiple-attachments.json"},
@@ -216,8 +216,8 @@ $td->runtest("remove multiple attachments (json)",
                   $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "b.pdf"},
+             {$td->COMMAND => "qpdf-test-compare b.pdf remove-multiple-attachments.pdf"},
-             {$td->FILE => "remove-multiple-attachments.pdf"});
+             {$td->FILE => "remove-multiple-attachments.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 $td->report($n_tests);

qpdf/qtest/c-api.test

@@ -36,8 +36,8 @@ foreach my $d (@capi)
                  {$td->STRING => "C test $n done\n", $td->EXIT_STATUS => 0},
                  $td->NORMALIZE_NEWLINES);
     $td->runtest("check $description",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf $outfile"},
-                 {$td->FILE => $outfile});
+                 {$td->FILE => $outfile, $td->EXIT_STATUS => 0});
 }
 $td->runtest("write to bad file name",
              {$td->COMMAND => "qpdf-ctest 2 hybrid-xref.pdf '' /:a:/:b:"},

qpdf/qtest/coalesce-contents.test

@@ -46,8 +46,8 @@ $td->runtest("coalesce contents without qdf",
                   " --coalesce-contents coalesce.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf coalesce-out.pdf"},
-             {$td->FILE => "coalesce-out.pdf"});
+             {$td->FILE => "coalesce-out.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 $td->report($n_tests);

qpdf/qtest/copy-annotations.test

@@ -110,8 +110,8 @@ $td->runtest("keeping some fields",
              {$td->STRING => "", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf kept-some-fields.pdf"},
-             {$td->FILE => "kept-some-fields.pdf"});
+             {$td->FILE => "kept-some-fields.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("not keeping any fields",
              {$td->COMMAND =>
                   "qpdf --static-id kept-some-fields.pdf" .

qpdf/qtest/custom-pipeline.test

@@ -21,8 +21,8 @@ $td->runtest("output to custom pipeline",
              {$td->STRING => "test 33 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf custom-pipeline.pdf"},
-             {$td->FILE => "custom-pipeline.pdf"});
+             {$td->FILE => "custom-pipeline.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 $td->report($n_tests);

qpdf/qtest/dangling-refs.test

@@ -24,8 +24,8 @@ foreach my $f (@dangling)
                  {$td->FILE => "$f-dangling.out", $td->EXIT_STATUS => 0},
                  $td->NORMALIZE_NEWLINES);
     $td->runtest("check output",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf $f-dangling-out.pdf"},
-                 {$td->FILE => "$f-dangling-out.pdf"});
+                 {$td->FILE => "$f-dangling-out.pdf", $td->EXIT_STATUS => 0});
 }
 cleanup();
 $td->report($n_tests);

qpdf/qtest/decode-parameters.test

@@ -34,8 +34,8 @@ $td->runtest("stream with indirect decode parms",
                   "qpdf --static-id indirect-decode-parms.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check file",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf indirect-decode-parms-out.pdf"},
-             {$td->FILE => "indirect-decode-parms-out.pdf"});
+             {$td->FILE => "indirect-decode-parms-out.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("decode parameters empty list",
              {$td->COMMAND => "qpdf --static-id empty-decode-parms.pdf a.pdf"},

qpdf/qtest/encryption.test

@@ -404,8 +404,8 @@ foreach my $d (@cenc)
     if (-f $pdf_outfile)
     {
         $td->runtest("check $description content",
-                     {$td->FILE => "a.pdf"},
+                     {$td->COMMAND => "qpdf-test-compare a.pdf $pdf_outfile"},
-                     {$td->FILE => $pdf_outfile});
+                     {$td->FILE => $pdf_outfile, $td->EXIT_STATUS => 0});
     }
     else
     {
@@ -491,8 +491,8 @@ $td->runtest("convert encrypted to qdf",
                   " --qdf a.pdf b.qdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("compare files",
-             {$td->FILE => 'a.qdf'},
+             {$td->COMMAND => "qpdf-test-compare a.qdf b.qdf"},
-             {$td->FILE => 'b.qdf'});
+             {$td->FILE => 'b.qdf', $td->EXIT_STATUS => 0});
 $td->runtest("linearize with AES and object streams",
              {$td->COMMAND => "qpdf --encrypt '' o 128 --use-aes=y --" .
                   " --linearize --object-streams=generate enc-base.pdf a.pdf"},
@@ -564,8 +564,8 @@ foreach my $d (['--force-V4', 'V4'],
                       " enc-base.pdf a.pdf"},
                  {$td->STRING => "", $td->EXIT_STATUS => 0});
     $td->runtest("check output",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf $out.pdf"},
-                 {$td->FILE => "$out.pdf"});
+                 {$td->FILE => "$out.pdf", $td->EXIT_STATUS => 0});
     $td->runtest("show encryption",
                  {$td->COMMAND => "qpdf --show-encryption a.pdf"},
                  {$td->FILE => "$out-encryption.out", $td->EXIT_STATUS => 0},

qpdf/qtest/extensions-dictionary.test

@@ -46,16 +46,16 @@ foreach my $input (@ext_inputs)
                 # Look at the actual file for a few cases to make sure
                 # qdf and non-qdf output are okay
                 $td->runtest("check file",
-                             {$td->FILE => "a.pdf"},
+                             {$td->COMMAND => "qpdf-test-compare a.pdf $base-$op-$version.pdf"},
-                             {$td->FILE => "$base-$op-$version.pdf"});
+                             {$td->FILE => "$base-$op-$version.pdf", $td->EXIT_STATUS => 0});
                 $td->runtest("$input: $op version to $version",
                              {$td->COMMAND =>
                                   "qpdf --qdf --static-id" .
                                   " --$op-version=$version $input a.qdf"},
                              {$td->STRING => "", $td->EXIT_STATUS => 0});
                 $td->runtest("check file",
-                             {$td->FILE => "a.qdf"},
+                             {$td->COMMAND => "qpdf-test-compare a.qdf $base-$op-$version.qdf"},
-                             {$td->FILE => "$base-$op-$version.qdf"});
+                             {$td->FILE => "$base-$op-$version.qdf", $td->EXIT_STATUS => 0});
             }
         }
     }

qpdf/qtest/filter-abbreviations.test

@@ -23,8 +23,8 @@ $td->runtest("stream filter abbreviations",
               $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf filter-abbreviation.out"},
-             {$td->FILE => "filter-abbreviation.out"});
+             {$td->FILE => "filter-abbreviation.out", $td->EXIT_STATUS => 0});
 
 cleanup();
 $td->report($n_tests);

qpdf/qtest/inline-images.test

@@ -68,8 +68,8 @@ $td->runtest("named colorspace",
              {$td->STRING => "", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf inline-image-colorspace-lookup-out.pdf"},
-             {$td->FILE => "inline-image-colorspace-lookup-out.pdf"});
+             {$td->FILE => "inline-image-colorspace-lookup-out.pdf", $td->EXIT_STATUS => 0});
 
 
 my @eii_tests = (

qpdf/qtest/merge-and-split.test

@@ -39,8 +39,8 @@ $td->runtest("merge three files",
 # as well as 20-pages have text on page n (from 1) that shows its page
 # position from 0, so page 1 says it's page 0.
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf merge-three-files-1.pdf"},
-             {$td->FILE => "merge-three-files-1.pdf"});
+             {$td->FILE => "merge-three-files-1.pdf", $td->EXIT_STATUS => 0});
 # Select the same pages but add them to an empty file
 $td->runtest("merge three files",
              {$td->COMMAND => "qpdf --empty a.pdf" .
@@ -49,8 +49,8 @@ $td->runtest("merge three files",
 # Manually verified about this file: it has the same pages but does
 # not contain outlines or other things from the original file.
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf merge-three-files-2.pdf"},
-             {$td->FILE => "merge-three-files-2.pdf"});
+             {$td->FILE => "merge-three-files-2.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("avoid respecification of password",
              {$td->COMMAND =>
                   "qpdf --empty a.pdf --copy-encryption=20-pages.pdf" .
@@ -69,16 +69,16 @@ $td->runtest("merge with implicit ranges",
                   " --static-id"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf merge-implicit-ranges.pdf"},
-             {$td->FILE => "merge-implicit-ranges.pdf"});
+             {$td->FILE => "merge-implicit-ranges.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("merge with . and implicit ranges",
              {$td->COMMAND =>
                   "qpdf minimal.pdf a.pdf --pages minimal.pdf . 1 --" .
                   " --static-id"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf merge-dot-implicit-ranges.pdf"},
-             {$td->FILE => "merge-dot-implicit-ranges.pdf"});
+             {$td->FILE => "merge-dot-implicit-ranges.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("merge with multiple labels",
              {$td->COMMAND =>
                   "qpdf --empty a.pdf" .
@@ -88,8 +88,8 @@ $td->runtest("merge with multiple labels",
                   " --static-id"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf merge-multiple-labels.pdf"},
-             {$td->FILE => "merge-multiple-labels.pdf"});
+             {$td->FILE => "merge-multiple-labels.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("remove labels",
              {$td->COMMAND =>
                   "qpdf --empty a.pdf" .
@@ -100,8 +100,8 @@ $td->runtest("remove labels",
                   " --static-id"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf remove-labels.pdf"},
-             {$td->FILE => "remove-labels.pdf"});
+             {$td->FILE => "remove-labels.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("split with shared resources",
              {$td->COMMAND =>
@@ -178,8 +178,8 @@ $td->runtest("force full page duplication",
              {$td->STRING => "", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf deep-duplicate-pages.pdf"},
-             {$td->FILE => "deep-duplicate-pages.pdf"});
+             {$td->FILE => "deep-duplicate-pages.pdf", $td->EXIT_STATUS => 0});
 
 
 cleanup();

qpdf/qtest/newline-before-endstream.test

@@ -36,8 +36,8 @@ foreach my $d (
                  {$td->STRING => "", $td->EXIT_STATUS => 0},
                  $td->NORMALIZE_NEWLINES);
     $td->runtest("check output ($description)",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf newline-before-endstream-$suffix.pdf"},
-                 {$td->FILE => "newline-before-endstream-$suffix.pdf"});
+                 {$td->FILE => "newline-before-endstream-$suffix.pdf", $td->EXIT_STATUS => 0});
     if ($flags =~ /qdf/)
     {
         $td->runtest("fix-qdf",
@@ -52,8 +52,8 @@ $td->runtest("newline before endstream (C)",
              {$td->STRING => "C test 22 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf newline-before-endstream-nl.pdf"},
-             {$td->FILE => "newline-before-endstream-nl.pdf"});
+             {$td->FILE => "newline-before-endstream-nl.pdf", $td->EXIT_STATUS => 0});
 
 cleanup();
 $td->report($n_tests);

qpdf/qtest/page-errors.test

@@ -32,8 +32,8 @@ $td->runtest("handle page with inherited MediaBox",
              {$td->STRING => "", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf page-inherit-mediabox-out.pdf"},
-             {$td->FILE => "page-inherit-mediabox-out.pdf"});
+             {$td->FILE => "page-inherit-mediabox-out.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("check no type key for page nodes",
              {$td->COMMAND => "qpdf --check no-pages-types.pdf"},
              {$td->FILE => "no-pages-types.out", $td->EXIT_STATUS => 3},
@@ -43,8 +43,8 @@ $td->runtest("no type key for page nodes",
              {$td->FILE => "no-pages-types-fix.out", $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a-split-out-1.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a-split-out-1.pdf no-pages-types-fixed.pdf"},
-             {$td->FILE => "no-pages-types-fixed.pdf"});
+             {$td->FILE => "no-pages-types-fixed.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("detect loops in pages structure",
              {$td->COMMAND => "qpdf --check pages-loop.pdf"},
              {$td->FILE => "pages-loop.out", $td->EXIT_STATUS => 2},

qpdf/qtest/pages-tree.test

@@ -24,8 +24,8 @@ $td->runtest("linearize duplicated pages",
                   $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("compare files",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf"},
-             {$td->FILE => "linearize-duplicate-page.pdf"});
+             {$td->FILE => "linearize-duplicate-page.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("extract duplicated pages",
              {$td->COMMAND =>
                   "qpdf --static-id page_api_2.pdf" .
@@ -34,16 +34,16 @@ $td->runtest("extract duplicated pages",
                   $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("compare files",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf extract-duplicate-page.pdf"},
-             {$td->FILE => "extract-duplicate-page.pdf"});
+             {$td->FILE => "extract-duplicate-page.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("direct pages",
              {$td->COMMAND =>
                   "qpdf --static-id direct-pages.pdf --pages . -- a.pdf"},
              {$td->FILE => "direct-page-warning.out", $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf direct-pages-fixed.pdf"},
-             {$td->FILE => "direct-pages-fixed.pdf"});
+             {$td->FILE => "direct-pages-fixed.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("show direct pages",
              {$td->COMMAND =>
                   "qpdf --show-pages direct-pages.pdf"},

qpdf/qtest/parsing.test

@@ -33,14 +33,14 @@ $td->runtest("extra header text",
              {$td->FILE => "test-32.out", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf extra-header-no-newline.pdf"},
-             {$td->FILE => "extra-header-no-newline.pdf"});
+             {$td->FILE => "extra-header-no-newline.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
              {$td->FILE => "b.pdf"},
              {$td->FILE => "extra-header-lin-no-newline.pdf"});
 $td->runtest("check output",
-             {$td->FILE => "c.pdf"},
+             {$td->COMMAND => "qpdf-test-compare c.pdf extra-header-newline.pdf"},
-             {$td->FILE => "extra-header-newline.pdf"});
+             {$td->FILE => "extra-header-newline.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
              {$td->FILE => "d.pdf"},
              {$td->FILE => "extra-header-lin-newline.pdf"});

qpdf/qtest/preserve-unref.test

@@ -20,22 +20,22 @@ $td->runtest("drop unused objects",
              {$td->COMMAND => "qpdf --static-id unreferenced-objects.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf unreferenced-dropped.pdf"},
-             {$td->FILE => "unreferenced-dropped.pdf"});
+             {$td->FILE => "unreferenced-dropped.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("keep unused objects",
              {$td->COMMAND => "qpdf --static-id --preserve-unreferenced" .
                   " unreferenced-objects.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf unreferenced-preserved.pdf"},
-             {$td->FILE => "unreferenced-preserved.pdf"});
+             {$td->FILE => "unreferenced-preserved.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("keep unused objects (C)",
              {$td->COMMAND =>
                   "qpdf-ctest 21 unreferenced-objects.pdf '' a.pdf"},
              {$td->STRING => "C test 21 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf unreferenced-preserved.pdf"},
-             {$td->FILE => "unreferenced-preserved.pdf"});
+             {$td->FILE => "unreferenced-preserved.pdf", $td->EXIT_STATUS => 0});
 cleanup();
 $td->report($n_tests);

qpdf/qtest/qpdf-json.test

@@ -294,31 +294,31 @@ $td->runtest("C API create from json file",
              {$td->STRING => "C test 42 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check C API create from file",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf qpdf-ctest-42-43.pdf"},
-             {$td->FILE => "qpdf-ctest-42-43.pdf"});
+             {$td->FILE => "qpdf-ctest-42-43.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("C API create from json buffer",
              {$td->COMMAND => "qpdf-ctest 43 minimal.json '' a.pdf"},
              {$td->STRING => "C test 43 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check C API create from buffer",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf qpdf-ctest-42-43.pdf"},
-             {$td->FILE => "qpdf-ctest-42-43.pdf"});
+             {$td->FILE => "qpdf-ctest-42-43.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("C API update from json file",
              {$td->COMMAND =>
                   "qpdf-ctest 44 minimal.pdf '' a.pdf minimal-update.json"},
              {$td->STRING => "C test 44 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check C API update from file",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf qpdf-ctest-44-45.pdf"},
-             {$td->FILE => "qpdf-ctest-44-45.pdf"});
+             {$td->FILE => "qpdf-ctest-44-45.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("C API update from json buffer",
              {$td->COMMAND =>
                   "qpdf-ctest 45 minimal.pdf '' a.pdf minimal-update.json"},
              {$td->STRING => "C test 45 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check C API update from buffer",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf qpdf-ctest-44-45.pdf"},
-             {$td->FILE => "qpdf-ctest-44-45.pdf"});
+             {$td->FILE => "qpdf-ctest-44-45.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("C API write to JSON 1",
              {$td->COMMAND =>
                   "qpdf-ctest 46 minimal.pdf '' a.json"},

qpdf/qtest/qpdf_test_helpers.pm

@@ -46,8 +46,8 @@ sub check_pdf
                  {$td->STRING => "",
                   $td->EXIT_STATUS => $status});
     $td->runtest("check output",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf $output"},
-                 {$td->FILE => $output});
+                 {$td->FILE => $output, $td->EXIT_STATUS => 0});
 }
 
 sub flush_tiff_cache

qpdf/qtest/qpdfjob.test

@@ -76,8 +76,8 @@ foreach my $i (@good_json)
     else
     {
         $td->runtest("check good json $i output",
-                     {$td->FILE => "a.pdf"},
+                     {$td->COMMAND => "qpdf-test-compare a.pdf job-json-$i.pdf"},
-                     {$td->FILE => "job-json-$i.pdf"});
+                     {$td->FILE => "job-json-$i.pdf", $td->EXIT_STATUS => 0});
     }
 }
 
@@ -107,8 +107,8 @@ $td->runtest("C job API",
 foreach my $i (['a.pdf', 1], ['b.pdf', 2], ['c.pdf', 3], ['d.pdf', 4])
 {
     $td->runtest("check output",
-                 {$td->FILE => $i->[0]},
+                 {$td->COMMAND => "qpdf-test-compare $i->[0] qpdfjob-ctest$i->[1].pdf"},
-                 {$td->FILE => "qpdfjob-ctest$i->[1].pdf"});
+                 {$td->FILE => "qpdfjob-ctest$i->[1].pdf", $td->EXIT_STATUS => 0});
 }
 my $wide_out = `qpdfjob-ctest wide`;
 $td->runtest("qpdfjob-ctest wide",
@@ -124,8 +124,8 @@ if ($wide_out =~ m/skipped/)
 else
 {
     $td->runtest("check output",
-                 {$td->FILE => "a.pdf"},
+                 {$td->COMMAND => "qpdf-test-compare a.pdf qpdfjob-ctest-wide.pdf"},
-                 {$td->FILE => "qpdfjob-ctest-wide.pdf"});
+                 {$td->FILE => "qpdfjob-ctest-wide.pdf", $td->EXIT_STATUS => 0});
 }
 
 cleanup();

qpdf/qtest/rotate-pages.test

@@ -29,24 +29,24 @@ $td->runtest("page rotation",
                   " --rotate=-90:3,15,17,18"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf rotated.pdf"},
-             {$td->FILE => "rotated.pdf"});
+             {$td->FILE => "rotated.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("remove rotation",
              {$td->COMMAND => "qpdf --static-id rotated.pdf a.pdf" .
                   " --qdf --no-original-object-ids --rotate=0"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf unrotated.pdf"},
-             {$td->FILE => "unrotated.pdf"});
+             {$td->FILE => "unrotated.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("rotate all pages",
              {$td->COMMAND =>
                   "qpdf --static-id --rotate=180 minimal.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf minimal-rotated.pdf"},
-             {$td->FILE => "minimal-rotated.pdf"});
+             {$td->FILE => "minimal-rotated.pdf", $td->EXIT_STATUS => 0});
 
 $td->runtest("flatten with inherited rotate",
              {$td->COMMAND =>
@@ -54,8 +54,8 @@ $td->runtest("flatten with inherited rotate",
                   " inherited-rotate.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf inherited-flattened.pdf"},
-             {$td->FILE => "inherited-flattened.pdf"});
+             {$td->FILE => "inherited-flattened.pdf", $td->EXIT_STATUS => 0});
 
 foreach my $angle (qw(90 180 270))
 {

qpdf/qtest/specific-file.test

@@ -33,15 +33,15 @@ $td->runtest("compress objstm and xref",
              {$td->STRING => "", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf compress-objstm-xref.pdf"},
-             {$td->FILE => "compress-objstm-xref.pdf"});
+             {$td->FILE => "compress-objstm-xref.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("qdf + preserved-unreferenced + xref streams",
              {$td->COMMAND => "qpdf --qdf --preserve-unreferenced" .
                   " --static-id compress-objstm-xref.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf compress-objstm-xref-qdf.pdf"},
-             {$td->FILE => "compress-objstm-xref-qdf.pdf"});
+             {$td->FILE => "compress-objstm-xref-qdf.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("check fix-qdf idempotency",
              {$td->COMMAND => "fix-qdf a.pdf"},
              {$td->FILE => "a.pdf", $td->EXIT_STATUS => 0});

qpdf/qtest/stream-replacements.test

@@ -21,8 +21,8 @@ $td->runtest("replace stream data",
              {$td->STRING => "test 7 done\n", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
-             {$td->FILE => "a.pdf"},
+             {$td->COMMAND => "qpdf-test-compare a.pdf replaced-stream-data.pdf"},
-             {$td->FILE => "replaced-stream-data.pdf"});
+             {$td->FILE => "replaced-stream-data.pdf", $td->EXIT_STATUS => 0});
 $td->runtest("replace stream data compressed",
              {$td->COMMAND => "test_driver 8 qstream.pdf"},
              {$td->FILE => "test8.out", $td->EXIT_STATUS => 0},