Honor repeated overlay/underlay

ChangeLog

@@ -1,3 +1,11 @@
+2024-01-10  Jay Berkenbilt  <ejb@ql.org>
+
+	* Allow --overlay and --underlay to be repeated. They may appear
+	multiple times on the command-line and will be stacked in the
+	order in which they appear. In QPDFJob JSON, the overlay and
+	underlay keys may contain arrays. For compatibility, they may also
+	contain a single dictionary.
+
 2024-01-09  Jay Berkenbilt  <ejb@ql.org>
 
 	* Add new command-line arguments --file and --range which can be

include/qpdf/QPDFJob.hh

@@ -514,14 +514,16 @@ class QPDFJob
     void handlePageSpecs(QPDF& pdf, std::vector<std::unique_ptr<QPDF>>& page_heap);
     bool shouldRemoveUnreferencedResources(QPDF& pdf);
     void handleRotations(QPDF& pdf);
-    void getUOPagenos(UnderOverlay& uo, std::map<int, std::vector<int>>& pagenos);
+    void getUOPagenos(
+        std::vector<UnderOverlay>& uo, std::map<int, std::map<size_t, std::vector<int>>>& pagenos);
     void handleUnderOverlay(QPDF& pdf);
     std::string doUnderOverlayForPage(
         QPDF& pdf,
         UnderOverlay& uo,
-        std::map<int, std::vector<int>>& pagenos,
+        std::map<int, std::map<size_t, std::vector<int>>>& pagenos,
         size_t page_idx,
-        std::map<int, QPDFObjectHandle>& fo,
+        size_t uo_idx,
+        std::map<int, std::map<size_t, QPDFObjectHandle>>& fo,
         std::vector<QPDFPageObjectHelper>& pages,
         QPDFPageObjectHelper& dest_page);
     void validateUnderOverlay(QPDF& pdf, UnderOverlay* uo);
@@ -696,8 +698,8 @@ class QPDFJob
         size_t oi_min_height{DEFAULT_OI_MIN_HEIGHT};
         size_t oi_min_area{DEFAULT_OI_MIN_AREA};
         size_t ii_min_bytes{DEFAULT_II_MIN_BYTES};
-        UnderOverlay underlay{"underlay"};
+        std::vector<UnderOverlay> underlay;
-        UnderOverlay overlay{"overlay"};
+        std::vector<UnderOverlay> overlay;
         UnderOverlay* under_overlay{nullptr};
         std::vector<PageSpec> page_specs;
         std::map<std::string, RotationSpec> rotations;

job.sums

@@ -9,12 +9,12 @@ include/qpdf/auto_job_c_pages.hh 09ca15649cc94fdaf6d9bdae28a20723f2a66616bf15aa8
 include/qpdf/auto_job_c_uo.hh 9c2f98a355858dd54d0bba444b73177a59c9e56833e02fa6406f429c07f39e62
 job.yml 53cad86659db6722e8f415aacb19fc51ab81bb1589c3cb8f65ec893bb4bf5566
 libqpdf/qpdf/auto_job_decl.hh 20d6affe1e260f5a1af4f1d82a820b933835440ff03020e877382da2e8dac6c6
-libqpdf/qpdf/auto_job_help.hh 5808d936f6cd41af278ca298ed0c0762ce0a16956cbe1757a40e4443485cf31e
+libqpdf/qpdf/auto_job_help.hh e4bb9e097516f35b4dbc676e1de99f294d8f42912541c8e3844ea401e44336ef
 libqpdf/qpdf/auto_job_init.hh 19d1da7c4c0c635bd1c5db8d5f17df8edad3442f8eba006adb075cec295fa158
 libqpdf/qpdf/auto_job_json_decl.hh 843892c8e8652a86b7eb573893ef24050b7f36fe313f7251874be5cd4cdbe3fd
 libqpdf/qpdf/auto_job_json_init.hh a87256c082427ec0318223762472970b2eced535c0c8b0288d45c8cdaaf62f74
 libqpdf/qpdf/auto_job_schema.hh 5dac568dff39614e161a0af59a0f328f1e28edf69b96f08bb76fd592d51bb053
 manual/_ext/qpdf.py 6add6321666031d55ed4aedf7c00e5662bba856dfcd66ccb526563bffefbb580
-manual/cli.rst 0e6a957defa4839abb9a69414de6a5ec5524fd6ff56fe9abf8f241bee54813e2
+manual/cli.rst 98219ac9942824b78119cca7cd75691f7c98a31ed3c8b4f108d60a699087c418
-manual/qpdf.1 7250b4e26033fca6b6b9cb23a51e1f46c26f8033663901d4af06b451e287e814
+manual/qpdf.1 2544e085c5f0f92e242944eea3bc5736e1036f67595a7a7c988f4ea8d75da901
 manual/qpdf.1.in 436ecc85d45c4c9e2dbd1725fb7f0177fb627179469f114561adf3cb6cbb677b

libqpdf/QPDFJob.cc

@@ -1834,9 +1834,6 @@ QPDFJob::processInputSource(
 void
 QPDFJob::validateUnderOverlay(QPDF& pdf, UnderOverlay* uo)
 {
-    if (uo->filename.empty()) {
-        return;
-    }
     QPDFPageDocumentHelper main_pdh(pdf);
     int main_npages = QIntC::to_int(main_pdh.getAllPages().size());
     processFile(uo->pdf, uo->filename.c_str(), uo->password.get(), true, false);
@@ -1878,14 +1875,15 @@ std::string
 QPDFJob::doUnderOverlayForPage(
     QPDF& pdf,
     UnderOverlay& uo,
-    std::map<int, std::vector<int>>& pagenos,
+    std::map<int, std::map<size_t, std::vector<int>>>& pagenos,
     size_t page_idx,
-    std::map<int, QPDFObjectHandle>& fo,
+    size_t uo_idx,
+    std::map<int, std::map<size_t, QPDFObjectHandle>>& fo,
     std::vector<QPDFPageObjectHelper>& pages,
     QPDFPageObjectHelper& dest_page)
 {
     int pageno = 1 + QIntC::to_int(page_idx);
-    if (!pagenos.count(pageno)) {
+    if (!(pagenos.count(pageno) && pagenos[pageno].count(uo_idx))) {
         return "";
     }
 
@@ -1899,13 +1897,13 @@ QPDFJob::doUnderOverlayForPage(
     std::string content;
     int min_suffix = 1;
     QPDFObjectHandle resources = dest_page.getAttribute("/Resources", true);
-    for (int from_pageno: pagenos[pageno]) {
+    for (int from_pageno: pagenos[pageno][uo_idx]) {
         doIfVerbose([&](Pipeline& v, std::string const& prefix) {
             v << "    " << uo.which << " " << from_pageno << "\n";
         });
         auto from_page = pages.at(QIntC::to_size(from_pageno - 1));
-        if (0 == fo.count(from_pageno)) {
+        if (fo[from_pageno].count(uo_idx) == 0) {
-            fo[from_pageno] = pdf.copyForeignObject(from_page.getFormXObjectForPage());
+            fo[from_pageno][uo_idx] = pdf.copyForeignObject(from_page.getFormXObjectForPage());
         }
 
         // If the same page is overlaid or underlaid multiple times, we'll generate multiple names
@@ -1913,13 +1911,13 @@ QPDFJob::doUnderOverlayForPage(
         std::string name = resources.getUniqueResourceName("/Fx", min_suffix);
         QPDFMatrix cm;
         std::string new_content = dest_page.placeFormXObject(
-            fo[from_pageno], name, dest_page.getTrimBox().getArrayAsRectangle(), cm);
+            fo[from_pageno][uo_idx], name, dest_page.getTrimBox().getArrayAsRectangle(), cm);
         dest_page.copyAnnotations(from_page, cm, dest_afdh, make_afdh(from_page));
         if (!new_content.empty()) {
             resources.mergeResources("<< /XObject << >> >>"_qpdf);
             auto xobject = resources.getKey("/XObject");
             if (xobject.isDictionary()) {
-                xobject.replaceKey(name, fo[from_pageno]);
+                xobject.replaceKey(name, fo[from_pageno][uo_idx]);
             }
             ++min_suffix;
             content += new_content;
@@ -1929,73 +1927,104 @@ QPDFJob::doUnderOverlayForPage(
 }
 
 void
-QPDFJob::getUOPagenos(QPDFJob::UnderOverlay& uo, std::map<int, std::vector<int>>& pagenos)
+QPDFJob::getUOPagenos(
+    std::vector<QPDFJob::UnderOverlay>& uos,
+    std::map<int, std::map<size_t, std::vector<int>>>& pagenos)
 {
-    size_t idx = 0;
+    size_t uo_idx = 0;
-    size_t from_size = uo.from_pagenos.size();
+    for (auto const& uo: uos) {
-    size_t repeat_size = uo.repeat_pagenos.size();
+        size_t page_idx = 0;
-    for (int to_pageno: uo.to_pagenos) {
+        size_t from_size = uo.from_pagenos.size();
-        if (idx < from_size) {
+        size_t repeat_size = uo.repeat_pagenos.size();
-            pagenos[to_pageno].push_back(uo.from_pagenos.at(idx));
+        for (int to_pageno: uo.to_pagenos) {
-        } else if (repeat_size) {
+            if (page_idx < from_size) {
-            pagenos[to_pageno].push_back(uo.repeat_pagenos.at((idx - from_size) % repeat_size));
+                pagenos[to_pageno][uo_idx].push_back(uo.from_pagenos.at(page_idx));
+            } else if (repeat_size) {
+                pagenos[to_pageno][uo_idx].push_back(
+                    uo.repeat_pagenos.at((page_idx - from_size) % repeat_size));
+            }
+            ++page_idx;
         }
-        ++idx;
+        ++uo_idx;
     }
 }
 
 void
 QPDFJob::handleUnderOverlay(QPDF& pdf)
 {
-    validateUnderOverlay(pdf, &m->underlay);
+    if (m->underlay.empty() && m->overlay.empty()) {
-    validateUnderOverlay(pdf, &m->overlay);
-    if ((nullptr == m->underlay.pdf) && (nullptr == m->overlay.pdf)) {
         return;
     }
-    std::map<int, std::vector<int>> underlay_pagenos;
+    for (auto& uo: m->underlay) {
-    getUOPagenos(m->underlay, underlay_pagenos);
+        validateUnderOverlay(pdf, &uo);
-    std::map<int, std::vector<int>> overlay_pagenos;
-    getUOPagenos(m->overlay, overlay_pagenos);
-    std::map<int, QPDFObjectHandle> underlay_fo;
-    std::map<int, QPDFObjectHandle> overlay_fo;
-    std::vector<QPDFPageObjectHelper> upages;
-    if (m->underlay.pdf.get()) {
-        upages = QPDFPageDocumentHelper(*(m->underlay.pdf)).getAllPages();
     }
-    std::vector<QPDFPageObjectHelper> opages;
+    for (auto& uo: m->overlay) {
-    if (m->overlay.pdf.get()) {
+        validateUnderOverlay(pdf, &uo);
-        opages = QPDFPageDocumentHelper(*(m->overlay.pdf)).getAllPages();
     }
 
-    QPDFPageDocumentHelper main_pdh(pdf);
+    // First map key is 1-based page number. Second is index into the overlay/underlay vector. Watch
-    std::vector<QPDFPageObjectHelper> main_pages = main_pdh.getAllPages();
+    // out to not reverse the keys or be off by one.
-    size_t main_npages = main_pages.size();
+    std::map<int, std::map<size_t, std::vector<int>>> underlay_pagenos;
+    std::map<int, std::map<size_t, std::vector<int>>> overlay_pagenos;
+    getUOPagenos(m->underlay, underlay_pagenos);
+    getUOPagenos(m->overlay, overlay_pagenos);
     doIfVerbose([&](Pipeline& v, std::string const& prefix) {
         v << prefix << ": processing underlay/overlay\n";
     });
-    for (size_t i = 0; i < main_npages; ++i) {
+
+    auto get_pages = [](std::vector<UnderOverlay>& v,
+                        std::vector<std::vector<QPDFPageObjectHelper>>& v_out) {
+        for (auto const& uo: v) {
+            if (uo.pdf) {
+                v_out.push_back(QPDFPageDocumentHelper(*(uo.pdf)).getAllPages());
+            }
+        }
+    };
+    std::vector<std::vector<QPDFPageObjectHelper>> upages;
+    get_pages(m->underlay, upages);
+    std::vector<std::vector<QPDFPageObjectHelper>> opages;
+    get_pages(m->overlay, opages);
+
+    std::map<int, std::map<size_t, QPDFObjectHandle>> underlay_fo;
+    std::map<int, std::map<size_t, QPDFObjectHandle>> overlay_fo;
+    QPDFPageDocumentHelper main_pdh(pdf);
+    auto main_pages = main_pdh.getAllPages();
+    size_t main_npages = main_pages.size();
+    for (size_t page_idx = 0; page_idx < main_npages; ++page_idx) {
+        auto pageno = QIntC::to_int(page_idx) + 1;
         doIfVerbose(
-            [&](Pipeline& v, std::string const& prefix) { v << "  page " << 1 + i << "\n"; });
+            [&](Pipeline& v, std::string const& prefix) { v << "  page " << pageno << "\n"; });
-        auto pageno = QIntC::to_int(i) + 1;
+        if (underlay_pagenos[pageno].empty() && overlay_pagenos[pageno].empty()) {
-        if (!(underlay_pagenos.count(pageno) || overlay_pagenos.count(pageno))) {
             continue;
         }
         // This code converts the original page, any underlays, and any overlays to form XObjects.
         // Then it concatenates display of all underlays, the original page, and all overlays. Prior
         // to 11.3.0, the original page contents were wrapped in q/Q, but this didn't work if the
-        // original page had unbalanced q/Q operators. See github issue #904.
+        // original page had unbalanced q/Q operators. See GitHub issue #904.
-        auto& dest_page = main_pages.at(i);
+        auto& dest_page = main_pages.at(page_idx);
         auto dest_page_oh = dest_page.getObjectHandle();
         auto this_page_fo = dest_page.getFormXObjectForPage();
         // The resulting form xobject lazily reads the content from the original page, which we are
-        // going to replace. Therefore we have to explicitly copy it.
+        // going to replace. Therefore, we have to explicitly copy it.
         auto content_data = this_page_fo.getRawStreamData();
         this_page_fo.replaceStreamData(content_data, QPDFObjectHandle(), QPDFObjectHandle());
         auto resources =
             dest_page_oh.replaceKeyAndGetNew("/Resources", "<< /XObject << >> >>"_qpdf);
         resources.getKey("/XObject").replaceKeyAndGetNew("/Fx0", this_page_fo);
-        auto content = doUnderOverlayForPage(
+        size_t uo_idx{0};
-            pdf, m->underlay, underlay_pagenos, i, underlay_fo, upages, dest_page);
+        std::string content;
+        for (auto& underlay: m->underlay) {
+            content += doUnderOverlayForPage(
+                pdf,
+                underlay,
+                underlay_pagenos,
+                page_idx,
+                uo_idx,
+                underlay_fo,
+                upages[uo_idx],
+                dest_page);
+            ++uo_idx;
+        }
         content += dest_page.placeFormXObject(
             this_page_fo,
             "/Fx0",
@@ -2003,8 +2032,19 @@ QPDFJob::handleUnderOverlay(QPDF& pdf)
             true,
             false,
             false);
-        content += doUnderOverlayForPage(
+        uo_idx = 0;
-            pdf, m->overlay, overlay_pagenos, i, overlay_fo, opages, dest_page);
+        for (auto& overlay: m->overlay) {
+            content += doUnderOverlayForPage(
+                pdf,
+                overlay,
+                overlay_pagenos,
+                page_idx,
+                uo_idx,
+                overlay_fo,
+                opages[uo_idx],
+                dest_page);
+            ++uo_idx;
+        }
         dest_page_oh.replaceKey("/Contents", pdf.newStream(content));
     }
 }
@@ -3057,9 +3097,10 @@ QPDFJob::writeOutfile(QPDF& pdf)
             try {
                 QUtil::remove_file(backup.c_str());
             } catch (QPDFSystemError& e) {
-                *m->log->getError() << m->message_prefix << ": unable to delete original file ("
+                *m->log->getError()
-                                    << e.what() << ");" << " original file left in " << backup
+                    << m->message_prefix << ": unable to delete original file (" << e.what() << ");"
-                                    << ", but the input was successfully replaced\n";
+                    << " original file left in " << backup
+                    << ", but the input was successfully replaced\n";
             }
         }
     }

libqpdf/QPDFJob_config.cc

@@ -1010,14 +1010,16 @@ QPDFJob::PagesConfig::password(std::string const& arg)
 std::shared_ptr<QPDFJob::UOConfig>
 QPDFJob::Config::overlay()
 {
-    o.m->under_overlay = &o.m->overlay;
+    o.m->overlay.emplace_back("overlay");
+    o.m->under_overlay = &o.m->overlay.back();
     return std::shared_ptr<UOConfig>(new UOConfig(this));
 }
 
 std::shared_ptr<QPDFJob::UOConfig>
 QPDFJob::Config::underlay()
 {
-    o.m->under_overlay = &o.m->underlay;
+    o.m->underlay.emplace_back("underlay");
+    o.m->under_overlay = &o.m->underlay.back();
     return std::shared_ptr<UOConfig>(new UOConfig(this));
 }
 

libqpdf/qpdf/auto_job_help.hh

@@ -711,6 +711,9 @@ of the primary output until it runs out of pages, and any extra pages are
 ignored. You can also give a page range with --repeat to cause
 those pages to be repeated after the original pages are exhausted.
 
+This options are repeatable. Pages will be stacked in order of
+appearance: first underlays, then the original page, then overlays.
+
 Run qpdf --help=page-ranges for help with page ranges.
 )");
 }

manual/cli.rst

@@ -2805,6 +2805,9 @@ Overlay and Underlay
    ignored. You can also give a page range with --repeat to cause
    those pages to be repeated after the original pages are exhausted.
 
+   This options are repeatable. Pages will be stacked in order of
+   appearance: first underlays, then the original page, then overlays.
+
    Run qpdf --help=page-ranges for help with page ranges.
 
 You can use :command:`qpdf` to overlay or underlay pages from other
@@ -2823,8 +2826,10 @@ are applied, possibly obscured by the original page, and overlay files
 are drawn on top of the page to which they are applied, possibly
 obscuring the page. The ability to specify the file using the
 :qpdf:ref:`--file` option was added in qpdf 11.9.0. You can combine
-overlay and underlay, but you can only specify each option at most one
+overlay and underlay. Starting in qpdf 11.9.0, you can specify these
-time.
+options multiple times. The final page will be a stack containing the
+underlays in order of appearance, then the original page, then the
+overlays in order of appearance.
 
 The default behavior of overlay and underlay is that pages are taken
 from the overlay/underlay file in sequence and applied to

manual/qpdf.1

@@ -849,6 +849,9 @@ of the primary output until it runs out of pages, and any extra pages are
 ignored. You can also give a page range with --repeat to cause
 those pages to be repeated after the original pages are exhausted.
 
+This options are repeatable. Pages will be stacked in order of
+appearance: first underlays, then the original page, then overlays.
+
 Run qpdf --help=page-ranges for help with page ranges.
 .PP
 Related Options:

manual/release-notes.rst

@@ -48,6 +48,13 @@ Planned changes for future 12.x (subject to change):
       as well. These new options can be freely intermixed with
       positional arguments.
 
+    - Allow :qpdf:ref:`--overlay` and :qpdf:ref:`--underlay` to be
+      repeated. They may appear multiple times on the command-line and
+      will be stacked in the order in which they appear. In QPDFJob
+      JSON (see :ref:`qpdf-job`), the `overlay` and `underlay` keys
+      may contain arrays. For compatibility, they may also contain a
+      single dictionary.
+
   - Library Enhancements
 
     - Add ``file()``, ``range()``, and ``password()`` to