Add copyAnnotations, use with overlay/underlay (fixes #395)

ChangeLog

@@ -1,5 +1,12 @@
 2021-02-21  Jay Berkenbilt  <ejb@ql.org>
 
+	* From qpdf CLI, --overlay and --underlay will copy annotations
+	and form fields from overlay/underlay file. Fixes #395.
+
+	* Add QPDFPageObjectHelper::copyAnnotations, which copies
+	annotations and, if applicable, associated form fields, from one
+	page to another, possibly transforming the rectangles.
+
 	* Bug fix: --flatten-rotation now applies the required
 	transformation to annotations on the page.
 

include/qpdf/QPDFPageObjectHelper.hh

@@ -335,6 +335,33 @@ class QPDFPageObjectHelper: public QPDFObjectHelper
     QPDF_DLL
     void flattenRotation(QPDFAcroFormDocumentHelper* afdh);
 
+    // Copy annotations from another page into this page. The other
+    // page may be from the same QPDF or from a different QPDF. Each
+    // annotation's rectangle is transformed by the given matrix. If
+    // the annotation is a widget annotation that is associated with a
+    // form field, the form field is copied into this document's
+    // AcroForm dictionary as well. You can use this to copy
+    // annotations from a page that was converted to a form XObject
+    // and added to another page. For example of this, see
+    // examples/pdf-overlay-page.cc. Note that if you use this to copy
+    // annotations from one page to another in the same document and
+    // you use a transformation matrix other than the identity matrix,
+    // it will alter the original annotation, which is probably not
+    // what you want. Also, if you copy the same page multiple times
+    // with different transformation matrices, the effect will be
+    // cumulative, which is probably also not what you want.
+    //
+    // If you pass in a QPDFAcroFormDocumentHelper*, the method will
+    // use that instead of creating one in the function. Creating
+    // QPDFAcroFormDocumentHelper objects is expensive, so if you're
+    // doing a lot of copying, it can be more efficient to create
+    // these outside and pass them in.
+    QPDF_DLL
+    void copyAnnotations(
+        QPDFPageObjectHelper from_page, QPDFMatrix const& cm = QPDFMatrix(),
+        QPDFAcroFormDocumentHelper* afdh = nullptr,
+        QPDFAcroFormDocumentHelper* from_afdh = nullptr);
+
   private:
     static bool
     removeUnreferencedResourcesHelper(

libqpdf/QPDFPageObjectHelper.cc

@@ -1236,3 +1236,78 @@ QPDFPageObjectHelper::flattenRotation(QPDFAcroFormDocumentHelper* afdh)
         this->oh.replaceKey("/Annots", QPDFObjectHandle::newArray(new_annots));
     }
 }
+
+void
+QPDFPageObjectHelper::copyAnnotations(
+    QPDFPageObjectHelper from_page, QPDFMatrix const& cm,
+    QPDFAcroFormDocumentHelper* afdh,
+    QPDFAcroFormDocumentHelper* from_afdh)
+{
+    auto old_annots = from_page.getObjectHandle().getKey("/Annots");
+    if (! old_annots.isArray())
+    {
+        return;
+    }
+
+    QPDF* from_qpdf = from_page.getObjectHandle().getOwningQPDF();
+    if (! from_qpdf)
+    {
+        throw std::runtime_error(
+            "QPDFPageObjectHelper::copyAnnotations:"
+            " from page is a direct object");
+    }
+    QPDF* this_qpdf = this->oh.getOwningQPDF();
+    if (! this_qpdf)
+    {
+        throw std::runtime_error(
+            "QPDFPageObjectHelper::copyAnnotations:"
+            " this page is a direct object");
+    }
+
+    std::vector<QPDFObjectHandle> new_annots;
+    std::vector<QPDFObjectHandle> new_fields;
+    std::set<QPDFObjGen> old_fields;
+    PointerHolder<QPDFAcroFormDocumentHelper> afdhph;
+    PointerHolder<QPDFAcroFormDocumentHelper> from_afdhph;
+    if (! afdh)
+    {
+        afdhph = new QPDFAcroFormDocumentHelper(*this_qpdf);
+        afdh = afdhph.getPointer();
+    }
+    if (this_qpdf == from_qpdf)
+    {
+        from_afdh = afdh;
+    }
+    else if (from_afdh)
+    {
+        if (from_afdh->getQPDF().getUniqueId() != from_qpdf->getUniqueId())
+        {
+            throw std::logic_error(
+                "QPDFAcroFormDocumentHelper::copyAnnotations: from_afdh"
+                " is not from the same QPDF as from_page");
+        }
+    }
+    else
+    {
+        from_afdhph = new QPDFAcroFormDocumentHelper(*from_qpdf);
+        from_afdh = from_afdhph.getPointer();
+    }
+
+    afdh->transformAnnotations(
+        old_annots, new_annots, new_fields, old_fields, cm,
+        from_qpdf, from_afdh);
+    for (auto const& f: new_fields)
+    {
+        afdh->addFormField(QPDFFormFieldObjectHelper(f));
+    }
+    auto annots = this->oh.getKey("/Annots");
+    if (! annots.isArray())
+    {
+        annots = QPDFObjectHandle::newArray();
+        this->oh.replaceKey("/Annots", annots);
+    }
+    for (auto const& annot: new_annots)
+    {
+        annots.appendItem(annot);
+    }
+}

manual/qpdf-manual.xml

@@ -5319,6 +5319,15 @@ print "\n";
           which applies a transformation to each annotation on a page.
          </para>
         </listitem>
+        <listitem>
+         <para>
+	  Add
+	  <function>QPDFPageObjectHelper::copyAnnotations</function>,
+	  which copies annotations and, if applicable, associated form
+	  fields, from one page to another, possibly transforming the
+	  rectangles.
+         </para>
+        </listitem>
         <listitem>
          <para>
           Add <function>QUtil::path_basename</function> to return the

qpdf/qpdf.cc

@@ -5160,6 +5160,19 @@ static void do_under_overlay_for_page(
         return;
     }
 
+    std::map<unsigned long long,
+             PointerHolder<QPDFAcroFormDocumentHelper>> afdh;
+    auto make_afdh = [&](QPDFPageObjectHelper& ph) {
+        QPDF* q = ph.getObjectHandle().getOwningQPDF();
+        auto uid = q->getUniqueId();
+        if (! afdh.count(uid))
+        {
+            afdh[uid] = new QPDFAcroFormDocumentHelper(*q);
+        }
+        return afdh[uid].getPointer();
+    };
+    auto dest_afdh = make_afdh(dest_page);
+
     std::string content;
     int min_suffix = 1;
     QPDFObjectHandle resources = dest_page.getAttribute("/Resources", true);
@@ -5171,13 +5184,19 @@ static void do_under_overlay_for_page(
         {
             std::cout << "    " << uo.which << " " << from_pageno << std::endl;
         }
+        auto from_page = pages.at(QIntC::to_size(from_pageno - 1));
         if (0 == fo.count(from_pageno))
         {
             fo[from_pageno] =
                 pdf.copyForeignObject(
-                    pages.at(QIntC::to_size(from_pageno - 1)).
+                    from_page.getFormXObjectForPage());
-                    getFormXObjectForPage());
         }
+        auto cm = dest_page.getMatrixForFormXObjectPlacement(
+            fo[from_pageno],
+            dest_page.getTrimBox().getArrayAsRectangle());
+        dest_page.copyAnnotations(
+            from_page, cm, dest_afdh, make_afdh(from_page));
+
         // If the same page is overlaid or underlaid multiple times,
         // we'll generate multiple names for it, but that's harmless
         // and also a pretty goofy case that's not worth coding

qpdf/qpdf.testcov

@@ -572,6 +572,6 @@ qpdf password file 0
 QPDFFileSpecObjectHelper empty compat_name 0
 QPDFFileSpecObjectHelper non-empty compat_name 0
 QPDFPageObjectHelper flatten inherit rotate 0
-QPDFAcroFormDocumentHelper copy annotation 1
+QPDFAcroFormDocumentHelper copy annotation 3
-QPDFAcroFormDocumentHelper field with parent 1
+QPDFAcroFormDocumentHelper field with parent 3
 QPDFAcroFormDocumentHelper modify ap matrix 0

qpdf/qtest/qpdf.test

@@ -2411,6 +2411,53 @@ foreach my $f (qw(screen print))
                  {$td->FILE => "manual-appearances-$f-out.pdf"});
 }
 
+show_ntests();
+# ----------
+$td->notify("--- Copy Annotations ---");
+$n_tests += 16;
+
+$td->runtest("complex copy annotations",
+             {$td->COMMAND =>
+                  "qpdf --qdf --static-id --no-original-object-ids" .
+                  " fxo-red.pdf --overlay form-fields-and-annotations.pdf" .
+                  " --repeat=1 -- a.pdf"},
+             {$td->STRING => "", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+$td->runtest("check output",
+             {$td->FILE => "a.pdf"},
+             {$td->FILE => "overlay-copy-annotations.pdf"});
+
+foreach my $page (1, 2, 5, 6)
+{
+    $td->runtest("copy annotations single page ($page)",
+                 {$td->COMMAND =>
+                      "qpdf --qdf --static-id --no-original-object-ids" .
+                      " --pages . $page --" .
+                      " fxo-red.pdf --overlay form-fields-and-annotations.pdf" .
+                      " --repeat=1 -- a.pdf"},
+                 {$td->STRING => "", $td->EXIT_STATUS => 0},
+                 $td->NORMALIZE_NEWLINES);
+    $td->runtest("check output",
+                 {$td->FILE => "a.pdf"},
+                 {$td->FILE => "overlay-copy-annotations-p$page.pdf"});
+}
+
+foreach my $d ([1, "appearances-1.pdf"],
+               [2, "appearances-1-rotated.pdf"])
+{
+    my ($n, $file1) = @$d;
+    $td->runtest("copy/transfer with defaults",
+                 {$td->COMMAND => "test_driver 80 $file1 minimal.pdf"},
+                 {$td->STRING => "test 80 done\n", $td->EXIT_STATUS => 0},
+                 $td->NORMALIZE_NEWLINES);
+    $td->runtest("check output A",
+                 {$td->FILE => "a.pdf"},
+                 {$td->FILE => "test80a$n.pdf"});
+    $td->runtest("check output B",
+                 {$td->FILE => "b.pdf"},
+                 {$td->FILE => "test80b$n.pdf"});
+}
+
 show_ntests();
 # ----------
 $td->notify("--- Page Tree Issues ---");

qpdf/test_driver.cc

@@ -2905,6 +2905,56 @@ void runtest(int n, char const* filename1, char const* arg2)
         w.setQDFMode(true);
         w.write();
     }
+    else if (n == 80)
+    {
+        // Exercise transform/copy annotations without passing in
+        // QPDFAcroFormDocumentHelper pointers. The case of passing
+        // them in is sufficiently exercised by testing through the
+        // qpdf CLI.
+
+        // The main file is a file that has lots of annotations. Arg2
+        // is a file to copy annotations to.
+
+        QPDFMatrix m;
+        m.translate(306, 396);
+        m.scale(0.4, 0.4);
+        auto page1 = pdf.getAllPages().at(0);
+        auto old_annots = page1.getKey("/Annots");
+        // Transform annotations and copy them back to the same page.
+        std::vector<QPDFObjectHandle> new_annots;
+        std::vector<QPDFObjectHandle> new_fields;
+        std::set<QPDFObjGen> old_fields;
+        QPDFAcroFormDocumentHelper afdh(pdf);
+        // Use defaults for from_qpdf and from_afdh.
+        afdh.transformAnnotations(
+            old_annots, new_annots, new_fields, old_fields, m);
+        for (auto const& annot: new_annots)
+        {
+            old_annots.appendItem(annot);
+        }
+        for (auto const& field: new_fields)
+        {
+            afdh.addFormField(QPDFFormFieldObjectHelper(field));
+        }
+
+        m = QPDFMatrix();
+        m.translate(612, 0);
+        m.scale(-1, 1);
+        QPDF pdf2;
+        pdf2.processFile(arg2);
+        auto page2 = QPDFPageDocumentHelper(pdf2).getAllPages().at(0);
+        page2.copyAnnotations(page1, m);
+
+        QPDFWriter w1(pdf, "a.pdf");
+        w1.setStaticID(true);
+        w1.setQDFMode(true);
+        w1.write();
+
+        QPDFWriter w2(pdf2, "b.pdf");
+        w2.setStaticID(true);
+        w2.setQDFMode(true);
+        w2.write();
+    }
     else
     {
 	throw std::runtime_error(std::string("invalid test ") +