implemented writing R4/V4 encryption except that the output files don't open in Adobe reader 9.1.3.

git-svn-id: svn+q:///qpdf/trunk@816 71b93d88-0707-0410-a8cf-f5a4172ac649

TODO

@@ -43,48 +43,53 @@
    (http://delphi.about.com). .. use at your own risk and for whatever
    the purpose you want .. no support provided. Sample code provided."
 
- * Implement as much of R = 4 encryption as possible.  Already able to
+ * R = 4, V = 4 encryption.
-   decode AES-128-CBC and check passwords.
 
-   aes test suite: use fips-197 test vector with cbc disabled; encrypt
+    - Update C API for R4 encryption
-   and decrypt some other files including multiples of 16 and not to
-   test cbc mode.
 
-   /Encrypt keys (if V == 4)
+    - When we write encrypted files, we must remember to omit any
+      encryption filter settings from original streams.
 
-     /StmF - name of crypt filter for streams; default /Identity
+    - test various combinations with and without cleartext-metadata
-     /StrF - name of crypt filter for strings; default /Identity
+      and aes in compression tests
-     /EFF - crypt filter for embedded files without their own crypt
-       filters; default is to use /StmF
 
-     /CF - keys are crypt filter names, values are are crypt
+    - figure out a way to test crypt filters defined on a stream
-     dictionaries
 
-     Individual streams may also have crypt filters.  Filter type
+    - would be nice to test strings and streams with different
-     /Crypt; /DecodeParms must contain a Crypt filter decode
+      encryption types, but without sample data, we'd have to write
-     parameters dictionary whose /Name entry specifies the particular
+      them ourselves which is not that useful
-     filter to be used.  If /Name is missing, use /Identity.
-     /DecodeParms << /Crypt << /Name /XYZ >> >> where /XYZ is
-     /Identity or a key in /CF.
 
-     /Identity means not to encrypt.
+    - figure out why xpdf can open my files but not acroread
 
-     Crypt Dictionaries
+    - figure out how to look at the metadata so I can tell whether
+      /EncryptMetadata is working the way it's supposed to
 
-       /Type (optional) /CryptFilter
+    - Do something with embedded files, but what and how?
-       /CFM:
-         /V2 - use rc4
-         /AESV2 - use aes
-       /Length - supposed to be key length, but the one file I have
-         has a bogus value for it, so I'm ignoring it.
 
-       We will ignore remaining fields and values.
+    - General notes:
 
-   Remember to honor /EncryptMetadata; applies to streams of /Type
+      /CF - keys are crypt filter names, values are are crypt
-   /Metadata
+      dictionaries
 
-   When we write encrypted files, we must remember to omit any
+      Individual streams may also have crypt filters.  Filter type
-   encryption filter settings from original streams.
+      /Crypt; /DecodeParms must contain a Crypt filter decode
+      parameters dictionary whose /Name entry specifies the particular
+      filter to be used.  If /Name is missing, use /Identity.
+      /DecodeParms << /Crypt << /Name /XYZ >> >> where /XYZ is
+      /Identity or a key in /CF.
+
+      /Identity means not to encrypt.
+
+      Crypt Dictionaries
+
+	/Type (optional) /CryptFilter
+	/CFM:
+	  /V2 - use rc4
+	  /AESV2 - use aes
+	/Length - supposed to be key length, but the one file I have
+	  has a bogus value for it, so I'm ignoring it.
+
+	We will ignore remaining fields and values.
 
 2.2
 ===

include/qpdf/QPDFWriter.hh

@@ -118,9 +118,10 @@ class DLL_EXPORT QPDFWriter
 
     // Set up for encrypted output.  Disables stream prefiltering and
     // content normalization.  Note that setting R2 encryption
-    // parameters sets the PDF version to at least 1.3, and setting R3
+    // parameters sets the PDF version to at least 1.3, setting R3
     // encryption parameters pushes the PDF version number to at least
-    // 1.4.
+    // 1.4, and setting R4 parameters pushes the version to at least
+    // 1.5, or if AES is used, 1.6.
     void setR2EncryptionParameters(
 	char const* user_password, char const* owner_password,
 	bool allow_print, bool allow_modify,
@@ -143,6 +144,11 @@ class DLL_EXPORT QPDFWriter
 	char const* user_password, char const* owner_password,
 	bool allow_accessibility, bool allow_extract,
 	r3_print_e print, r3_modify_e modify);
+    void setR4EncryptionParameters(
+	char const* user_password, char const* owner_password,
+	bool allow_accessibility, bool allow_extract,
+	r3_print_e print, r3_modify_e modify,
+	bool encrypt_metadata, bool use_aes);
 
     // Create linearized output.  Disables qdf mode, content
     // normalization, and stream prefiltering.
@@ -182,6 +188,11 @@ class DLL_EXPORT QPDFWriter
     void preserveObjectStreams();
     void generateObjectStreams();
     void generateID();
+    void interpretR3EncryptionParameters(
+	std::set<int>& bits_to_clear,
+	char const* user_password, char const* owner_password,
+	bool allow_accessibility, bool allow_extract,
+	r3_print_e print, r3_modify_e modify);
     void setEncryptionParameters(
 	char const* user_password, char const* owner_password,
 	int V, int R, int key_len, std::set<int>& bits_to_clear);
@@ -231,6 +242,7 @@ class DLL_EXPORT QPDFWriter
     // stack items are of type Pl_Buffer, the buffer is retrieved.
     void popPipelineStack(PointerHolder<Buffer>* bp = 0);
 
+    void adjustAESStreamLength(unsigned long& length);
     void pushEncryptionFilter();
     void pushDiscardFilter();
 
@@ -251,6 +263,8 @@ class DLL_EXPORT QPDFWriter
     bool linearized;
     object_stream_e object_stream_mode;
     std::string encryption_key;
+    bool encrypt_metadata;
+    bool encrypt_use_aes;
     std::map<std::string, std::string> encryption_dictionary;
 
     std::string id1;		// for /ID key of
@@ -267,7 +281,7 @@ class DLL_EXPORT QPDFWriter
     std::map<int, size_t> lengths;
     int next_objid;
     int cur_stream_length_id;
-    int cur_stream_length;
+    unsigned long cur_stream_length;
     bool added_newline;
     int max_ostream_index;
     std::set<int> normalized_streams;

libqpdf/QPDFWriter.cc

@@ -6,6 +6,7 @@
 #include <qpdf/Pl_Discard.hh>
 #include <qpdf/Pl_Buffer.hh>
 #include <qpdf/Pl_RC4.hh>
+#include <qpdf/Pl_AES_PDF.hh>
 #include <qpdf/Pl_Flate.hh>
 #include <qpdf/Pl_PNGFilter.hh>
 #include <qpdf/QUtil.hh>
@@ -37,6 +38,8 @@ QPDFWriter::QPDFWriter(QPDF& pdf, char const* filename) :
     preserve_encryption(true),
     linearized(false),
     object_stream_mode(o_preserve),
+    encrypt_metadata(true),
+    encrypt_use_aes(false),
     encryption_dict_objid(0),
     next_objid(1),
     cur_stream_length_id(0),
@@ -187,6 +190,38 @@ QPDFWriter::setR3EncryptionParameters(
     char const* user_password, char const* owner_password,
     bool allow_accessibility, bool allow_extract,
     r3_print_e print, r3_modify_e modify)
+{
+    std::set<int> clear;
+    interpretR3EncryptionParameters(
+	clear, user_password, owner_password,
+	allow_accessibility, allow_extract, print, modify);
+    setMinimumPDFVersion("1.4");
+    setEncryptionParameters(user_password, owner_password, 2, 3, 16, clear);
+}
+
+void
+QPDFWriter::setR4EncryptionParameters(
+    char const* user_password, char const* owner_password,
+    bool allow_accessibility, bool allow_extract,
+    r3_print_e print, r3_modify_e modify,
+    bool encrypt_metadata, bool use_aes)
+{
+    std::set<int> clear;
+    interpretR3EncryptionParameters(
+	clear, user_password, owner_password,
+	allow_accessibility, allow_extract, print, modify);
+    this->encrypt_use_aes = use_aes;
+    this->encrypt_metadata = encrypt_metadata;
+    setMinimumPDFVersion(use_aes ? "1.6" : "1.5");
+    setEncryptionParameters(user_password, owner_password, 4, 4, 16, clear);
+}
+
+void
+QPDFWriter::interpretR3EncryptionParameters(
+    std::set<int>& clear,
+    char const* user_password, char const* owner_password,
+    bool allow_accessibility, bool allow_extract,
+    r3_print_e print, r3_modify_e modify)
 {
     // Acrobat 5 security options:
 
@@ -206,7 +241,6 @@ QPDFWriter::setR3EncryptionParameters(
     //   Low Resolution
     //   Full printing
 
-    std::set<int> clear;
     if (! allow_accessibility)
     {
 	clear.insert(10);
@@ -251,9 +285,6 @@ QPDFWriter::setR3EncryptionParameters(
 
 	// no default so gcc warns for missing cases
     }
-
-    setMinimumPDFVersion("1.4");
-    setEncryptionParameters(user_password, owner_password, 2, 3, 16, clear);
 }
 
 void
@@ -282,7 +313,7 @@ QPDFWriter::setEncryptionParameters(
     std::string U;
     QPDF::compute_encryption_O_U(
 	user_password, owner_password, V, R, key_len, P,
-	/*XXX encrypt_metadata*/true, this->id1, O, U);
+	this->encrypt_metadata, this->id1, O, U);
     setEncryptionParametersInternal(
 	V, R, key_len, P, O, U, this->id1, user_password);
 }
@@ -326,9 +357,22 @@ QPDFWriter::setEncryptionParametersInternal(
     encryption_dictionary["/P"] = QUtil::int_to_string(P);
     encryption_dictionary["/O"] = QPDF_String(O).unparse(true);
     encryption_dictionary["/U"] = QPDF_String(U).unparse(true);
+    if ((R >= 4) && (! encrypt_metadata))
+    {
+	encryption_dictionary["/EncryptMetadata"] = "false";
+    }
+    if (V == 4)
+    {
+	encryption_dictionary["/StmF"] = "/CF1";
+	encryption_dictionary["/StrF"] = "/CF1";
+	std::string method = (this->encrypt_use_aes ? "/AESV2" : "/V2");
+	encryption_dictionary["/CF"] =
+	    "<< /CF1 << /AuthEvent /DocOpen /CFM " + method + " >> >>";
+    }
+
     this->encrypted = true;
-    QPDF::EncryptionData encryption_data(V, R, key_len, P, O, U, this->id1,
+    QPDF::EncryptionData encryption_data(
-					 /*XXX encrypt_metadata*/true);
+	V, R, key_len, P, O, U, this->id1, this->encrypt_metadata);
     this->encryption_key = QPDF::compute_encryption_key(
 	user_password, encryption_data);
 }
@@ -337,7 +381,7 @@ void
 QPDFWriter::setDataKey(int objid)
 {
     this->cur_data_key = QPDF::compute_data_key(
-	this->encryption_key, objid, 0, /*XXX use_aes */false);
+	this->encryption_key, objid, 0, this->encrypt_use_aes);
 }
 
 int
@@ -435,15 +479,37 @@ QPDFWriter::popPipelineStack(PointerHolder<Buffer>* bp)
     this->pipeline = dynamic_cast<Pl_Count*>(this->pipeline_stack.back());
 }
 
+void
+QPDFWriter::adjustAESStreamLength(unsigned long& length)
+{
+    if (this->encrypted && (! this->cur_data_key.empty()) &&
+	this->encrypt_use_aes)
+    {
+	// Stream length will be padded with 1 to 16 bytes to end up
+	// as a multiple of 16.  It will also be prepended by 16 bits
+	// of random data.
+	length += 32 - (length & 0xf);
+    }
+}
+
 void
 QPDFWriter::pushEncryptionFilter()
 {
     if (this->encrypted && (! this->cur_data_key.empty()))
     {
-	Pipeline* p =
+	Pipeline* p = 0;
-	    new Pl_RC4("stream encryption", this->pipeline,
+	if (this->encrypt_use_aes)
-		       (unsigned char*) this->cur_data_key.c_str(),
+	{
-		       this->cur_data_key.length());
+	    p = new Pl_AES_PDF(
+		"aes stream encryption", this->pipeline, true,
+		(unsigned char*) this->cur_data_key.c_str());
+	}
+	else
+	{
+	    p = new Pl_RC4("rc4 stream encryption", this->pipeline,
+			   (unsigned char*) this->cur_data_key.c_str(),
+			   this->cur_data_key.length());
+	}
 	pushPipeline(p);
     }
     // Must call this unconditionally so we can call popPipelineStack
@@ -722,6 +788,8 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
     }
     else if (object.isDictionary())
     {
+	// XXX Must not preserve Crypt filters from original stream
+	// dictionary
 	writeString("<<");
 	writeStringQDF("\n");
 	std::set<std::string> keys = object.getKeys();
@@ -836,6 +904,15 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
 	}
 
 	this->cur_stream_length = stream_data.getPointer()->getSize();
+	if (this->encrypted &&
+	    stream_dict.getKey("/Type").isName() &&
+	    (stream_dict.getKey("/Type").getName() == "/Metadata") &&
+	    (! this->encrypt_metadata))
+	{
+	    // Don't encrypt stream data for the metadata stream
+	    this->cur_data_key.clear();
+	}
+	adjustAESStreamLength(this->cur_stream_length);
 	unparseObject(stream_dict, 0, flags, this->cur_stream_length, compress);
 	writeString("\nstream\n");
 	pushEncryptionFilter();
@@ -864,13 +941,29 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
 	    (! this->cur_data_key.empty()))
 	{
 	    val = object.getStringValue();
-	    char* tmp = QUtil::copy_string(val);
+	    if (this->encrypt_use_aes)
-	    unsigned int vlen = val.length();
+	    {
-	    RC4 rc4((unsigned char const*)this->cur_data_key.c_str(),
+		Pl_Buffer bufpl("encrypted string");
-		    this->cur_data_key.length());
+		Pl_AES_PDF pl("aes encrypt string", &bufpl, true,
-	    rc4.process((unsigned char*)tmp, vlen);
+			      (unsigned char const*)this->cur_data_key.c_str());
-	    val = QPDF_String(std::string(tmp, vlen)).unparse();
+		pl.write((unsigned char*) val.c_str(), val.length());
-	    delete [] tmp;
+		pl.finish();
+		Buffer* buf = bufpl.getBuffer();
+		val = QPDF_String(
+		    std::string((char*)buf->getBuffer(),
+				(size_t)buf->getSize())).unparse();
+		delete buf;
+	    }
+	    else
+	    {
+		char* tmp = QUtil::copy_string(val);
+		unsigned int vlen = val.length();
+		RC4 rc4((unsigned char const*)this->cur_data_key.c_str(),
+			this->cur_data_key.length());
+		rc4.process((unsigned char*)tmp, vlen);
+		val = QPDF_String(std::string(tmp, vlen)).unparse();
+		delete [] tmp;
+	    }
 	}
 	else
 	{
@@ -1000,8 +1093,9 @@ QPDFWriter::writeObjectStream(QPDFObjectHandle object)
     writeStringQDF("\n ");
     writeString(" /Type /ObjStm");
     writeStringQDF("\n ");
-    writeString(" /Length " +
+    unsigned long length = stream_buffer.getPointer()->getSize();
-		QUtil::int_to_string(stream_buffer.getPointer()->getSize()));
+    adjustAESStreamLength(length);
+    writeString(" /Length " + QUtil::int_to_string(length));
     writeStringQDF("\n ");
     if (compressed)
     {
@@ -1489,6 +1583,7 @@ QPDFWriter::writeHintStream(int hint_id)
 	writeString(QUtil::int_to_string(O));
     }
     writeString(" /Length ");
+    adjustAESStreamLength(hlen);
     writeString(QUtil::int_to_string(hlen));
     writeString(" >>\nstream\n");
 

qpdf/qpdf.cc

@@ -72,6 +72,9 @@ Additional flags are dependent upon key length.\n\
     --extract=[yn]           allow other text/graphic extraction\n\
     --print=print-opt        control printing access\n\
     --modify=modify-opt      control modify access\n\
+    --cleartext-metadata     prevents encryption of metadata\n\
+    --use-aes=[yn]           indicates whether to use AES encryption\n\
+    --force-V4               forces use of V=4 encryption handler\n\
 \n\
     print-opt may be:\n\
 \n\
@@ -89,6 +92,14 @@ Additional flags are dependent upon key length.\n\
 \n\
 The default for each permission option is to be fully permissive.\n\
 \n\
+Specifying cleartext-metadata forces the PDF version to at least 1.5.\n\
+Specifying use of AES forces the PDF version to at least 1.6.  These\n\
+options are both off by default.\n\
+\n\
+The --force-V4 flag forces the V=4 encryption handler introduced in PDF 1.5\n\
+to be used even if not otherwise needed.  This option is primarily useful\n\
+for testing qpdf and has no other practical use.\n\
+\n\
 \n\
 Advanced Transformation Options\n\
 -------------------------------\n\
@@ -220,7 +231,8 @@ parse_encrypt_options(
     std::string& user_password, std::string& owner_password, int& keylen,
     bool& r2_print, bool& r2_modify, bool& r2_extract, bool& r2_annotate,
     bool& r3_accessibility, bool& r3_extract,
-    QPDFWriter::r3_print_e& r3_print, QPDFWriter::r3_modify_e& r3_modify)
+    QPDFWriter::r3_print_e& r3_print, QPDFWriter::r3_modify_e& r3_modify,
+    bool& force_V4, bool& cleartext_metadata, bool& use_aes)
 {
     if (cur_arg + 3 >= argc)
     {
@@ -450,6 +462,65 @@ parse_encrypt_options(
 		usage("-accessibility invalid for 40-bit keys");
 	    }
 	}
+	else if (strcmp(arg, "cleartext-metadata") == 0)
+	{
+	    if (parameter)
+	    {
+		usage("--cleartext-metadata does not take a parameter");
+	    }
+	    if (keylen == 40)
+	    {
+		usage("--cleartext-metadata is invalid for 40-bit keys");
+	    }
+	    else
+	    {
+		cleartext_metadata = true;
+	    }
+	}
+	else if (strcmp(arg, "force-V4") == 0)
+	{
+	    if (parameter)
+	    {
+		usage("--force-V4 does not take a parameter");
+	    }
+	    if (keylen == 40)
+	    {
+		usage("--force-V4 is invalid for 40-bit keys");
+	    }
+	    else
+	    {
+		force_V4 = true;
+	    }
+	}
+	else if (strcmp(arg, "use-aes") == 0)
+	{
+	    if (parameter == 0)
+	    {
+		usage("--use-aes must be given as --extract=option");
+	    }
+	    std::string val = parameter;
+	    bool result = false;
+	    if (val == "y")
+	    {
+		result = true;
+	    }
+	    else if (val == "n")
+	    {
+		result = false;
+	    }
+	    else
+	    {
+		usage("invalid -use-aes parameter");
+	    }
+	    if (keylen == 40)
+	    {
+		usage("use-aes is invalid for 40-bit keys");
+	    }
+	    else
+	    {
+		use_aes = result;
+	    }
+	}
 	else
 	{
 	    usage(std::string("invalid encryption parameter --") + arg);
@@ -516,6 +587,9 @@ int main(int argc, char* argv[])
     bool r3_extract = true;
     QPDFWriter::r3_print_e r3_print = QPDFWriter::r3p_full;
     QPDFWriter::r3_modify_e r3_modify = QPDFWriter::r3m_all;
+    bool force_V4 = false;
+    bool cleartext_metadata = false;
+    bool use_aes = false;
 
     bool stream_data_set = false;
     QPDFWriter::stream_data_e stream_data_mode = QPDFWriter::s_compress;
@@ -582,7 +656,8 @@ int main(int argc, char* argv[])
 		    argc, argv, ++i,
 		    user_password, owner_password, keylen,
 		    r2_print, r2_modify, r2_extract, r2_annotate,
-		    r3_accessibility, r3_extract, r3_print, r3_modify);
+		    r3_accessibility, r3_extract, r3_print, r3_modify,
+		    force_V4, cleartext_metadata, use_aes);
 		encrypt = true;
 	    }
 	    else if (strcmp(arg, "decrypt") == 0)
@@ -988,9 +1063,19 @@ int main(int argc, char* argv[])
 		}
 		else if (keylen == 128)
 		{
-		    w.setR3EncryptionParameters(
+		    if (force_V4 || cleartext_metadata || use_aes)
-			user_password.c_str(), owner_password.c_str(),
+		    {
-			r3_accessibility, r3_extract, r3_print, r3_modify);
+			w.setR4EncryptionParameters(
+			    user_password.c_str(), owner_password.c_str(),
+			    r3_accessibility, r3_extract, r3_print, r3_modify,
+			    !cleartext_metadata, use_aes);
+		    }
+		    else
+		    {
+			w.setR3EncryptionParameters(
+			    user_password.c_str(), owner_password.c_str(),
+			    r3_accessibility, r3_extract, r3_print, r3_modify);
+		    }
 		}
 		else
 		{