checkpoint -- partially implemented /V=4 encryption

git-svn-id: svn+q:///qpdf/trunk@811 71b93d88-0707-0410-a8cf-f5a4172ac649

TODO

@@ -43,6 +43,49 @@
    (http://delphi.about.com). .. use at your own risk and for whatever
    the purpose you want .. no support provided. Sample code provided."
 
+ * Implement as much of R = 4 encryption as possible.  Already able to
+   decode AES-128-CBC and check passwords.
+
+   aes test suite: use fips-197 test vector with cbc disabled; encrypt
+   and decrypt some other files including multiples of 16 and not to
+   test cbc mode.
+
+   /Encrypt keys (if V == 4)
+
+     /StmF - name of crypt filter for streams; default /Identity
+     /StrF - name of crypt filter for strings; default /Identity
+     /EFF - crypt filter for embedded files without their own crypt
+       filters; default is to use /StmF
+
+     /CF - keys are crypt filter names, values are are crypt
+     dictionaries
+
+     Individual streams may also have crypt filters.  Filter type
+     /Crypt; /DecodeParms must contain a Crypt filter decode
+     parameters dictionary whose /Name entry specifies the particular
+     filter to be used.  If /Name is missing, use /Identity.
+     /DecodeParms << /Crypt << /Name /XYZ >> >> where /XYZ is
+     /Identity or a key in /CF.
+
+     /Identity means not to encrypt.
+
+     Crypt Dictionaries
+
+       /Type (optional) /CryptFilter
+       /CFM:
+         /V2 - use rc4
+         /AESV2 - use aes
+       /Length - supposed to be key length, but the one file I have
+         has a bogus value for it, so I'm ignoring it.
+
+       We will ignore remaining fields and values.
+
+   Remember to honor /EncryptMetadata; applies to streams of /Type
+   /Metadata
+
+   When we write encrypted files, we must remember to omit any
+   encryption filter settings from original streams.
+
 2.2
 ===
 
@@ -52,22 +95,6 @@
    Stefan Heinsen <stefan.heinsen@gmx.de> in August, 2009.  He seems
    to like to send encrypted mail.  (key 01FCC336)
 
- * See whether we can do anything with /V > 3 in the encryption
-   dictionary.  (V = 4 is Crypt Filters.)  See
-   ~/Q/pdf-collection/R4-encrypt-PDF_Inside_and_Out.pdf
-
-   Search for XXX in the code.  Implementation has been started.
-
-   Algorithms from PDF Spec in QPDF_encrypt.cc have been updated.  We
-   can at least properly verify the user password with an R4 file.  In
-   order to finish the job, we need an aes-128-cbc implementation.
-   Then we can fill in the gaps for the aes pipeline and actually run
-   the test suite.  The pipeline may be able to hard-code the
-   initialization vector stuff by taking the first block of input and
-   by writing a random block for output.  The padding is already in
-   the code, but the initialization vector is not since I accidentally
-   started using an aes256 implementation instead of aes128-cbc.
-
  * Look at page splitting.
 
 
@@ -109,9 +136,9 @@ General
    of doing this seems very low since no viewer seems to care, so it's
    probably not worth it.
 
- * Embedded files streams: figure out why running qpdf over the pdf
+ * Embedded file streams: figure out why running qpdf over the pdf 1.7
-   1.7 spec results in a file that crashes acrobat reader when you try
+   spec results in a file that crashes acrobat reader when you try to
-   to save nested documents.
+   save nested documents.
 
  * QPDFObjectHandle::getPageImages() doesn't notice images in
    inherited resource dictionaries.  See comments in that function.

include/qpdf/QPDF.hh

@@ -141,7 +141,7 @@ class DLL_EXPORT QPDF
 
     static void compute_encryption_O_U(
 	char const* user_password, char const* owner_password,
-	int V, int R, int key_len, int P,
+	int V, int R, int key_len, int P, bool encrypt_metadata,
 	std::string const& id1,
 	std::string& O, std::string& U);
     // Return the full user password as stored in the PDF file.  If
@@ -398,10 +398,12 @@ class DLL_EXPORT QPDF
 
     // methods to support encryption -- implemented in QPDF_encryption.cc
     void initializeEncryption();
-    std::string getKeyForObject(int objid, int generation);
+    std::string getKeyForObject(int objid, int generation, bool use_aes);
     void decryptString(std::string&, int objid, int generation);
-    void decryptStream(Pipeline*& pipeline, int objid, int generation,
+    void decryptStream(
-		       std::vector<PointerHolder<Pipeline> >& heap);
+	Pipeline*& pipeline, int objid, int generation,
+	QPDFObjectHandle& stream_dict,
+	std::vector<PointerHolder<Pipeline> >& heap);
 
     // Linearization Hint table structures.
     // Naming conventions:
@@ -735,7 +737,9 @@ class DLL_EXPORT QPDF
     bool ignore_xref_streams;
     bool suppress_warnings;
     bool attempt_recovery;
-    bool encryption_use_aes;
+    int encryption_V;
+    bool encrypt_metadata;
+    QPDFObjectHandle encryption_dictionary;
     std::string provided_password;
     std::string user_password;
     std::string encryption_key;

libqpdf/Pl_AES_PDF.cc

@@ -1,16 +1,19 @@
 #include <qpdf/Pl_AES_PDF.hh>
 #include <qpdf/QUtil.hh>
+#include <qpdf/MD5.hh>
 #include <cstring>
 #include <assert.h>
 #include <stdexcept>
 #include <qpdf/rijndael.h>
-
+#include <string>
-// XXX Still need CBC
+#include <stdlib.h>
 
 Pl_AES_PDF::Pl_AES_PDF(char const* identifier, Pipeline* next,
 		       bool encrypt, unsigned char key[key_size]) :
     Pipeline(identifier, next),
     encrypt(encrypt),
+    cbc_mode(true),
+    first(true),
     offset(0),
     nrounds(0)
 {
@@ -21,6 +24,7 @@ Pl_AES_PDF::Pl_AES_PDF(char const* identifier, Pipeline* next,
     std::memset(this->rk, 0, sizeof(this->rk));
     std::memset(this->inbuf, 0, this->buf_size);
     std::memset(this->outbuf, 0, this->buf_size);
+    std::memset(this->cbc_block, 0, this->buf_size);
     if (encrypt)
     {
 	this->nrounds = rijndaelSetupEncrypt(this->rk, this->key, keybits);
@@ -37,6 +41,12 @@ Pl_AES_PDF::~Pl_AES_PDF()
     // nothing needed
 }
 
+void
+Pl_AES_PDF::disableCBC()
+{
+    this->cbc_mode = false;
+}
+
 void
 Pl_AES_PDF::write(unsigned char* data, int len)
 {
@@ -90,17 +100,80 @@ Pl_AES_PDF::finish()
     getNext()->finish();
 }
 
+void
+Pl_AES_PDF::initializeVector()
+{
+    std::string seed_str;
+    seed_str += QUtil::int_to_string((int)QUtil::get_current_time());
+    seed_str += " QPDF aes random";
+    MD5 m;
+    m.encodeString(seed_str.c_str());
+    MD5::Digest digest;
+    m.digest(digest);
+    assert(sizeof(digest) >= sizeof(unsigned int));
+    unsigned int seed;
+    memcpy((void*)(&seed), digest, sizeof(unsigned int));
+    srandom(seed);
+    for (unsigned int i = 0; i < this->buf_size; ++i)
+    {
+	this->cbc_block[i] = (unsigned char)(random() & 0xff);
+    }
+}
+
 void
 Pl_AES_PDF::flush(bool strip_padding)
 {
     assert(this->offset == this->buf_size);
+
+    if (first)
+    {
+	first = false;
+	if (this->cbc_mode)
+	{
+	    if (encrypt)
+	    {
+		// Set cbc_block to a random initialization vector and
+		// write it to the output stream
+		initializeVector();
+		getNext()->write(this->cbc_block, this->buf_size);
+	    }
+	    else
+	    {
+		// Take the first block of input as the initialization
+		// vector.  There's nothing to write at this time.
+		memcpy(this->cbc_block, this->inbuf, this->buf_size);
+		this->offset = 0;
+		return;
+	    }
+	}
+    }
+
     if (this->encrypt)
     {
+	if (this->cbc_mode)
+	{
+	    for (unsigned int i = 0; i < this->buf_size; ++i)
+	    {
+		this->inbuf[i] ^= this->cbc_block[i];
+	    }
+	}
 	rijndaelEncrypt(this->rk, this->nrounds, this->inbuf, this->outbuf);
+	if (this->cbc_mode)
+	{
+	    memcpy(this->cbc_block, this->outbuf, this->buf_size);
+	}
     }
     else
     {
 	rijndaelDecrypt(this->rk, this->nrounds, this->inbuf, this->outbuf);
+	if (this->cbc_mode)
+	{
+	    for (unsigned int i = 0; i < this->buf_size; ++i)
+	    {
+		this->outbuf[i] ^= this->cbc_block[i];
+	    }
+	    memcpy(this->cbc_block, this->inbuf, this->buf_size);
+	}
     }
     unsigned int bytes = this->buf_size;
     if (strip_padding)

libqpdf/QPDF.cc

@@ -253,7 +253,8 @@ QPDF::QPDF() :
     ignore_xref_streams(false),
     suppress_warnings(false),
     attempt_recovery(true),
-    encryption_use_aes(false),
+    encryption_V(0),
+    encrypt_metadata(true),
     cached_key_objid(0),
     cached_key_generation(0),
     first_xref_item_offset(0),
@@ -1813,17 +1814,7 @@ QPDF::pipeStreamData(int objid, int generation,
     std::vector<PointerHolder<Pipeline> > to_delete;
     if (this->encrypted)
     {
-	bool xref_stream = false;
+	decryptStream(pipeline, objid, generation, stream_dict, to_delete);
-	if (stream_dict.getKey("/Type").isName() &&
-	    (stream_dict.getKey("/Type").getName() == "/XRef"))
-	{
-	    QTC::TC("qpdf", "QPDF piping xref stream from encrypted file");
-	    xref_stream = true;
-	}
-	if (! xref_stream)
-	{
-	    decryptStream(pipeline, objid, generation, to_delete);
-	}
     }
 
     try

libqpdf/QPDFWriter.cc

@@ -281,7 +281,8 @@ QPDFWriter::setEncryptionParameters(
     std::string O;
     std::string U;
     QPDF::compute_encryption_O_U(
-	user_password, owner_password, V, R, key_len, P, this->id1, O, U);
+	user_password, owner_password, V, R, key_len, P,
+	/*XXX encrypt_metadata*/true, this->id1, O, U);
     setEncryptionParametersInternal(
 	V, R, key_len, P, O, U, this->id1, user_password);
 }

libqpdf/QPDF_encryption.cc

@@ -5,11 +5,14 @@
 
 #include <qpdf/QPDFExc.hh>
 
+#include <qpdf/QTC.hh>
 #include <qpdf/QUtil.hh>
 #include <qpdf/Pl_RC4.hh>
+#include <qpdf/Pl_AES_PDF.hh>
 #include <qpdf/RC4.hh>
 #include <qpdf/MD5.hh>
 
+#include <assert.h>
 #include <string.h>
 
 static char const padding_string[] = {
@@ -123,9 +126,6 @@ QPDF::compute_data_key(std::string const& encryption_key,
     md5.digest(digest);
     return std::string((char*) digest,
 		       std::min(result.length(), (size_t) 16));
-
-    // XXX Item 4 in Algorithm 3.1 mentions CBC and a random number.
-    // We still have to incorporate that.
 }
 
 std::string
@@ -322,7 +322,8 @@ QPDF::initializeEncryption()
 		      "incorrect length");
     }
 
-    QPDFObjectHandle encryption_dict = this->trailer.getKey("/Encrypt");
+    this->encryption_dictionary = this->trailer.getKey("/Encrypt");
+    QPDFObjectHandle& encryption_dict = this->encryption_dictionary;
     if (! encryption_dict.isDictionary())
     {
 	throw QPDFExc(this->file.getName(), this->file.getLastOffset(),
@@ -360,12 +361,7 @@ QPDF::initializeEncryption()
 		      "Unsupported /R or /V in encryption dictionary");
     }
 
-    // XXX remove this check to continue implementing R4.
+    this->encryption_V = V;
-    if ((R == 4) || (V == 4))
-    {
-	throw QPDFExc(this->file.getName(), this->file.getLastOffset(),
-		      "PDF >= 1.5 encryption support is not fully implemented");
-    }
 
     if (! ((O.length() == key_bytes) && (U.length() == key_bytes)))
     {
@@ -385,19 +381,21 @@ QPDF::initializeEncryption()
 	}
     }
 
-    bool encrypt_metadata = true;
+    this->encrypt_metadata = true;
     if ((V >= 4) && (encryption_dict.getKey("/EncryptMetadata").isBool()))
     {
-	encrypt_metadata =
+	this->encrypt_metadata =
 	    encryption_dict.getKey("/EncryptMetadata").getBoolValue();
     }
-    // XXX not really...
+
-    if (R >= 4)
+    // XXX warn if /SubFilter is present
+    if (V == 4)
     {
-	this->encryption_use_aes = true;
+	// XXX get CF
     }
-    EncryptionData data(V, R, Length / 8, P, O, U, id1, encrypt_metadata);
+    EncryptionData data(V, R, Length / 8, P, O, U, id1, this->encrypt_metadata);
-    if (check_owner_password(this->user_password, this->provided_password, data))
+    if (check_owner_password(
+	    this->user_password, this->provided_password, data))
     {
 	// password supplied was owner password; user_password has
 	// been initialized
@@ -415,7 +413,7 @@ QPDF::initializeEncryption()
 }
 
 std::string
-QPDF::getKeyForObject(int objid, int generation)
+QPDF::getKeyForObject(int objid, int generation, bool use_aes)
 {
     if (! this->encrypted)
     {
@@ -427,8 +425,7 @@ QPDF::getKeyForObject(int objid, int generation)
 	   (generation == this->cached_key_generation)))
     {
 	this->cached_object_encryption_key =
-	    compute_data_key(this->encryption_key, objid, generation,
+	    compute_data_key(this->encryption_key, objid, generation, use_aes);
-			     this->encryption_use_aes);
 	this->cached_key_objid = objid;
 	this->cached_key_generation = generation;
     }
@@ -443,23 +440,62 @@ QPDF::decryptString(std::string& str, int objid, int generation)
     {
 	return;
     }
-    std::string key = getKeyForObject(objid, generation);
+    bool use_aes = false;	// XXX
-    char* tmp = QUtil::copy_string(str);
+    std::string key = getKeyForObject(objid, generation, use_aes);
-    unsigned int vlen = str.length();
+    if (use_aes)
-    RC4 rc4((unsigned char const*)key.c_str(), key.length());
+    {
-    rc4.process((unsigned char*)tmp, vlen);
+	// XXX
-    str = std::string(tmp, vlen);
+	throw std::logic_error("XXX");
-    delete [] tmp;
+    }
+    else
+    {
+	unsigned int vlen = str.length();
+	char* tmp = QUtil::copy_string(str);
+	RC4 rc4((unsigned char const*)key.c_str(), key.length());
+	rc4.process((unsigned char*)tmp, vlen);
+	str = std::string(tmp, vlen);
+	delete [] tmp;
+    }
 }
 
 void
 QPDF::decryptStream(Pipeline*& pipeline, int objid, int generation,
+		    QPDFObjectHandle& stream_dict,
 		    std::vector<PointerHolder<Pipeline> >& heap)
 {
-    std::string key = getKeyForObject(objid, generation);
+    bool decrypt = true;
-    if (this->encryption_use_aes)
+    std::string type;
+    if (stream_dict.getKey("/Type").isName())
     {
-	throw std::logic_error("aes not yet implemented"); // XXX
+	type = stream_dict.getKey("/Type").getName();
+    }
+    if (type == "/XRef")
+    {
+	QTC::TC("qpdf", "QPDF piping xref stream from encrypted file");
+	decrypt = false;
+    }
+    bool use_aes = false;
+    if (this->encryption_V == 4)
+    {
+	if ((! this->encrypt_metadata) && (type == "/Metadata"))
+	{
+	    // XXX no test case for this
+	    decrypt = false;
+	}
+	// XXX check crypt filter; if not found, use StmF; see TODO
+	use_aes = true;		// XXX
+    }
+    if (! decrypt)
+    {
+	return;
+    }
+
+    std::string key = getKeyForObject(objid, generation, use_aes);
+    if (use_aes)
+    {
+	assert(key.length() == Pl_AES_PDF::key_size);
+	pipeline = new Pl_AES_PDF("AES stream decryption", pipeline,
+				  false, (unsigned char*) key.c_str());
     }
     else
     {
@@ -472,11 +508,10 @@ QPDF::decryptStream(Pipeline*& pipeline, int objid, int generation,
 void
 QPDF::compute_encryption_O_U(
     char const* user_password, char const* owner_password,
-    int V, int R, int key_len, int P,
+    int V, int R, int key_len, int P, bool encrypt_metadata,
     std::string const& id1, std::string& O, std::string& U)
 {
-    EncryptionData data(V, R, key_len, P, "", "", id1,
+    EncryptionData data(V, R, key_len, P, "", "", id1, encrypt_metadata);
-			/*XXX encrypt_metadata*/true);
     data.O = compute_O_value(user_password, owner_password, data);
     O = data.O;
     U = compute_U_value(user_password, data);

libqpdf/qpdf/Pl_AES_PDF.hh

@@ -18,17 +18,24 @@ class DLL_EXPORT Pl_AES_PDF: public Pipeline
     virtual void write(unsigned char* data, int len);
     virtual void finish();
 
+    // For testing only; PDF always uses CBC
+    void disableCBC();
+
   private:
     void flush(bool discard_padding);
+    void initializeVector();
 
     static unsigned int const buf_size = 16;
 
     bool encrypt;
+    bool cbc_mode;
+    bool first;
     unsigned int offset;
     unsigned char key[key_size];
     uint32_t rk[key_size + 28];
     unsigned char inbuf[buf_size];
     unsigned char outbuf[buf_size];
+    unsigned char cbc_block[buf_size];
     unsigned int nrounds;
 };