mirror of
https://github.com/qpdf/qpdf.git
synced 2024-12-22 19:08:59 +00:00
4a4736c695
CR, CRLF, and LF are all supposed to be treated as LF; only one EOL is to be ignored after backslash.
721 lines
17 KiB
C++
721 lines
17 KiB
C++
#include <qpdf/QPDFTokenizer.hh>
|
|
|
|
// DO NOT USE ctype -- it is locale dependent for some things, and
|
|
// it's not worth the risk of including it in case it may accidentally
|
|
// be used.
|
|
|
|
#include <qpdf/QTC.hh>
|
|
#include <qpdf/QPDFExc.hh>
|
|
#include <qpdf/QUtil.hh>
|
|
#include <qpdf/QPDFObjectHandle.hh>
|
|
|
|
#include <stdexcept>
|
|
#include <string.h>
|
|
#include <cstdlib>
|
|
|
|
QPDFTokenizer::Members::Members() :
|
|
pound_special_in_name(true),
|
|
allow_eof(false),
|
|
include_ignorable(false)
|
|
{
|
|
reset();
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::Members::reset()
|
|
{
|
|
state = st_top;
|
|
type = tt_bad;
|
|
val = "";
|
|
raw_val = "";
|
|
error_message = "";
|
|
unread_char = false;
|
|
char_to_unread = '\0';
|
|
string_depth = 0;
|
|
string_ignoring_newline = false;
|
|
last_char_was_bs = false;
|
|
last_char_was_cr = false;
|
|
}
|
|
|
|
QPDFTokenizer::Members::~Members()
|
|
{
|
|
}
|
|
|
|
QPDFTokenizer::Token::Token(token_type_e type, std::string const& value) :
|
|
type(type),
|
|
value(value),
|
|
raw_value(value)
|
|
{
|
|
if (type == tt_string)
|
|
{
|
|
raw_value = QPDFObjectHandle::newString(value).unparse();
|
|
}
|
|
else if (type == tt_name)
|
|
{
|
|
raw_value = QPDFObjectHandle::newName(value).unparse();
|
|
}
|
|
}
|
|
|
|
|
|
|
|
QPDFTokenizer::QPDFTokenizer() :
|
|
m(new Members())
|
|
{
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::allowPoundAnywhereInName()
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer allow pound anywhere in name");
|
|
this->m->pound_special_in_name = false;
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::allowEOF()
|
|
{
|
|
this->m->allow_eof = true;
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::includeIgnorable()
|
|
{
|
|
this->m->include_ignorable = true;
|
|
}
|
|
|
|
bool
|
|
QPDFTokenizer::isSpace(char ch)
|
|
{
|
|
return ((ch == '\0') || QUtil::is_space(ch));
|
|
}
|
|
|
|
bool
|
|
QPDFTokenizer::isDelimiter(char ch)
|
|
{
|
|
return (strchr(" \t\n\v\f\r()<>[]{}/%", ch) != 0);
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::resolveLiteral()
|
|
{
|
|
if ((this->m->val.length() > 0) && (this->m->val.at(0) == '/'))
|
|
{
|
|
this->m->type = tt_name;
|
|
// Deal with # in name token. Note: '/' by itself is a
|
|
// valid name, so don't strip leading /. That way we
|
|
// don't have to deal with the empty string as a name.
|
|
std::string nval = "/";
|
|
char const* valstr = this->m->val.c_str() + 1;
|
|
for (char const* p = valstr; *p; ++p)
|
|
{
|
|
if ((*p == '#') && this->m->pound_special_in_name)
|
|
{
|
|
if (p[1] && p[2] &&
|
|
QUtil::is_hex_digit(p[1]) && QUtil::is_hex_digit(p[2]))
|
|
{
|
|
char num[3];
|
|
num[0] = p[1];
|
|
num[1] = p[2];
|
|
num[2] = '\0';
|
|
char ch = static_cast<char>(strtol(num, 0, 16));
|
|
if (ch == '\0')
|
|
{
|
|
this->m->type = tt_bad;
|
|
QTC::TC("qpdf", "QPDFTokenizer null in name");
|
|
this->m->error_message =
|
|
"null character not allowed in name token";
|
|
nval += "#00";
|
|
}
|
|
else
|
|
{
|
|
nval += ch;
|
|
}
|
|
p += 2;
|
|
}
|
|
else
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer bad name");
|
|
this->m->type = tt_bad;
|
|
this->m->error_message = "invalid name token";
|
|
nval += *p;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
nval += *p;
|
|
}
|
|
}
|
|
this->m->val = nval;
|
|
}
|
|
else if (QUtil::is_number(this->m->val.c_str()))
|
|
{
|
|
if (this->m->val.find('.') != std::string::npos)
|
|
{
|
|
this->m->type = tt_real;
|
|
}
|
|
else
|
|
{
|
|
this->m->type = tt_integer;
|
|
}
|
|
}
|
|
else if ((this->m->val == "true") || (this->m->val == "false"))
|
|
{
|
|
this->m->type = tt_bool;
|
|
}
|
|
else if (this->m->val == "null")
|
|
{
|
|
this->m->type = tt_null;
|
|
}
|
|
else
|
|
{
|
|
// I don't really know what it is, so leave it as tt_word.
|
|
// Lots of cases ($, #, etc.) other than actual words fall
|
|
// into this category, but that's okay at least for now.
|
|
this->m->type = tt_word;
|
|
}
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::presentCharacter(char ch)
|
|
{
|
|
if (this->m->state == st_token_ready)
|
|
{
|
|
throw std::logic_error(
|
|
"INTERNAL ERROR: QPDF tokenizer presented character "
|
|
"while token is waiting");
|
|
}
|
|
|
|
char orig_ch = ch;
|
|
|
|
// State machine is implemented such that some characters may be
|
|
// handled more than once. This happens whenever you have to use
|
|
// the character that caused a state change in the new state.
|
|
|
|
bool handled = true;
|
|
if (this->m->state == st_top)
|
|
{
|
|
// Note: we specifically do not use ctype here. It is
|
|
// locale-dependent.
|
|
if (isSpace(ch))
|
|
{
|
|
if (this->m->include_ignorable)
|
|
{
|
|
this->m->state = st_in_space;
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else if (ch == '%')
|
|
{
|
|
this->m->state = st_in_comment;
|
|
if (this->m->include_ignorable)
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else if (ch == '(')
|
|
{
|
|
this->m->string_depth = 1;
|
|
this->m->string_ignoring_newline = false;
|
|
memset(this->m->bs_num_register, '\0',
|
|
sizeof(this->m->bs_num_register));
|
|
this->m->last_char_was_bs = false;
|
|
this->m->last_char_was_cr = false;
|
|
this->m->state = st_in_string;
|
|
}
|
|
else if (ch == '<')
|
|
{
|
|
this->m->state = st_lt;
|
|
}
|
|
else if (ch == '>')
|
|
{
|
|
this->m->state = st_gt;
|
|
}
|
|
else
|
|
{
|
|
this->m->val += ch;
|
|
if (ch == ')')
|
|
{
|
|
this->m->type = tt_bad;
|
|
QTC::TC("qpdf", "QPDFTokenizer bad )");
|
|
this->m->error_message = "unexpected )";
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else if (ch == '[')
|
|
{
|
|
this->m->type = tt_array_open;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else if (ch == ']')
|
|
{
|
|
this->m->type = tt_array_close;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else if (ch == '{')
|
|
{
|
|
this->m->type = tt_brace_open;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else if (ch == '}')
|
|
{
|
|
this->m->type = tt_brace_close;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->state = st_literal;
|
|
}
|
|
}
|
|
}
|
|
else if (this->m->state == st_in_space)
|
|
{
|
|
// We only enter this state if include_ignorable is true.
|
|
if (! isSpace(ch))
|
|
{
|
|
this->m->type = tt_space;
|
|
this->m->unread_char = true;
|
|
this->m->char_to_unread = ch;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else if (this->m->state == st_in_comment)
|
|
{
|
|
if ((ch == '\r') || (ch == '\n'))
|
|
{
|
|
if (this->m->include_ignorable)
|
|
{
|
|
this->m->type = tt_comment;
|
|
this->m->unread_char = true;
|
|
this->m->char_to_unread = ch;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->state = st_top;
|
|
}
|
|
}
|
|
else if (this->m->include_ignorable)
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else if (this->m->state == st_lt)
|
|
{
|
|
if (ch == '<')
|
|
{
|
|
this->m->val = "<<";
|
|
this->m->type = tt_dict_open;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
handled = false;
|
|
this->m->state = st_in_hexstring;
|
|
}
|
|
}
|
|
else if (this->m->state == st_gt)
|
|
{
|
|
if (ch == '>')
|
|
{
|
|
this->m->val = ">>";
|
|
this->m->type = tt_dict_close;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->val = ">";
|
|
this->m->type = tt_bad;
|
|
QTC::TC("qpdf", "QPDFTokenizer bad >");
|
|
this->m->error_message = "unexpected >";
|
|
this->m->unread_char = true;
|
|
this->m->char_to_unread = ch;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
}
|
|
else if (this->m->state == st_in_string)
|
|
{
|
|
if (this->m->string_ignoring_newline && (ch != '\n'))
|
|
{
|
|
this->m->string_ignoring_newline = false;
|
|
}
|
|
|
|
size_t bs_num_count = strlen(this->m->bs_num_register);
|
|
bool ch_is_octal = ((ch >= '0') && (ch <= '7'));
|
|
if ((bs_num_count == 3) || ((bs_num_count > 0) && (! ch_is_octal)))
|
|
{
|
|
// We've accumulated \ddd. PDF Spec says to ignore
|
|
// high-order overflow.
|
|
this->m->val += static_cast<char>(
|
|
strtol(this->m->bs_num_register, 0, 8));
|
|
memset(this->m->bs_num_register, '\0',
|
|
sizeof(this->m->bs_num_register));
|
|
bs_num_count = 0;
|
|
}
|
|
|
|
if (this->m->string_ignoring_newline && (ch == '\n'))
|
|
{
|
|
// ignore
|
|
this->m->string_ignoring_newline = false;
|
|
}
|
|
else if (ch_is_octal &&
|
|
(this->m->last_char_was_bs || (bs_num_count > 0)))
|
|
{
|
|
this->m->bs_num_register[bs_num_count++] = ch;
|
|
}
|
|
else if (this->m->last_char_was_bs)
|
|
{
|
|
switch (ch)
|
|
{
|
|
case 'n':
|
|
this->m->val += '\n';
|
|
break;
|
|
|
|
case 'r':
|
|
this->m->val += '\r';
|
|
break;
|
|
|
|
case 't':
|
|
this->m->val += '\t';
|
|
break;
|
|
|
|
case 'b':
|
|
this->m->val += '\b';
|
|
break;
|
|
|
|
case 'f':
|
|
this->m->val += '\f';
|
|
break;
|
|
|
|
case '\n':
|
|
break;
|
|
|
|
case '\r':
|
|
this->m->string_ignoring_newline = true;
|
|
break;
|
|
|
|
default:
|
|
// PDF spec says backslash is ignored before anything else
|
|
this->m->val += ch;
|
|
break;
|
|
}
|
|
}
|
|
else if (ch == '\\')
|
|
{
|
|
// last_char_was_bs is set/cleared below as appropriate
|
|
if (bs_num_count)
|
|
{
|
|
throw std::logic_error(
|
|
"INTERNAL ERROR: QPDFTokenizer: bs_num_count != 0 "
|
|
"when ch == '\\'");
|
|
}
|
|
}
|
|
else if (ch == '(')
|
|
{
|
|
this->m->val += ch;
|
|
++this->m->string_depth;
|
|
}
|
|
else if ((ch == ')') && (--this->m->string_depth == 0))
|
|
{
|
|
this->m->type = tt_string;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else if (ch == '\r')
|
|
{
|
|
// CR by itself is converted to LF
|
|
this->m->val += '\n';
|
|
}
|
|
else if (ch == '\n')
|
|
{
|
|
// CR LF is converted to LF
|
|
if (! this->m->last_char_was_cr)
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
|
|
this->m->last_char_was_cr =
|
|
((! this->m->string_ignoring_newline) && (ch == '\r'));
|
|
this->m->last_char_was_bs =
|
|
((! this->m->last_char_was_bs) && (ch == '\\'));
|
|
}
|
|
else if (this->m->state == st_literal)
|
|
{
|
|
if (isDelimiter(ch))
|
|
{
|
|
// A C-locale whitespace character or delimiter terminates
|
|
// token. It is important to unread the whitespace
|
|
// character even though it is ignored since it may be the
|
|
// newline after a stream keyword. Removing it here could
|
|
// make the stream-reading code break on some files,
|
|
// though not on any files in the test suite as of this
|
|
// writing.
|
|
|
|
this->m->type = tt_word;
|
|
this->m->unread_char = true;
|
|
this->m->char_to_unread = ch;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else if (this->m->state == st_inline_image)
|
|
{
|
|
size_t len = this->m->val.length();
|
|
if ((len >= 4) &&
|
|
isDelimiter(this->m->val.at(len-4)) &&
|
|
(this->m->val.at(len-3) == 'E') &&
|
|
(this->m->val.at(len-2) == 'I') &&
|
|
isDelimiter(this->m->val.at(len-1)))
|
|
{
|
|
this->m->type = tt_inline_image;
|
|
this->m->unread_char = true;
|
|
this->m->char_to_unread = ch;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
else
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
handled = false;
|
|
}
|
|
|
|
|
|
if (handled)
|
|
{
|
|
// okay
|
|
}
|
|
else if (this->m->state == st_in_hexstring)
|
|
{
|
|
if (ch == '>')
|
|
{
|
|
this->m->type = tt_string;
|
|
this->m->state = st_token_ready;
|
|
if (this->m->val.length() % 2)
|
|
{
|
|
// PDF spec says odd hexstrings have implicit
|
|
// trailing 0.
|
|
this->m->val += '0';
|
|
}
|
|
char num[3];
|
|
num[2] = '\0';
|
|
std::string nval;
|
|
for (unsigned int i = 0; i < this->m->val.length(); i += 2)
|
|
{
|
|
num[0] = this->m->val.at(i);
|
|
num[1] = this->m->val.at(i+1);
|
|
char nch = static_cast<char>(strtol(num, 0, 16));
|
|
nval += nch;
|
|
}
|
|
this->m->val = nval;
|
|
}
|
|
else if (QUtil::is_hex_digit(ch))
|
|
{
|
|
this->m->val += ch;
|
|
}
|
|
else if (isSpace(ch))
|
|
{
|
|
// ignore
|
|
}
|
|
else
|
|
{
|
|
this->m->type = tt_bad;
|
|
QTC::TC("qpdf", "QPDFTokenizer bad hexstring character");
|
|
this->m->error_message = std::string("invalid character (") +
|
|
ch + ") in hexstring";
|
|
this->m->state = st_token_ready;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw std::logic_error(
|
|
"INTERNAL ERROR: invalid state while reading token");
|
|
}
|
|
|
|
if ((this->m->state == st_token_ready) && (this->m->type == tt_word))
|
|
{
|
|
resolveLiteral();
|
|
}
|
|
|
|
if (! (betweenTokens() ||
|
|
((this->m->state == st_token_ready) && this->m->unread_char)))
|
|
{
|
|
this->m->raw_val += orig_ch;
|
|
}
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::presentEOF()
|
|
{
|
|
if (this->m->state == st_inline_image)
|
|
{
|
|
size_t len = this->m->val.length();
|
|
if ((len >= 3) &&
|
|
isDelimiter(this->m->val.at(len-3)) &&
|
|
(this->m->val.at(len-2) == 'E') &&
|
|
(this->m->val.at(len-1) == 'I'))
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer inline image at EOF");
|
|
this->m->type = tt_inline_image;
|
|
this->m->state = st_token_ready;
|
|
}
|
|
}
|
|
|
|
if (this->m->state == st_literal)
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer EOF reading appendable token");
|
|
resolveLiteral();
|
|
}
|
|
else if ((this->m->include_ignorable) && (this->m->state == st_in_space))
|
|
{
|
|
this->m->type = tt_space;
|
|
}
|
|
else if ((this->m->include_ignorable) && (this->m->state == st_in_comment))
|
|
{
|
|
this->m->type = tt_comment;
|
|
}
|
|
else if (betweenTokens())
|
|
{
|
|
this->m->type = tt_eof;
|
|
}
|
|
else if (this->m->state != st_token_ready)
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer EOF reading token");
|
|
this->m->type = tt_bad;
|
|
this->m->error_message = "EOF while reading token";
|
|
}
|
|
|
|
this->m->state = st_token_ready;
|
|
}
|
|
|
|
void
|
|
QPDFTokenizer::expectInlineImage()
|
|
{
|
|
if (this->m->state != st_top)
|
|
{
|
|
throw std::logic_error("QPDFTokenizer::expectInlineImage called"
|
|
" when tokenizer is in improper state");
|
|
}
|
|
this->m->state = st_inline_image;
|
|
}
|
|
|
|
bool
|
|
QPDFTokenizer::getToken(Token& token, bool& unread_char, char& ch)
|
|
{
|
|
bool ready = (this->m->state == st_token_ready);
|
|
unread_char = this->m->unread_char;
|
|
ch = this->m->char_to_unread;
|
|
if (ready)
|
|
{
|
|
if (this->m->type == tt_bad)
|
|
{
|
|
this->m->val = this->m->raw_val;
|
|
}
|
|
token = Token(this->m->type, this->m->val,
|
|
this->m->raw_val, this->m->error_message);
|
|
this->m->reset();
|
|
}
|
|
return ready;
|
|
}
|
|
|
|
bool
|
|
QPDFTokenizer::betweenTokens()
|
|
{
|
|
return ((this->m->state == st_top) ||
|
|
((! this->m->include_ignorable) &&
|
|
((this->m->state == st_in_comment) ||
|
|
(this->m->state == st_in_space))));
|
|
}
|
|
|
|
QPDFTokenizer::Token
|
|
QPDFTokenizer::readToken(PointerHolder<InputSource> input,
|
|
std::string const& context,
|
|
bool allow_bad,
|
|
size_t max_len)
|
|
{
|
|
qpdf_offset_t offset = input->tell();
|
|
Token token;
|
|
bool unread_char;
|
|
char char_to_unread;
|
|
bool presented_eof = false;
|
|
while (! getToken(token, unread_char, char_to_unread))
|
|
{
|
|
char ch;
|
|
if (input->read(&ch, 1) == 0)
|
|
{
|
|
if (! presented_eof)
|
|
{
|
|
presentEOF();
|
|
presented_eof = true;
|
|
if ((this->m->type == tt_eof) && (! this->m->allow_eof))
|
|
{
|
|
// Nothing in the qpdf library calls readToken
|
|
// without allowEOF anymore, so this case is not
|
|
// exercised.
|
|
this->m->type = tt_bad;
|
|
this->m->error_message = "unexpected EOF";
|
|
offset = input->getLastOffset();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw std::logic_error(
|
|
"getToken returned false after presenting EOF");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
presentCharacter(ch);
|
|
if (betweenTokens() && (input->getLastOffset() == offset))
|
|
{
|
|
++offset;
|
|
}
|
|
if (max_len && (this->m->raw_val.length() >= max_len) &&
|
|
(this->m->state != st_token_ready))
|
|
{
|
|
// terminate this token now
|
|
QTC::TC("qpdf", "QPDFTokenizer block long token");
|
|
this->m->type = tt_bad;
|
|
this->m->state = st_token_ready;
|
|
this->m->error_message =
|
|
"exceeded allowable length while reading token";
|
|
}
|
|
}
|
|
}
|
|
|
|
if (unread_char)
|
|
{
|
|
input->unreadCh(char_to_unread);
|
|
}
|
|
|
|
if (token.getType() != tt_eof)
|
|
{
|
|
input->setLastOffset(offset);
|
|
}
|
|
|
|
if (token.getType() == tt_bad)
|
|
{
|
|
if (allow_bad)
|
|
{
|
|
QTC::TC("qpdf", "QPDFTokenizer allowing bad token");
|
|
}
|
|
else
|
|
{
|
|
throw QPDFExc(qpdf_e_damaged_pdf, input->getName(),
|
|
context, offset, token.getErrorMessage());
|
|
}
|
|
}
|
|
|
|
return token;
|
|
}
|