octoleo/qpdf
2
1
Fork 0
mirror of https://github.com/qpdf/qpdf.git synced 2025-01-23 07:08:30 +00:00
Code Issues Releases Wiki Activity

Use space tokens in tokenizer filter

Browse Source
This commit is contained in:
Jay Berkenbilt 2018-01-29 21:16:56 -05:00
parent ec538792fa
commit ba453ba4ff
2 changed files with 36 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
libqpdf/Pl_QPDFTokenizer.cc
View File

@ -8,12 +8,13 @@
Pl_QPDFTokenizer::Pl_QPDFTokenizer(char const* identifier, Pipeline* next) : Pl_QPDFTokenizer::Pl_QPDFTokenizer(char const* identifier, Pipeline* next) :
Pipeline(identifier, next), Pipeline(identifier, next),
newline_after_next_token(false),
just_wrote_nl(false), just_wrote_nl(false),
last_char_was_cr(false), last_char_was_cr(false),
unread_char(false), unread_char(false),
char_to_unread('\0') char_to_unread('\0')
{ {
tokenizer.allowEOF();
tokenizer.includeIgnorable();
} }
Pl_QPDFTokenizer::~Pl_QPDFTokenizer() Pl_QPDFTokenizer::~Pl_QPDFTokenizer()
@ -37,8 +38,35 @@ Pl_QPDFTokenizer::writeToken(QPDFTokenizer::Token& token)
switch (token.getType()) switch (token.getType())
{ {
case QPDFTokenizer::tt_space:
{
size_t len = value.length();
for (size_t i = 0; i < len; ++i)
{
char ch = value.at(i);
if (ch == '\r')
{
if ((i + 1 < len) && (value.at(i + 1) == '\n'))
{
// ignore
}
else
{
writeNext("\n", 1);
}
}
else
{
writeNext(&ch, 1);
}
}
}
value.clear();
break;
case QPDFTokenizer::tt_string: case QPDFTokenizer::tt_string:
value = QPDF_String(token.getValue()).unparse(); value = QPDF_String(token.getValue()).unparse();
break; break;
case QPDFTokenizer::tt_name: case QPDFTokenizer::tt_name:
@ -59,10 +87,14 @@ Pl_QPDFTokenizer::processChar(char ch)
if (tokenizer.getToken(token, this->unread_char, this->char_to_unread)) if (tokenizer.getToken(token, this->unread_char, this->char_to_unread))
{ {
writeToken(token); writeToken(token);
if (this->newline_after_next_token) std::string value = token.getRawValue();
{ QPDFTokenizer::token_type_e token_type = token.getType();
if (((token_type == QPDFTokenizer::tt_string) ||
(token_type == QPDFTokenizer::tt_name)) &&
((value.find('\r') != std::string::npos) ||
(value.find('\n') != std::string::npos)))
{
writeNext("\n", 1); writeNext("\n", 1);
this->newline_after_next_token = false;
} }
if ((token.getType() == QPDFTokenizer::tt_word) && if ((token.getType() == QPDFTokenizer::tt_word) &&
(token.getValue() == "ID")) (token.getValue() == "ID"))
@ -71,35 +103,6 @@ Pl_QPDFTokenizer::processChar(char ch)
tokenizer.expectInlineImage(); tokenizer.expectInlineImage();
} }
} }
else
{
bool suppress = false;
if ((ch == '\n') && (this->last_char_was_cr))
{
// Always ignore \n following \r
suppress = true;
}
if ((this->last_char_was_cr = (ch == '\r')))
{
ch = '\n';
}
if (this->tokenizer.betweenTokens())
{
if (! suppress)
{
writeNext(&ch, 1);
}
}
else
{
if (ch == '\n')
{
this->newline_after_next_token = true;
}
}
}
} }

1
libqpdf/qpdf/Pl_QPDFTokenizer.hh
View File

@ -28,7 +28,6 @@ class Pl_QPDFTokenizer: public Pipeline
void writeToken(QPDFTokenizer::Token&); void writeToken(QPDFTokenizer::Token&);
QPDFTokenizer tokenizer; QPDFTokenizer tokenizer;
bool newline_after_next_token;
bool just_wrote_nl; bool just_wrote_nl;
bool last_char_was_cr; bool last_char_was_cr;
bool unread_char; bool unread_char;