From b45420a98038829e4d8d505d4332edc14caa340f Mon Sep 17 00:00:00 2001
From: m-holger <m-holger@kubitscheck.org>
Date: Wed, 24 Aug 2022 18:53:34 +0100
Subject: [PATCH] Remove QPDFTokenizer::unread_char

---
 include/qpdf/QPDFTokenizer.hh |  1 -
 libqpdf/QPDFTokenizer.cc      | 21 ++++++++++-----------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/include/qpdf/QPDFTokenizer.hh b/include/qpdf/QPDFTokenizer.hh
index d7acacd4..33b2e710 100644
--- a/include/qpdf/QPDFTokenizer.hh
+++ b/include/qpdf/QPDFTokenizer.hh
@@ -260,7 +260,6 @@ class QPDFTokenizer
     std::string error_message;
     bool before_token;
     bool in_token;
-    bool unread_char;
     char char_to_unread;
     size_t inline_image_bytes;
     bool bad;
diff --git a/libqpdf/QPDFTokenizer.cc b/libqpdf/QPDFTokenizer.cc
index d0cff1e4..07f6bfb7 100644
--- a/libqpdf/QPDFTokenizer.cc
+++ b/libqpdf/QPDFTokenizer.cc
@@ -83,7 +83,6 @@ QPDFTokenizer::reset()
     error_message = "";
     before_token = true;
     in_token = false;
-    unread_char = false;
     char_to_unread = '\0';
     inline_image_bytes = 0;
     string_depth = 0;
@@ -138,7 +137,7 @@ QPDFTokenizer::presentCharacter(char ch)
 {
     handleCharacter(ch);
 
-    if (this->in_token && !this->unread_char) {
+    if (this->in_token) { //} && !this->unread_char) {
         this->raw_val += ch;
     }
 }
@@ -370,7 +369,7 @@ QPDFTokenizer::inSpace(char ch)
     // We only enter this state if include_ignorable is true.
     if (!isSpace(ch)) {
         this->type = tt_space;
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
         this->state = st_token_ready;
         return;
@@ -386,7 +385,7 @@ QPDFTokenizer::inComment(char ch)
     if ((ch == '\r') || (ch == '\n')) {
         if (this->include_ignorable) {
             this->type = tt_comment;
-            this->unread_char = true;
+            this->in_token = false;
             this->char_to_unread = ch;
             this->state = st_token_ready;
         } else {
@@ -449,7 +448,7 @@ QPDFTokenizer::inName(char ch)
         // writing.
 
         this->type = this->bad ? tt_bad : tt_name;
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
         this->state = st_token_ready;
     } else if (ch == '#') {
@@ -561,7 +560,7 @@ QPDFTokenizer::inNumber(char ch)
     } else if (isDelimiter(ch)) {
         this->type = tt_integer;
         this->state = st_token_ready;
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
     } else {
         this->state = st_literal;
@@ -577,7 +576,7 @@ QPDFTokenizer::inReal(char ch)
     } else if (isDelimiter(ch)) {
         this->type = tt_real;
         this->state = st_token_ready;
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
     } else {
         this->state = st_literal;
@@ -672,7 +671,7 @@ QPDFTokenizer::inGt(char ch)
         this->type = tt_bad;
         QTC::TC("qpdf", "QPDFTokenizer bad >");
         this->error_message = "unexpected >";
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
         this->state = st_token_ready;
     }
@@ -690,7 +689,7 @@ QPDFTokenizer::inLiteral(char ch)
         // though not on any files in the test suite as of this
         // writing.
 
-        this->unread_char = true;
+        this->in_token = false;
         this->char_to_unread = ch;
         this->state = st_token_ready;
         this->type = (this->val == "true") || (this->val == "false")
@@ -809,7 +808,7 @@ QPDFTokenizer::presentEOF()
         // Push any delimiter to the state machine to finish off the final
         // token.
         presentCharacter('\f');
-        this->unread_char = false;
+        this->in_token = true;
         break;
 
     case st_top:
@@ -949,7 +948,7 @@ bool
 QPDFTokenizer::getToken(Token& token, bool& unread_char, char& ch)
 {
     bool ready = (this->state == st_token_ready);
-    unread_char = this->unread_char;
+    unread_char = !this->in_token && !this->before_token;
     ch = this->char_to_unread;
     if (ready) {
         if (this->type == tt_bad) {