diff --git a/include/qpdf/QPDF.hh b/include/qpdf/QPDF.hh
index a2cc2993..8bb7120a 100644
--- a/include/qpdf/QPDF.hh
+++ b/include/qpdf/QPDF.hh
@@ -836,6 +836,7 @@ class QPDF
     void optimize(
         QPDFWriter::ObjTable const& obj,
         std::function<int(QPDFObjectHandle&)> skip_stream_parameters);
+    void optimize(Xref_table const& obj);
     size_t tableSize();
 
     // Get lists of all objects in order according to the part of a linearized file that they belong
@@ -935,6 +936,7 @@ class QPDF
     QPDFObjectHandle
     getUncompressedObject(QPDFObjectHandle&, std::map<int, int> const& object_stream_data);
     QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, QPDFWriter::ObjTable const& obj);
+    QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, Xref_table const& obj);
     int lengthNextN(int first_object, int n);
     void
     checkHPageOffset(std::vector<QPDFObjectHandle> const& pages, std::map<int, int>& idx_to_obj);
@@ -980,6 +982,7 @@ class QPDF
         std::function<int(QPDFObjectHandle&)> skip_stream_parameters);
     void filterCompressedObjects(std::map<int, int> const& object_stream_data);
     void filterCompressedObjects(QPDFWriter::ObjTable const& object_stream_data);
+    void filterCompressedObjects(Xref_table const& object_stream_data);
 
     // JSON import
     void importJSON(std::shared_ptr<InputSource>, bool must_be_complete);
diff --git a/libqpdf/QPDF_linearization.cc b/libqpdf/QPDF_linearization.cc
index 5295363c..89527fb9 100644
--- a/libqpdf/QPDF_linearization.cc
+++ b/libqpdf/QPDF_linearization.cc
@@ -484,18 +484,9 @@ QPDF::checkLinearizationInternal()
     // Further checking requires optimization and order calculation. Don't allow optimization to
     // make changes.  If it has to, then the file is not properly linearized.  We use the xref table
     // to figure out which objects are compressed and which are uncompressed.
-    { // local scope
-        std::map<int, int> object_stream_data;
-        for (auto const& iter: m->xref_table.as_map()) {
-            QPDFObjGen const& og = iter.first;
-            QPDFXRefEntry const& entry = iter.second;
-            if (entry.getType() == 2) {
-                object_stream_data[og.getObj()] = entry.getObjStreamNumber();
-            }
-        }
-        optimize(object_stream_data, false);
-        calculateLinearizationData(object_stream_data);
-    }
+
+    optimize(m->xref_table);
+    calculateLinearizationData(m->xref_table);
 
     // E: offset of end of first page -- Implementation note 123 says Acrobat includes on extra
     // object here by mistake.  pdlin fails to place thumbnail images in section 9, so when
@@ -581,6 +572,16 @@ QPDF::getUncompressedObject(QPDFObjectHandle& obj, std::map<int, int> const& obj
     }
 }
 
+QPDFObjectHandle
+QPDF::getUncompressedObject(QPDFObjectHandle& obj, Xref_table const& xref)
+{
+    auto og = obj.getObjGen();
+    if (obj.isNull() || xref.type(og) != 2) {
+        return obj;
+    }
+    return getObject(xref.stream_number(og.getObj()), 0);
+}
+
 QPDFObjectHandle
 QPDF::getUncompressedObject(QPDFObjectHandle& oh, QPDFWriter::ObjTable const& obj)
 {
diff --git a/libqpdf/QPDF_optimization.cc b/libqpdf/QPDF_optimization.cc
index c5586947..3dcaa360 100644
--- a/libqpdf/QPDF_optimization.cc
+++ b/libqpdf/QPDF_optimization.cc
@@ -78,6 +78,12 @@ QPDF::optimize(
     optimize_internal(obj, true, skip_stream_parameters);
 }
 
+void
+QPDF::optimize(QPDF::Xref_table const& xref)
+{
+    optimize_internal(xref, false, nullptr);
+}
+
 template <typename T>
 void
 QPDF::optimize_internal(
@@ -442,3 +448,45 @@ QPDF::filterCompressedObjects(QPDFWriter::ObjTable const& obj)
     m->obj_user_to_objects = t_obj_user_to_objects;
     m->object_to_obj_users = t_object_to_obj_users;
 }
+
+void
+QPDF::filterCompressedObjects(QPDF::Xref_table const& xref)
+{
+    if (!xref.object_streams()) {
+        return;
+    }
+
+    // Transform object_to_obj_users and obj_user_to_objects so that they refer only to uncompressed
+    // objects.  If something is a user of a compressed object, then it is really a user of the
+    // object stream that contains it.
+
+    std::map<ObjUser, std::set<QPDFObjGen>> t_obj_user_to_objects;
+    std::map<QPDFObjGen, std::set<ObjUser>> t_object_to_obj_users;
+
+    for (auto const& i1: m->obj_user_to_objects) {
+        ObjUser const& ou = i1.first;
+        // Loop over objects.
+        for (auto const& og: i1.second) {
+            if (auto stream = xref.stream_number(og.getObj())) {
+                t_obj_user_to_objects[ou].insert(QPDFObjGen(stream, 0));
+            } else {
+                t_obj_user_to_objects[ou].insert(og);
+            }
+        }
+    }
+
+    for (auto const& i1: m->object_to_obj_users) {
+        QPDFObjGen const& og = i1.first;
+        // Loop over obj_users.
+        for (auto const& ou: i1.second) {
+            if (auto stream = xref.stream_number(og.getObj())) {
+                t_object_to_obj_users[QPDFObjGen(stream, 0)].insert(ou);
+            } else {
+                t_object_to_obj_users[og].insert(ou);
+            }
+        }
+    }
+
+    m->obj_user_to_objects = t_obj_user_to_objects;
+    m->object_to_obj_users = t_object_to_obj_users;
+}