diff --git a/include/qpdf/QPDF.hh b/include/qpdf/QPDF.hh
index 469bad55..701c5e55 100644
--- a/include/qpdf/QPDF.hh
+++ b/include/qpdf/QPDF.hh
@@ -851,12 +851,6 @@ class QPDF
         {
             return qpdf->resolve(og);
         }
-        static bool
-        objectChanged(
-            QPDF* qpdf, QPDFObjGen const& og, std::shared_ptr<QPDFObject>& oph)
-        {
-            return qpdf->objectChanged(og, oph);
-        }
     };
     friend class Resolver;
 
@@ -1174,7 +1168,6 @@ class QPDF
         std::string const& description,
         QPDFObjGen const& exp_og,
         QPDFObjGen& og);
-    bool objectChanged(QPDFObjGen const& og, std::shared_ptr<QPDFObject>& oph);
     std::shared_ptr<QPDFObject> resolve(QPDFObjGen const& og);
     void resolveObjectsInStream(int obj_stream_number);
     void stopOnError(std::string const& message);
@@ -1729,7 +1722,6 @@ class QPDF
         bool in_parse;
         bool parsed;
         std::set<int> resolved_object_streams;
-        bool ever_replaced_objects;
 
         // Linearization data
         qpdf_offset_t first_xref_item_offset; // actual value from file
diff --git a/libqpdf/QPDF.cc b/libqpdf/QPDF.cc
index 22f62a8b..000dab5d 100644
--- a/libqpdf/QPDF.cc
+++ b/libqpdf/QPDF.cc
@@ -223,7 +223,6 @@ QPDF::Members::Members() :
     immediate_copy_from(false),
     in_parse(false),
     parsed(false),
-    ever_replaced_objects(false),
     first_xref_item_offset(0),
     uncompressed_after_compressed(false)
 {
@@ -1928,28 +1927,6 @@ QPDF::readObjectAtOffset(
     return oh;
 }
 
-bool
-QPDF::objectChanged(QPDFObjGen const& og, std::shared_ptr<QPDFObject>& oph)
-{
-    // See if the object cached at og, if any, is the one passed in.
-    // QPDFObjectHandle uses this to detect outdated handles to
-    // replaced or swapped objects. This is a somewhat expensive check
-    // because it happens with every dereference of a
-    // QPDFObjectHandle. To reduce the hit somewhat, short-circuit the
-    // check if we never called a function that replaces an object
-    // already in cache. It is important for functions that do this to
-    // set ever_replaced_objects = true.
-
-    if (!this->m->ever_replaced_objects) {
-        return false;
-    }
-    auto c = this->m->obj_cache.find(og);
-    if (c == this->m->obj_cache.end()) {
-        return true;
-    }
-    return (c->second.object.get() != oph.get());
-}
-
 std::shared_ptr<QPDFObject>
 QPDF::resolve(QPDFObjGen const& og)
 {
@@ -2207,14 +2184,12 @@ QPDF::replaceObject(QPDFObjGen const& og, QPDFObjectHandle oh)
         throw std::logic_error(
             "QPDF::replaceObject called with indirect object handle");
     }
-
     // Force new object to appear in the cache
     resolve(og);
 
     // Replace the object in the object cache
-    this->m->ever_replaced_objects = true;
-    this->m->obj_cache[og] =
-        ObjCache(QPDFObjectHandle::ObjAccessor::getObject(oh), -1, -1);
+    m->obj_cache[og].object->assign(
+        QPDFObjectHandle::ObjAccessor::getObject(oh));
 }
 
 void
@@ -2536,10 +2511,7 @@ QPDF::swapObjects(QPDFObjGen const& og1, QPDFObjGen const& og2)
     // cache.
     resolve(og1);
     resolve(og2);
-    ObjCache t = this->m->obj_cache[og1];
-    this->m->ever_replaced_objects = true;
-    this->m->obj_cache[og1] = this->m->obj_cache[og2];
-    this->m->obj_cache[og2] = t;
+    m->obj_cache[og1].object->swapWith(m->obj_cache[og2].object);
 }
 
 unsigned long long
diff --git a/libqpdf/QPDFObjectHandle.cc b/libqpdf/QPDFObjectHandle.cc
index 72a35390..32988b53 100644
--- a/libqpdf/QPDFObjectHandle.cc
+++ b/libqpdf/QPDFObjectHandle.cc
@@ -2611,9 +2611,7 @@ QPDFObjectHandle::dereference()
     if (!this->initialized) {
         return false;
     }
-    if ((this->obj->getTypeCode() == QPDFObject::ot_unresolved) ||
-        (getObjectID() &&
-         QPDF::Resolver::objectChanged(this->qpdf, getObjGen(), this->obj))) {
+    if (this->obj->getTypeCode() == QPDFObject::ot_unresolved) {
         this->obj = QPDF::Resolver::resolve(this->qpdf, getObjGen());
     }
     return true;