diff --git a/src/core/document.js b/src/core/document.js
index 7e1adf0a68316..53a238cabb424 100644
--- a/src/core/document.js
+++ b/src/core/document.js
@@ -327,44 +327,45 @@ class Page {
   async #replaceIdByRef(annotations, deletedAnnotations, existingAnnotations) {
     const promises = [];
     for (const annotation of annotations) {
-      if (annotation.id) {
-        const ref = Ref.fromString(annotation.id);
-        if (!ref) {
-          warn(`A non-linked annotation cannot be modified: ${annotation.id}`);
-          continue;
-        }
-        if (annotation.deleted) {
-          deletedAnnotations.put(ref, ref);
-          if (annotation.popupRef) {
-            const popupRef = Ref.fromString(annotation.popupRef);
-            if (popupRef) {
-              deletedAnnotations.put(popupRef, popupRef);
-            }
-          }
-          continue;
-        }
-        if (annotation.popup?.deleted) {
+      if (!annotation.id) {
+        continue;
+      }
+      const ref = Ref.fromString(annotation.id);
+      if (!ref) {
+        warn(`A non-linked annotation cannot be modified: ${annotation.id}`);
+        continue;
+      }
+      if (annotation.deleted) {
+        deletedAnnotations.put(ref);
+        if (annotation.popupRef) {
           const popupRef = Ref.fromString(annotation.popupRef);
           if (popupRef) {
-            deletedAnnotations.put(popupRef, popupRef);
+            deletedAnnotations.put(popupRef);
           }
         }
-        existingAnnotations?.put(ref);
-        annotation.ref = ref;
-        promises.push(
-          this.xref.fetchAsync(ref).then(
-            obj => {
-              if (obj instanceof Dict) {
-                annotation.oldAnnotation = obj.clone();
-              }
-            },
-            () => {
-              warn(`Cannot fetch \`oldAnnotation\` for: ${ref}.`);
-            }
-          )
-        );
-        delete annotation.id;
+        continue;
+      }
+      if (annotation.popup?.deleted) {
+        const popupRef = Ref.fromString(annotation.popupRef);
+        if (popupRef) {
+          deletedAnnotations.put(popupRef);
+        }
       }
+      existingAnnotations?.put(ref);
+      annotation.ref = ref;
+      promises.push(
+        this.xref.fetchAsync(ref).then(
+          obj => {
+            if (obj instanceof Dict) {
+              annotation.oldAnnotation = obj.clone();
+            }
+          },
+          () => {
+            warn(`Cannot fetch \`oldAnnotation\` for: ${ref}.`);
+          }
+        )
+      );
+      delete annotation.id;
     }
     await Promise.all(promises);
   }
@@ -375,7 +376,7 @@ class Page {
     }
     const partialEvaluator = this.#createPartialEvaluator(handler);
 
-    const deletedAnnotations = new RefMap();
+    const deletedAnnotations = new RefSet();
     const existingAnnotations = new RefSet();
     await this.#replaceIdByRef(
       annotations,