diff --git a/inline-diff.el b/inline-diff.el
index fb9da3a..2d02998 100644
--- a/inline-diff.el
+++ b/inline-diff.el
@@ -335,7 +335,7 @@ Optionally:
   "Remove the parent or child overlay OV."
   (pcase (overlay-get ov 'category)
     ('inline-diff-overlay
-     (let ((linked (overlay-get ov 'linked))
+     (let ((linked (overlay-get ov 'inline-diff-linked))
            (parent (overlay-get ov 'parent)))
        (delete-overlay ov)
        (dolist (link linked)
@@ -349,12 +349,18 @@ Optionally:
        (delete-overlay child))
      (delete-overlay ov))))
 
-(defun inline-diff--overlays-within (start stop)
-  "Return a list of overlays within the region from START to STOP."
-  (let (overlays)
+(defun inline-diff--overlays-within (start stop &optional skip-linked)
+  "Return a list of difference overlays within the region from START to STOP.
+When SKIP-LINKED is non-nil, linked overlays are skipped over."
+  (let (overlays skippable-links)
     (dolist (ov (overlays-in start stop))
       (when (eq (overlay-get ov 'category) 'inline-diff-overlay)
-        (push ov overlays)))
+        (if (not skip-linked)
+            (push ov overlays)
+          (unless (memq ov skippable-links)
+            (push ov overlays)
+            (when-let ((linked (inline-diff--get-linked ov)))
+          (setq skippable-links (nconc skippable-links linked)))))))
     (nreverse overlays)))
 
 (defun inline-diff--overlay-at (&optional pt)
@@ -447,8 +453,8 @@ position of an insert which is the very first entry of DIFF."
     (dolist (op-ov diff-ovs)
       (when (and (eq (caar last-op-ov) 'delete)
                  (eq (caar op-ov) 'insert))
-        (push (cdr op-ov) (overlay-get (cdr last-op-ov) 'linked))
-        (push (cdr last-op-ov) (overlay-get (cdr op-ov) 'linked))
+        (push (cdr op-ov) (overlay-get (cdr last-op-ov) 'inline-diff-linked))
+        (push (cdr last-op-ov) (overlay-get (cdr op-ov) 'inline-diff-linked))
         (overlay-put (cdr last-op-ov) 'after-string
                      (propertize inline-diff-change-joiner
                                  'face '(shadow inline-diff-removed))))
@@ -458,7 +464,7 @@ position of an insert which is the very first entry of DIFF."
   "Get the linked overlays from OV, recursively.
 SO-FAR is a list of overlays already seen."
   (let ((first (not so-far)))
-    (dolist (link (overlay-get ov 'linked))
+    (dolist (link (overlay-get ov 'inline-diff-linked))
       (unless (or (not (overlay-buffer link)) (memq link so-far))
         (push link so-far)
         (inline-diff--get-linked link so-far)))
@@ -569,7 +575,7 @@ This takes into account whether or not OV has a display string."
                                 (+ (cdr range) offset))))
           (overlay-put hl 'category 'inline-diff-refinement-overlay)
           (overlay-put hl 'face face)
-          (push hl (overlay-get ov 'linked)))))))
+          (push hl (overlay-get ov 'inline-diff-linked)))))))
 
 ;;;; Public functions
 
@@ -714,9 +720,9 @@ May move `point' when CAN-MOVE is non-nil."
                      (overlay-get ov 'parent))
             (let ((parent (overlay-get ov 'parent)))
               (setq ovs (inline-diff--overlays-within
-                         (overlay-start parent) (overlay-end parent)))))))
+                         (overlay-start parent) (overlay-end parent) t))))))
       (unless ovs
-        (setq ovs (inline-diff--overlays-within (point-min) (point-max))))
+        (setq ovs (inline-diff--overlays-within (point-min) (point-max) t)))
       (when ovs
         (let ((ov-slice ovs)
               (o 0))