diff --git a/exwm-workspace.el b/exwm-workspace.el
index 09112d2..7ac1fec 100644
--- a/exwm-workspace.el
+++ b/exwm-workspace.el
@@ -168,12 +168,14 @@ The optional FORCE option is for internal use only."
   (unless id (setq id (exwm--buffer->id (window-buffer))))
   (unless (and (<= 0 index) (< index exwm-workspace-number))
     (user-error "[EXWM] Workspace index out of range: %d" index))
-  (when (/= exwm-workspace-current-index index)
+  (with-current-buffer (exwm--id->buffer id)
     (let ((frame (elt exwm-workspace--list index)))
-      (with-current-buffer (exwm--id->buffer id)
+      (when (not (equal exwm--frame frame))
+        (let ((name (replace-regexp-in-string "^\\s-*" "" (buffer-name))))
+          (exwm-workspace-rename-buffer (if (= index exwm-workspace-current-index)
+                                            name
+                                          (concat " " name))))
         (setq exwm--frame frame)
-        (exwm-workspace-rename-buffer
-         (concat " " (replace-regexp-in-string "^\\s-*" "" (buffer-name))))
         (if exwm--floating-frame
             ;; Move the floating frame is enough
             (progn