diff --git a/src/core/NavigationOverlayHandler.ttslua b/src/core/NavigationOverlayHandler.ttslua
index e79f5184..66fe917b 100644
--- a/src/core/NavigationOverlayHandler.ttslua
+++ b/src/core/NavigationOverlayHandler.ttslua
@@ -189,7 +189,6 @@ end
 
 -- XML button creation
 function createXmlButtonHelper(ui, params)
-  local color
   local guid = self.getGUID()
   local xml = findTagWithId(ui, params.id)
 
@@ -338,8 +337,8 @@ function loadCamera(player, camera)
       end
     end
 
-    -- swap to that color if it isn't claimed by someone else
-    if #getSeatedPlayers() == 1 or not isClaimed then
+    -- swap to that color if it isn't claimed by someone else and it's currently unoccopied
+    if #getSeatedPlayers() == 1 or (not isClaimed and isPlaymatAvailable(matColor)) then
       local newPlayerColor = playmatApi.getPlayerColor(matColor)
       copyVisibility({ startColor = player.color, targetColor = newPlayerColor })
       player.changeColor(newPlayerColor)
@@ -372,6 +371,17 @@ function loadCamera(player, camera)
   Wait.frames(function() player.lookAt(lookHere) end, 2)
 end
 
+-- helper function to check if a playmat is available for a color swap
+function isPlaymatAvailable(matColor)
+  local newPlayerColor = playmatApi.getPlayerColor(matColor)
+  for _, color in ipairs(getSeatedPlayers()) do
+    if color == newPlayerColor then
+      return false
+    end
+  end
+  return true
+end
+
 ---------------------------------------------------------
 -- settings related functionality
 ---------------------------------------------------------