diff --git a/src/components/map/Map.js b/src/components/map/Map.js
index 46e2609..8459ab6 100644
--- a/src/components/map/Map.js
+++ b/src/components/map/Map.js
@@ -33,7 +33,7 @@ function Map({
   onFogDrawRedo,
   allowMapDrawing,
   allowFogDrawing,
-  allowTokenChange,
+  disabledTokens,
 }) {
   const mapSource = useDataSource(map, defaultMapSources);
 
@@ -302,20 +302,18 @@ function Map({
         {map && mapFog}
         {map && mapTokens}
       </MapInteraction>
-      {allowTokenChange && (
-        <>
-          <ProxyToken
-            tokenClassName={mapTokenProxyClassName}
-            onProxyDragEnd={handleProxyDragEnd}
-            tokens={mapState && mapState.tokens}
-          />
-          <TokenMenu
-            tokenClassName={mapTokenMenuClassName}
-            onTokenChange={onMapTokenStateChange}
-            tokens={mapState && mapState.tokens}
-          />
-        </>
-      )}
+      <ProxyToken
+        tokenClassName={mapTokenProxyClassName}
+        onProxyDragEnd={handleProxyDragEnd}
+        tokens={mapState && mapState.tokens}
+        disabledTokens={disabledTokens}
+      />
+      <TokenMenu
+        tokenClassName={mapTokenMenuClassName}
+        onTokenChange={onMapTokenStateChange}
+        tokens={mapState && mapState.tokens}
+        disabledTokens={disabledTokens}
+      />
     </>
   );
 }
diff --git a/src/components/token/ProxyToken.js b/src/components/token/ProxyToken.js
index 08e26c7..a690bec 100644
--- a/src/components/token/ProxyToken.js
+++ b/src/components/token/ProxyToken.js
@@ -19,8 +19,15 @@ import TokenStatus from "./TokenStatus";
  * @param {string} tokenClassName The class name to attach the interactjs handler to
  * @param {onProxyDragEnd} onProxyDragEnd Called when the proxy token is dropped
  * @param {Object} tokens An optional mapping of tokens to use as a base when calling OnProxyDragEnd
+ * @param {Object} disabledTokens An optional mapping of tokens that shouldn't allow movement
+
  */
-function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
+function ProxyToken({
+  tokenClassName,
+  onProxyDragEnd,
+  tokens,
+  disabledTokens,
+}) {
   const proxyContainer = usePortal("root");
 
   const [imageSource, setImageSource] = useState("");
@@ -30,9 +37,11 @@ function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
   // Store the tokens in a ref and access in the interactjs loop
   // This is needed to stop interactjs from creating multiple listeners
   const tokensRef = useRef(tokens);
+  const disabledTokensRef = useRef(disabledTokens);
   useEffect(() => {
     tokensRef.current = tokens;
-  }, [tokens]);
+    disabledTokensRef.current = disabledTokens;
+  }, [tokens, disabledTokens]);
 
   const proxyOnMap = useRef(false);
 
@@ -41,10 +50,15 @@ function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
       listeners: {
         start: (event) => {
           let target = event.target;
+          const id = target.dataset.id;
+          if (id in disabledTokensRef.current) {
+            return;
+          }
+
           // Hide the token and copy it's image to the proxy
           target.parentElement.style.opacity = "0.25";
           setImageSource(target.src);
-          setTokenId(target.dataset.id);
+          setTokenId(id);
 
           let proxy = proxyRef.current;
           if (proxy) {
@@ -90,6 +104,10 @@ function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
 
         end: (event) => {
           let target = event.target;
+          const id = target.dataset.id;
+          if (id in disabledTokensRef.current) {
+            return;
+          }
           let proxy = proxyRef.current;
           if (proxy) {
             if (onProxyDragEnd) {
@@ -106,7 +124,6 @@ function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
               y = y / (mapImageRect.bottom - mapImageRect.top);
 
               // Get the token from the supplied tokens if it exists
-              const id = target.getAttribute("data-id");
               const token = tokensRef.current[id] || {};
 
               onProxyDragEnd(proxyOnMap.current, {
@@ -172,6 +189,7 @@ function ProxyToken({ tokenClassName, onProxyDragEnd, tokens }) {
 
 ProxyToken.defaultProps = {
   tokens: {},
+  disabledTokens: {},
 };
 
 export default ProxyToken;
diff --git a/src/components/token/TokenMenu.js b/src/components/token/TokenMenu.js
index 74c4eb2..d64063a 100644
--- a/src/components/token/TokenMenu.js
+++ b/src/components/token/TokenMenu.js
@@ -16,8 +16,9 @@ import colors, { colorOptions } from "../../helpers/colors";
  * @param {string} tokenClassName The class name to attach the interactjs handler to
  * @param {onProxyDragEnd} onTokenChange Called when the the token data is changed
  * @param {Object} tokens An mapping of tokens to use as a base when calling onTokenChange
+ * @param {Object} disabledTokens An optional mapping of tokens that shouldn't allow interaction
  */
-function TokenMenu({ tokenClassName, onTokenChange, tokens }) {
+function TokenMenu({ tokenClassName, onTokenChange, tokens, disabledTokens }) {
   const [isOpen, setIsOpen] = useState(false);
 
   function handleRequestClose() {
@@ -27,9 +28,11 @@ function TokenMenu({ tokenClassName, onTokenChange, tokens }) {
   // Store the tokens in a ref and access in the interactjs loop
   // This is needed to stop interactjs from creating multiple listeners
   const tokensRef = useRef(tokens);
+  const disabledTokensRef = useRef(disabledTokens);
   useEffect(() => {
     tokensRef.current = tokens;
-  }, [tokens]);
+    disabledTokensRef.current = disabledTokens;
+  }, [tokens, disabledTokens]);
 
   const [currentToken, setCurrentToken] = useState({});
   const [menuLeft, setMenuLeft] = useState(0);
@@ -67,6 +70,9 @@ function TokenMenu({ tokenClassName, onTokenChange, tokens }) {
     function handleTokenMenuOpen(event) {
       const target = event.target;
       const id = target.getAttribute("data-id");
+      if (id in disabledTokensRef.current) {
+        return;
+      }
       const token = tokensRef.current[id] || {};
       setCurrentToken(token);
 
diff --git a/src/routes/Game.js b/src/routes/Game.js
index 348b21d..5f70a1a 100644
--- a/src/routes/Game.js
+++ b/src/routes/Game.js
@@ -54,10 +54,21 @@ function Game() {
     mapState !== null &&
     (mapState.editFlags.includes("fog") || map.owner === userId);
 
-  const canEditTokens =
-    map !== null &&
+  const disabledMapTokens = {};
+  // If we have a map and state and have the token permission disabled
+  // and are not the map owner
+  if (
     mapState !== null &&
-    (mapState.editFlags.includes("tokens") || map.owner === userId);
+    map !== null &&
+    !mapState.editFlags.includes("tokens") &&
+    map.owner !== userId
+  ) {
+    for (let token of Object.values(mapState.tokens)) {
+      if (token.owner !== userId) {
+        disabledMapTokens[token.id] = true;
+      }
+    }
+  }
 
   // Sync the map state to the database after 500ms of inactivity
   const debouncedMapState = useDebounce(mapState, 500);
@@ -472,7 +483,7 @@ function Game() {
             onFogDrawRedo={handleFogDrawRedo}
             allowMapDrawing={canEditMapDrawing}
             allowFogDrawing={canEditFogDrawing}
-            allowTokenChange={canEditTokens}
+            disabledTokens={disabledMapTokens}
           />
           <Tokens
             tokens={tokens}