diff --git a/src/components/Map.js b/src/components/Map.js
index 517cbfc..b0b1f01 100644
--- a/src/components/Map.js
+++ b/src/components/Map.js
@@ -2,37 +2,60 @@ import React from "react";
 import { Image, Flex, Box } from "theme-ui";
 
 import Token from "../components/Token";
+import ProxyToken from "../components/ProxyToken";
+
+const mapTokenClassName = "map-token";
+
+function Map({ imageSource, tokens, onMapTokenMove, onMapTokenRemove }) {
+  function handleProxyDragEnd(isOnMap, token) {
+    if (isOnMap && onMapTokenMove) {
+      onMapTokenMove(token);
+    }
+
+    if (!isOnMap && onMapTokenRemove) {
+      onMapTokenRemove(token);
+    }
+  }
 
-function Map({ imageSource, tokens }) {
   return (
-    <Flex
-      className="map"
-      sx={{ justifyContent: "center", flexGrow: 1 }}
-      bg="background"
-    >
-      <Box
-        sx={{
-          position: "absolute",
-          top: 0,
-          left: 0,
-          right: 0,
-          userSelect: "none"
-        }}
+    <>
+      <Flex
+        className="map"
+        sx={{ justifyContent: "center", flexGrow: 1 }}
+        bg="background"
       >
-        {Object.values(tokens).map(token => (
-          <Box
-            key={token.id}
-            sx={{
-              position: "absolute",
-              transform: `translate(${token.x}px, ${token.y}px)`
-            }}
-          >
-            <Token image={token.image} className="map-token" />
-          </Box>
-        ))}
-      </Box>
-      <Image src={imageSource} sx={{ objectFit: "contain" }} />
-    </Flex>
+        <Box
+          sx={{
+            position: "absolute",
+            top: 0,
+            left: 0,
+            right: 0,
+            userSelect: "none"
+          }}
+        >
+          {Object.values(tokens).map(token => (
+            <Box
+              key={token.id}
+              sx={{
+                position: "absolute",
+                transform: `translate(${token.x}px, ${token.y}px)`
+              }}
+            >
+              <Token
+                tokenId={token.id}
+                image={token.image}
+                className={mapTokenClassName}
+              />
+            </Box>
+          ))}
+        </Box>
+        <Image src={imageSource} sx={{ objectFit: "contain" }} />
+      </Flex>
+      <ProxyToken
+        tokenClassName={mapTokenClassName}
+        onProxyDragEnd={handleProxyDragEnd}
+      />
+    </>
   );
 }
 
diff --git a/src/components/ProxyToken.js b/src/components/ProxyToken.js
index 18f0069..069fdae 100644
--- a/src/components/ProxyToken.js
+++ b/src/components/ProxyToken.js
@@ -67,12 +67,14 @@ function ProxyToken({ tokenClassName, onProxyDragEnd }) {
           let proxy = imageRef.current;
           if (proxy) {
             if (onProxyDragEnd) {
-              const endX = parseFloat(proxy.getAttribute("data-x")) || 0;
-              const endY = parseFloat(proxy.getAttribute("data-y")) || 0;
+              const x = parseFloat(proxy.getAttribute("data-x")) || 0;
+              const y = parseFloat(proxy.getAttribute("data-y")) || 0;
+              const id = target.getAttribute("data-token-id");
               onProxyDragEnd(proxyOnMap.current, {
                 image: imageSource,
-                x: endX,
-                y: endY
+                x,
+                y,
+                id
               });
             }
 
diff --git a/src/components/Token.js b/src/components/Token.js
index 2037760..317efb8 100644
--- a/src/components/Token.js
+++ b/src/components/Token.js
@@ -1,7 +1,9 @@
 import React from "react";
 import { Image } from "theme-ui";
 
-function Token({ image, className }) {
+function Token({ image, className, tokenId }) {
+  // Store the token id in the html element for the drag code to pick it up
+  const idProp = tokenId ? { "data-token-id": tokenId } : {};
   return (
     <Image
       p={2}
@@ -11,6 +13,7 @@ function Token({ image, className }) {
         width: "64px",
         height: "64px"
       }}
+      {...idProp}
     />
   );
 }
diff --git a/src/components/Tokens.js b/src/components/Tokens.js
index 663d233..9d3f39c 100644
--- a/src/components/Tokens.js
+++ b/src/components/Tokens.js
@@ -7,13 +7,13 @@ import * as tokens from "../tokens";
 import Token from "./Token";
 import ProxyToken from "./ProxyToken";
 
-function Tokens({ onCreateMapToken }) {
-  const tokenClassName = "list-token";
+const listTokenClassName = "list-token";
 
+function Tokens({ onCreateMapToken }) {
   function handleProxyDragEnd(isOnMap, token) {
     if (isOnMap && onCreateMapToken) {
       // Give the token an id
-      onCreateMapToken({ id: shortid.generate(), ...token });
+      onCreateMapToken({ ...token, id: shortid.generate() });
     }
   }
 
@@ -30,11 +30,11 @@ function Tokens({ onCreateMapToken }) {
         px={2}
       >
         {Object.entries(tokens).map(([id, image]) => (
-          <Token key={id} image={image} className={tokenClassName} />
+          <Token key={id} image={image} className={listTokenClassName} />
         ))}
       </Flex>
       <ProxyToken
-        tokenClassName={tokenClassName}
+        tokenClassName={listTokenClassName}
         onProxyDragEnd={handleProxyDragEnd}
       />
     </>
diff --git a/src/helpers/shared.js b/src/helpers/shared.js
new file mode 100644
index 0000000..1e641cb
--- /dev/null
+++ b/src/helpers/shared.js
@@ -0,0 +1,10 @@
+export function omit(obj, keys) {
+  let tmp = {};
+  for (let [key, value] of Object.entries(obj)) {
+    if (keys.includes(key)) {
+      continue;
+    }
+    tmp[key] = value;
+  }
+  return tmp;
+}
diff --git a/src/routes/Game.js b/src/routes/Game.js
index edbf9d9..3afd2a2 100644
--- a/src/routes/Game.js
+++ b/src/routes/Game.js
@@ -7,6 +7,8 @@ import React, {
 } from "react";
 import { Box, Flex } from "theme-ui";
 
+import { omit } from "../helpers/shared";
+
 import GameContext from "../contexts/GameContext";
 import useSession from "../helpers/useSession";
 
@@ -43,14 +45,25 @@ function Game() {
 
   const [mapTokens, setMapTokens] = useState({});
 
-  function handleCreateMapToken(token) {
+  function handleEditMapToken(token) {
     setMapTokens(prevMapTokens => ({
       ...prevMapTokens,
       [token.id]: token
     }));
     for (let connection of Object.values(connections)) {
       const data = { [token.id]: token };
-      connection.send({ id: "token", data });
+      connection.send({ id: "tokenEdit", data });
+    }
+  }
+
+  function handleRemoveMapToken(token) {
+    setMapTokens(prevMapTokens => {
+      const { [token.id]: old, ...rest } = prevMapTokens;
+      return rest;
+    });
+    for (let connection of Object.values(connections)) {
+      const data = { [token.id]: token };
+      connection.send({ id: "tokenRemove", data });
     }
   }
 
@@ -61,12 +74,17 @@ function Game() {
         imageDataRef.current = blob;
         setImageSource(URL.createObjectURL(imageDataRef.current));
       }
-      if (data.id === "token") {
+      if (data.id === "tokenEdit") {
         setMapTokens(prevMapTokens => ({
           ...prevMapTokens,
           ...data.data
         }));
       }
+      if (data.id === "tokenRemove") {
+        setMapTokens(prevMapTokens =>
+          omit(prevMapTokens, Object.keys(data.data))
+        );
+      }
     });
   }
 
@@ -74,7 +92,7 @@ function Game() {
     if (imageSource) {
       connection.send({ id: "image", data: imageDataRef.current });
     }
-    connection.send({ id: "token", data: mapTokens });
+    connection.send({ id: "tokenEdit", data: mapTokens });
   }
 
   return (
@@ -94,8 +112,13 @@ function Game() {
         sx={{ justifyContent: "space-between", flexGrow: 1, height: "100%" }}
       >
         <Party streams={streams} localStreamId={peerId} />
-        <Map imageSource={imageSource} tokens={mapTokens} />
-        <Tokens onCreateMapToken={handleCreateMapToken} />
+        <Map
+          imageSource={imageSource}
+          tokens={mapTokens}
+          onMapTokenMove={handleEditMapToken}
+          onMapTokenRemove={handleRemoveMapToken}
+        />
+        <Tokens onCreateMapToken={handleEditMapToken} />
       </Flex>
     </Flex>
   );