diff --git a/src/components/token/Tokens.js b/src/components/token/Tokens.js
index becf119..5fde471 100644
--- a/src/components/token/Tokens.js
+++ b/src/components/token/Tokens.js
@@ -1,4 +1,4 @@
-import React, { useState } from "react";
+import React, { useState, useContext } from "react";
 import { Box } from "theme-ui";
 import shortid from "shortid";
 import SimpleBar from "simplebar-react";
@@ -9,10 +9,13 @@ import NumberInput from "../NumberInput";
 
 import { fromEntries } from "../../helpers/shared";
 
+import AuthContext from "../../contexts/AuthContext";
+
 const listTokenClassName = "list-token";
 
 function Tokens({ onCreateMapTokenState, tokens }) {
   const [tokenSize, setTokenSize] = useState(1);
+  const { userId } = useContext(AuthContext);
 
   function handleProxyDragEnd(isOnMap, token) {
     if (isOnMap && onCreateMapTokenState) {
@@ -20,6 +23,7 @@ function Tokens({ onCreateMapTokenState, tokens }) {
       onCreateMapTokenState({
         id: shortid.generate(),
         tokenId: token.id,
+        owner: userId,
         size: tokenSize,
         label: "",
         statuses: [],
diff --git a/src/database.js b/src/database.js
index 338a97f..c1ad236 100644
--- a/src/database.js
+++ b/src/database.js
@@ -2,9 +2,9 @@ import Dexie from "dexie";
 
 const db = new Dexie("OwlbearRodeoDB");
 db.version(1).stores({
-  maps: "id",
+  maps: "id, owner",
   states: "mapId",
-  tokens: "id",
+  tokens: "id, owner",
   user: "key",
 });
 
diff --git a/src/modals/SelectMapModal.js b/src/modals/SelectMapModal.js
index 980edb8..0b2281a 100644
--- a/src/modals/SelectMapModal.js
+++ b/src/modals/SelectMapModal.js
@@ -67,7 +67,8 @@ function SelectMapModal({
           id,
           owner: userId,
           // Emulate the time increasing to avoid sort errors
-          timestamp: Date.now() + i,
+          created: Date.now() + i,
+          lastModified: Date.now() + i,
           ...defaultMapProps,
         });
         // Add a state for the map if there isn't one already
@@ -80,12 +81,16 @@ function SelectMapModal({
     }
 
     async function loadMaps() {
-      let storedMaps = await db.table("maps").toArray();
+      let storedMaps = await db
+        .table("maps")
+        .where({ owner: userId })
+        .toArray();
       const defaultMapsWithIds = await getDefaultMaps();
       const sortedMaps = [...defaultMapsWithIds, ...storedMaps].sort(
-        (a, b) => a.timestamp - b.timestamp
+        (a, b) => a.created - b.created
       );
       setMaps(sortedMaps);
+      // TODO: Does this work with default maps?
       if (selectedMap) {
         const map = await db.table("maps").get(selectedMap.id);
         const state = await db.table("states").get(selectedMap.id);
@@ -147,7 +152,8 @@ function SelectMapModal({
         width: image.width,
         height: image.height,
         id: shortid.generate(),
-        timestamp: Date.now(),
+        created: Date.now(),
+        lastModified: Date.now(),
         owner: userId,
         ...defaultMapProps,
       });
@@ -246,8 +252,9 @@ function SelectMapModal({
   const [showMoreSettings, setShowMoreSettings] = useState(false);
 
   async function handleMapSettingsChange(key, value) {
-    db.table("maps").update(selectedMap.id, { [key]: value });
-    const newMap = { ...selectedMap, [key]: value };
+    const change = { [key]: value, lastModified: Date.now() };
+    db.table("maps").update(selectedMap.id, change);
+    const newMap = { ...selectedMap, ...change };
     setMaps((prevMaps) => {
       const newMaps = [...prevMaps];
       const i = newMaps.findIndex((map) => map.id === selectedMap.id);
diff --git a/src/routes/Game.js b/src/routes/Game.js
index e2ee57e..348b21d 100644
--- a/src/routes/Game.js
+++ b/src/routes/Game.js
@@ -79,7 +79,18 @@ function Game() {
       // Clear the map so the new map state isn't shown on an old map
       peer.connection.send({ id: "map", data: null });
       peer.connection.send({ id: "mapState", data: newMapState });
-      peer.connection.send({ id: "map", data: newMap });
+      sendMapDataToPeer(peer, newMap);
+    }
+  }
+
+  function sendMapDataToPeer(peer, mapData) {
+    // Omit file from map change, receiver will request the file if
+    // they have an outdated version
+    if (mapData.type === "file") {
+      const { file, ...rest } = mapData;
+      peer.connection.send({ id: "map", data: rest });
+    } else {
+      peer.connection.send({ id: "map", data: mapData });
     }
   }
 
@@ -237,14 +248,42 @@ function Game() {
         peer.connection.send({ id: "mapState", data: mapState });
       }
       if (map) {
-        peer.connection.send({ id: "map", data: map });
+        sendMapDataToPeer(peer, map);
       }
     }
     if (data.id === "map") {
+      const newMap = data.data;
+      // If is a file map check cache and request the full file if outdated
+      if (newMap && newMap.type === "file") {
+        db.table("maps")
+          .get(newMap.id)
+          .then((cachedMap) => {
+            if (cachedMap && cachedMap.lastModified === newMap.lastModified) {
+              setMap(cachedMap);
+            } else {
+              peer.connection.send({ id: "mapRequest" });
+            }
+          });
+      } else {
+        setMap(newMap);
+      }
+    }
+    // Send full map data including file
+    if (data.id === "mapRequest") {
+      peer.connection.send({ id: "mapResponse", data: map });
+    }
+    // A new map response with a file attached
+    if (data.id === "mapResponse") {
       if (data.data && data.data.type === "file") {
         // Convert file back to blob after peer transfer
         const file = new Blob([data.data.file]);
-        setMap({ ...data.data, file });
+        const newMap = { ...data.data, file };
+        // Store in db
+        db.table("maps")
+          .put(newMap)
+          .then(() => {
+            setMap(newMap);
+          });
       } else {
         setMap(data.data);
       }
@@ -387,15 +426,19 @@ function Game() {
    */
   const [tokens, setTokens] = useState([]);
   useEffect(() => {
+    if (!userId) {
+      return;
+    }
     const defaultTokensWithIds = [];
     for (let defaultToken of defaultTokens) {
       defaultTokensWithIds.push({
         ...defaultToken,
         id: `__default-${defaultToken.name}`,
+        owner: userId,
       });
     }
     setTokens(defaultTokensWithIds);
-  }, []);
+  }, [userId]);
 
   return (
     <>