diff --git a/src/components/map/Map.js b/src/components/map/Map.js
index b213f8f..edc1d01 100644
--- a/src/components/map/Map.js
+++ b/src/components/map/Map.js
@@ -29,6 +29,9 @@ function Map({
   onMapDraw,
   onMapDrawUndo,
   onMapDrawRedo,
+  allowDrawing,
+  allowTokenChange,
+  allowMapChange,
 }) {
   const mapSource = useDataSource(map, defaultMapSources);
 
@@ -305,41 +308,49 @@ function Map({
             {map && mapTokens}
           </Box>
         </Box>
-        <MapControls
-          onMapChange={onMapChange}
-          onMapStateChange={onMapStateChange}
-          currentMap={map}
-          onToolChange={setSelectedTool}
-          selectedTool={selectedTool}
-          disabledTools={disabledTools}
-          onUndo={onMapDrawUndo}
-          onRedo={onMapDrawRedo}
-          undoDisabled={!mapState || mapState.drawActionIndex < 0}
-          redoDisabled={
-            !mapState ||
-            mapState.drawActionIndex === mapState.drawActions.length - 1
-          }
-          brushColor={brushColor}
-          onBrushColorChange={setBrushColor}
-          onEraseAll={handleShapeRemoveAll}
-          useBrushGridSnapping={useBrushGridSnapping}
-          onBrushGridSnappingChange={setUseBrushGridSnapping}
-          useBrushBlending={useBrushBlending}
-          onBrushBlendingChange={setUseBrushBlending}
-          useBrushGesture={useBrushGesture}
-          onBrushGestureChange={setUseBrushGesture}
-        />
+        {(allowMapChange || allowDrawing) && (
+          <MapControls
+            onMapChange={onMapChange}
+            onMapStateChange={onMapStateChange}
+            currentMap={map}
+            onToolChange={setSelectedTool}
+            selectedTool={selectedTool}
+            disabledTools={disabledTools}
+            onUndo={onMapDrawUndo}
+            onRedo={onMapDrawRedo}
+            undoDisabled={!mapState || mapState.drawActionIndex < 0}
+            redoDisabled={
+              !mapState ||
+              mapState.drawActionIndex === mapState.drawActions.length - 1
+            }
+            brushColor={brushColor}
+            onBrushColorChange={setBrushColor}
+            onEraseAll={handleShapeRemoveAll}
+            useBrushGridSnapping={useBrushGridSnapping}
+            onBrushGridSnappingChange={setUseBrushGridSnapping}
+            useBrushBlending={useBrushBlending}
+            onBrushBlendingChange={setUseBrushBlending}
+            useBrushGesture={useBrushGesture}
+            onBrushGestureChange={setUseBrushGesture}
+            allowDrawing={allowDrawing}
+            allowMapChange={allowMapChange}
+          />
+        )}
       </Box>
-      <ProxyToken
-        tokenClassName={mapTokenProxyClassName}
-        onProxyDragEnd={handleProxyDragEnd}
-        tokens={mapState && mapState.tokens}
-      />
-      <TokenMenu
-        tokenClassName={mapTokenMenuClassName}
-        onTokenChange={onMapTokenStateChange}
-        tokens={mapState && mapState.tokens}
-      />
+      {allowTokenChange && (
+        <>
+          <ProxyToken
+            tokenClassName={mapTokenProxyClassName}
+            onProxyDragEnd={handleProxyDragEnd}
+            tokens={mapState && mapState.tokens}
+          />
+          <TokenMenu
+            tokenClassName={mapTokenMenuClassName}
+            onTokenChange={onMapTokenStateChange}
+            tokens={mapState && mapState.tokens}
+          />
+        </>
+      )}
     </>
   );
 }
diff --git a/src/components/map/MapControls.js b/src/components/map/MapControls.js
index e50de3a..196056d 100644
--- a/src/components/map/MapControls.js
+++ b/src/components/map/MapControls.js
@@ -1,5 +1,5 @@
 import React, { useState, useEffect, useRef } from "react";
-import { Flex, Box, IconButton, Label } from "theme-ui";
+import { Flex, Box, IconButton, Label, Divider } from "theme-ui";
 
 import SelectMapButton from "./SelectMapButton";
 import ExpandMoreIcon from "../../icons/ExpandMoreIcon";
@@ -40,6 +40,8 @@ function MapControls({
   onBrushBlendingChange,
   useBrushGesture,
   onBrushGestureChange,
+  allowDrawing,
+  allowMapChange,
 }) {
   const [isExpanded, setIsExpanded] = useState(false);
 
@@ -198,49 +200,24 @@ function MapControls({
 
   const expanedMenuRef = useRef();
 
-  return (
-    <>
-      <Flex
-        sx={{
-          position: "absolute",
-          top: 0,
-          right: 0,
-          flexDirection: "column",
-          alignItems: "center",
-        }}
-        mx={1}
-      >
-        <IconButton
-          aria-label={isExpanded ? "Hide Map Controls" : "Show Map Controls"}
-          title={isExpanded ? "Hide Map Controls" : "Show Map Controls"}
-          onClick={() => setIsExpanded(!isExpanded)}
-          sx={{
-            transform: `rotate(${isExpanded ? "0" : "180deg"})`,
-            display: "block",
-            backgroundColor: "overlay",
-            borderRadius: "50%",
-          }}
-          m={2}
-        >
-          <ExpandMoreIcon />
-        </IconButton>
-        <Box
-          sx={{
-            flexDirection: "column",
-            alignItems: "center",
-            display: isExpanded ? "flex" : "none",
-            backgroundColor: "overlay",
-            borderRadius: "4px",
-          }}
-          p={2}
-          ref={expanedMenuRef}
-        >
-          <SelectMapButton
-            onMapChange={onMapChange}
-            onMapStateChange={onMapStateChange}
-            currentMap={currentMap}
-          />
-          {divider}
+  const sections = [];
+  if (allowMapChange) {
+    sections.push({
+      id: "map",
+      component: (
+        <SelectMapButton
+          onMapChange={onMapChange}
+          onMapStateChange={onMapStateChange}
+          currentMap={currentMap}
+        />
+      ),
+    });
+  }
+  if (allowDrawing) {
+    sections.push({
+      id: "drawing",
+      component: (
+        <>
           <IconButton
             aria-label="Pan Tool"
             title="Pan Tool"
@@ -285,7 +262,78 @@ function MapControls({
           >
             <RedoIcon />
           </IconButton>
+        </>
+      ),
+    });
+  }
+
+  let controls = null;
+  if (sections.length === 1 && sections[0].id === "map") {
+    controls = (
+      <Box
+        sx={{
+          display: "block",
+          backgroundColor: "overlay",
+          borderRadius: "4px",
+        }}
+        m={2}
+      >
+        {sections[0].component}
+      </Box>
+    );
+  } else if (sections.length > 0) {
+    controls = (
+      <>
+        <IconButton
+          aria-label={isExpanded ? "Hide Map Controls" : "Show Map Controls"}
+          title={isExpanded ? "Hide Map Controls" : "Show Map Controls"}
+          onClick={() => setIsExpanded(!isExpanded)}
+          sx={{
+            transform: `rotate(${isExpanded ? "0" : "180deg"})`,
+            display: "block",
+            backgroundColor: "overlay",
+            borderRadius: "50%",
+          }}
+          m={2}
+        >
+          <ExpandMoreIcon />
+        </IconButton>
+
+        <Box
+          sx={{
+            flexDirection: "column",
+            alignItems: "center",
+            display: isExpanded ? "flex" : "none",
+            backgroundColor: "overlay",
+            borderRadius: "4px",
+          }}
+          p={2}
+          ref={expanedMenuRef}
+        >
+          {sections.map((section, index) => (
+            <>
+              {section.component}
+              {index !== sections.length - 1 && <Divider />}
+            </>
+          ))}
         </Box>
+      </>
+    );
+  }
+
+  return (
+    <>
+      <Flex
+        sx={{
+          position: "absolute",
+          top: 0,
+          right: 0,
+          flexDirection: "column",
+          alignItems: "center",
+        }}
+        mx={1}
+      >
+        {controls}
       </Flex>
       <MapMenu
         isOpen={!!currentSubmenu}
diff --git a/src/components/map/MapSettings.js b/src/components/map/MapSettings.js
index 1e7d0fd..79b25c0 100644
--- a/src/components/map/MapSettings.js
+++ b/src/components/map/MapSettings.js
@@ -79,6 +79,14 @@ function MapSettings({
             />
             Tokens
           </Label>
+          <Label>
+            <Checkbox
+              checked={mapState !== null && mapState.editFlags.includes("map")}
+              disabled={mapState === null}
+              onChange={(e) => handleFlagChange(e, "map")}
+            />
+            Map
+          </Label>
         </Flex>
       </Box>
     </Flex>
diff --git a/src/modals/SelectMapModal.js b/src/modals/SelectMapModal.js
index 295ac4d..1cc6370 100644
--- a/src/modals/SelectMapModal.js
+++ b/src/modals/SelectMapModal.js
@@ -20,7 +20,7 @@ const defaultMapState = {
   drawActionIndex: -1,
   drawActions: [],
   // Flags to determine what other people can edit
-  editFlags: ["drawings", "tokens"],
+  editFlags: ["map", "drawings", "tokens"],
 };
 
 const defaultMapProps = {
diff --git a/src/routes/Game.js b/src/routes/Game.js
index 879cb09..4c99a5a 100644
--- a/src/routes/Game.js
+++ b/src/routes/Game.js
@@ -43,6 +43,22 @@ function Game() {
   const [map, setMap] = useState(null);
   const [mapState, setMapState] = useState(null);
 
+  const canChangeMap =
+    map === null ||
+    (map !== null &&
+      mapState !== null &&
+      (mapState.editFlags.includes("map") || map.owner === userId));
+
+  const canEditMapDrawings =
+    map !== null &&
+    mapState !== null &&
+    (mapState.editFlags.includes("drawings") || map.owner === userId);
+
+  const canEditTokens =
+    map !== null &&
+    mapState !== null &&
+    (mapState.editFlags.includes("tokens") || map.owner === userId);
+
   // Sync the map state to the database after 500ms of inactivity
   const debouncedMapState = useDebounce(mapState, 500);
   useEffect(() => {
@@ -361,11 +377,16 @@ function Game() {
             onMapDraw={handleMapDraw}
             onMapDrawUndo={handleMapDrawUndo}
             onMapDrawRedo={handleMapDrawRedo}
+            allowDrawing={canEditMapDrawings}
+            allowTokenChange={canEditTokens}
+            allowMapChange={canChangeMap}
           />
-          <Tokens
-            tokens={tokens}
-            onCreateMapTokenState={handleMapTokenStateChange}
-          />
+          {canEditTokens && (
+            <Tokens
+              tokens={tokens}
+              onCreateMapTokenState={handleMapTokenStateChange}
+            />
+          )}
         </Flex>
       </Flex>
       <Banner isOpen={!!peerError} onRequestClose={() => setPeerError(null)}>