diff --git a/package.json b/package.json
index b038cdb..094e8a7 100644
--- a/package.json
+++ b/package.json
@@ -15,6 +15,7 @@
     "ammo.js": "kripken/ammo.js#aab297a4164779c3a9d8dc8d9da26958de3cb778",
     "case": "^1.6.3",
     "comlink": "^4.3.0",
+    "deep-diff": "^1.0.2",
     "dexie": "^3.0.3",
     "err-code": "^2.0.3",
     "fake-indexeddb": "^3.1.2",
diff --git a/src/helpers/useNetworkedState.js b/src/helpers/useNetworkedState.js
index 2997747..3878492 100644
--- a/src/helpers/useNetworkedState.js
+++ b/src/helpers/useNetworkedState.js
@@ -1,4 +1,5 @@
 import { useEffect, useState, useRef, useCallback } from "react";
+import { applyChange, diff } from "deep-diff";
 
 import useDebounce from "./useDebounce";
 
@@ -31,8 +32,18 @@ function useNetworkedState(
     }
   }, [session.socket, eventName, debouncedState]);
 
+  // Store the uncommitted changes so we can re-apply them when receiving new data
+  const uncommittedChangesRef = useRef();
+  useEffect(() => {
+    uncommittedChangesRef.current = diff(debouncedState, state);
+  }, [state, debouncedState]);
+
   useEffect(() => {
     function handleSocketEvent(data) {
+      const uncommittedChanges = uncommittedChangesRef.current || [];
+      for (let change of uncommittedChanges) {
+        applyChange(data, undefined, change);
+      }
       _setState(data);
     }
 
diff --git a/yarn.lock b/yarn.lock
index ffd0ea8..b34bfbd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4421,6 +4421,11 @@ dedent@^0.7.0:
   resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"
   integrity sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=
 
+deep-diff@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/deep-diff/-/deep-diff-1.0.2.tgz#afd3d1f749115be965e89c63edc7abb1506b9c26"
+  integrity sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==
+
 deep-equal@^1.0.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.1.tgz#b5c98c942ceffaf7cb051e24e1434a25a2e6076a"