diff --git a/connector/src/main/java/org/geysermc/connector/network/session/GeyserSession.java b/connector/src/main/java/org/geysermc/connector/network/session/GeyserSession.java
index 0a28b11f1..300877f81 100644
--- a/connector/src/main/java/org/geysermc/connector/network/session/GeyserSession.java
+++ b/connector/src/main/java/org/geysermc/connector/network/session/GeyserSession.java
@@ -729,6 +729,10 @@ public class GeyserSession implements CommandSender {
         // Required to make command blocks destroyable
         adventureSettingsPacket.setPlayerPermission(opPermissionLevel >= 2 ? PlayerPermission.OPERATOR : PlayerPermission.MEMBER);
 
+        // Update the noClip and worldImmutable values based on the current gamemode
+        noClip = gameMode == GameMode.SPECTATOR;
+        worldImmutable = gameMode == GameMode.ADVENTURE || gameMode == GameMode.SPECTATOR;
+
         Set<AdventureSetting> flags = new HashSet<>();
         if (canFly) {
             flags.add(AdventureSetting.MAY_FLY);
diff --git a/connector/src/main/java/org/geysermc/connector/network/translators/java/world/JavaNotifyClientTranslator.java b/connector/src/main/java/org/geysermc/connector/network/translators/java/world/JavaNotifyClientTranslator.java
index a7bc7b61f..493a7ca1e 100644
--- a/connector/src/main/java/org/geysermc/connector/network/translators/java/world/JavaNotifyClientTranslator.java
+++ b/connector/src/main/java/org/geysermc/connector/network/translators/java/world/JavaNotifyClientTranslator.java
@@ -103,8 +103,6 @@ public class JavaNotifyClientTranslator extends PacketTranslator<ServerNotifyCli
             case CHANGE_GAMEMODE:
                 GameMode gameMode = (GameMode) packet.getValue();
 
-                session.setNoClip(gameMode == GameMode.SPECTATOR);
-                session.setWorldImmutable(gameMode == GameMode.ADVENTURE || gameMode == GameMode.SPECTATOR);
                 session.sendAdventureSettings();
 
                 SetPlayerGameTypePacket playerGameTypePacket = new SetPlayerGameTypePacket();