diff --git a/tmk_core/common/action.c b/tmk_core/common/action.c
index acc6d11eab..bf609f5e7b 100644
--- a/tmk_core/common/action.c
+++ b/tmk_core/common/action.c
@@ -55,7 +55,7 @@ void action_exec(keyevent_t event)
 
 #if !defined(NO_ACTION_LAYER) && defined(PREVENT_STUCK_MODIFIERS)
 bool disable_action_cache = false;
-int8_t pressed_actions_cache[MATRIX_ROWS][MATRIX_COLS];
+uint8_t source_layers_cache[5][(MATRIX_ROWS * MATRIX_COLS + 7) / 8] = {0};
 
 void process_action_nocache(keyrecord_t *record)
 {
@@ -82,11 +82,23 @@ action_t store_or_get_action(bool pressed, keypos_t key)
     if (disable_action_cache) {
         return layer_switch_get_action(key);
     }
-
+    uint8_t key_number = key.col + (key.row * MATRIX_COLS);
+    uint8_t storage_row = key_number / 8;
+    uint8_t storage_bit = key_number % 8;
+    uint8_t layer;
     if (pressed) {
-        pressed_actions_cache[key.row][key.col] = layer_switch_get_layer(key);
+        layer = layer_switch_get_layer(key);
+        for (uint8_t bit_number = 0; bit_number <= 4; bit_number++) {
+            source_layers_cache[bit_number][storage_row] ^= (-(bool)((layer & (1U << bit_number)) != 0) ^ source_layers_cache[bit_number][storage_row])) & (1U << storage_bit);
+        }
     }
-    return action_for_key(pressed_actions_cache[key.row][key.col], key);
+    else {
+        layer = 0;
+        for (uint8_t bit_number = 0; bit_number <= 4; bit_number++) {
+            layer |= (uint8_t)((source_layers_cache[bit_number][storage_row] & (1U << storage_bit)) != 0) << bit_number;
+        }
+    }
+    return action_for_key(layer, key);
 #else
     return layer_switch_get_action(key);
 #endif
diff --git a/tmk_core/common/action.h b/tmk_core/common/action.h
index 2b43d001e1..533e5d1a01 100644
--- a/tmk_core/common/action.h
+++ b/tmk_core/common/action.h
@@ -61,7 +61,6 @@ void action_function(keyrecord_t *record, uint8_t id, uint8_t opt);
 /* Utilities for actions.  */
 #if !defined(NO_ACTION_LAYER) && defined(PREVENT_STUCK_MODIFIERS)
 extern bool disable_action_cache;
-extern int8_t pressed_actions_cache[MATRIX_ROWS][MATRIX_COLS];
 #endif
 void process_action_nocache(keyrecord_t *record);
 void process_action(keyrecord_t *record);