Standardize how unicode is processed (fixes #8768) (#8770)

Co-authored-by: Konstantin Đorđević <vomindoraan@gmail.com>

docs/feature_unicode.md

@@ -68,11 +68,14 @@ Then define a table like this in your keymap file:
 const qk_ucis_symbol_t ucis_symbol_table[] = UCIS_TABLE(
     UCIS_SYM("poop", 0x1F4A9),                // 💩
     UCIS_SYM("rofl", 0x1F923),                // 🤣
-    UCIS_SYM("kiss", 0x1F619)  // 😙
+    UCIS_SYM("cuba", 0x1F1E8, 0x1F1FA),       // 🇨🇺
+    UCIS_SYM("look", 0x0CA0, 0x005F, 0x0CA0), // ಠ_ಠ
 );
 ```
 
-To use it, call `qk_ucis_start()`. Then, type the mnemonic for the character (such as "rofl"), and hit Space or Enter. QMK should erase the "rofl" text and insert the laughing emoji.
+By default, each table entry may be up to 3 code points long. This number can be changed by adding `#define UCIS_MAX_CODE_POINTS n` to your `config.h` file.
+
+To use UCIS input, call `qk_ucis_start()`. Then, type the mnemonic for the character (such as "rofl") and hit Space, Enter or Esc. QMK should erase the "rofl" text and insert the laughing emoji.
 
 ### Customization
 
@@ -90,7 +93,7 @@ Unicode input in QMK works by inputting a sequence of characters to the OS, sort
 
 The following input modes are available:
 
-* **`UC_MAC`**: macOS built-in Unicode hex input. Supports code points up to `0xFFFF` (`0x10FFFF` with Unicode Map).
+* **`UC_MAC`**: macOS built-in Unicode hex input. Supports code points up to `0x10FFFF` (all possible code points).
 
   To enable, go to _System Preferences > Keyboard > Input Sources_, add _Unicode Hex Input_ to the list (it's under _Other_), then activate it from the input dropdown in the Menu Bar.
   By default, this mode uses the left Option key (`KC_LALT`) for Unicode input, but this can be changed by defining [`UNICODE_KEY_MAC`](#input-key-configuration) with another keycode.

quantum/process_keycode/process_ucis.c

@@ -27,7 +27,7 @@ void qk_ucis_start(void) {
 
 __attribute__((weak)) void qk_ucis_start_user(void) {
     unicode_input_start();
-    register_hex(0x2328);
+    register_hex(0x2328);  // ⌨
     unicode_input_finish();
 }
 
@@ -35,74 +35,54 @@ __attribute__((weak)) void qk_ucis_success(uint8_t symbol_index) {}
 
 static bool is_uni_seq(char *seq) {
     uint8_t i;
-
     for (i = 0; seq[i]; i++) {
-        uint16_t code;
+        uint16_t keycode;
-        if (('1' <= seq[i]) && (seq[i] <= '0'))
+        if ('1' <= seq[i] && seq[i] <= '0') {
-            code = seq[i] - '1' + KC_1;
+            keycode = seq[i] - '1' + KC_1;
-        else
+        } else {
-            code = seq[i] - 'a' + KC_A;
+            keycode = seq[i] - 'a' + KC_A;
-
-        if (i > qk_ucis_state.count || qk_ucis_state.codes[i] != code) return false;
         }
-
+        if (i > qk_ucis_state.count || qk_ucis_state.codes[i] != keycode) {
-    return (qk_ucis_state.codes[i] == KC_ENT || qk_ucis_state.codes[i] == KC_SPC);
+            return false;
+        }
+    }
+    return qk_ucis_state.codes[i] == KC_ENT || qk_ucis_state.codes[i] == KC_SPC;
 }
 
 __attribute__((weak)) void qk_ucis_symbol_fallback(void) {
     for (uint8_t i = 0; i < qk_ucis_state.count - 1; i++) {
-        uint8_t code = qk_ucis_state.codes[i];
+        uint8_t keycode = qk_ucis_state.codes[i];
-        register_code(code);
+        register_code(keycode);
-        unregister_code(code);
+        unregister_code(keycode);
         wait_ms(UNICODE_TYPE_DELAY);
     }
 }
 
 __attribute__((weak)) void qk_ucis_cancel(void) {}
 
-void register_ucis(const char *hex) {
+void register_ucis(const uint32_t *code_points) {
-    for (int i = 0; hex[i]; i++) {
+    for (int i = 0; i < UCIS_MAX_CODE_POINTS && code_points[i]; i++) {
-        uint8_t kc = 0;
+        register_unicode(code_points[i]);
-        char    c  = hex[i];
-
-        switch (c) {
-            case '0':
-                kc = KC_0;
-                break;
-            case '1' ... '9':
-                kc = c - '1' + KC_1;
-                break;
-            case 'a' ... 'f':
-                kc = c - 'a' + KC_A;
-                break;
-            case 'A' ... 'F':
-                kc = c - 'A' + KC_A;
-                break;
-        }
-
-        if (kc) {
-            register_code(kc);
-            unregister_code(kc);
         wait_ms(UNICODE_TYPE_DELAY);
     }
-    }
 }
 
 bool process_ucis(uint16_t keycode, keyrecord_t *record) {
-    uint8_t i;
+    if (!qk_ucis_state.in_progress || !record->event.pressed) {
-
+        return true;
-    if (!qk_ucis_state.in_progress) return true;
-
-    if (qk_ucis_state.count >= UCIS_MAX_SYMBOL_LENGTH && !(keycode == KC_BSPC || keycode == KC_ESC || keycode == KC_SPC || keycode == KC_ENT)) {
-        return false;
     }
 
-    if (!record->event.pressed) return true;
+    bool special = keycode == KC_SPC || keycode == KC_ENT ||
+                   keycode == KC_ESC || keycode == KC_BSPC;
+    if (qk_ucis_state.count >= UCIS_MAX_SYMBOL_LENGTH && !special) {
+        return false;
+    }
 
     qk_ucis_state.codes[qk_ucis_state.count] = keycode;
     qk_ucis_state.count++;
 
-    if (keycode == KC_BSPC) {
+    switch (keycode) {
+    case KC_BSPC:
         if (qk_ucis_state.count >= 2) {
             qk_ucis_state.count -= 2;
             return true;
@@ -110,12 +90,11 @@ bool process_ucis(uint16_t keycode, keyrecord_t *record) {
             qk_ucis_state.count--;
             return false;
         }
-    }
 
-    if (keycode == KC_ENT || keycode == KC_SPC || keycode == KC_ESC) {
+    case KC_SPC:
-        bool symbol_found = false;
+    case KC_ENT:
-
+    case KC_ESC:
-        for (i = qk_ucis_state.count; i > 0; i--) {
+        for (uint8_t i = 0; i < qk_ucis_state.count; i++) {
             register_code(KC_BSPC);
             unregister_code(KC_BSPC);
             wait_ms(UNICODE_TYPE_DELAY);
@@ -127,25 +106,25 @@ bool process_ucis(uint16_t keycode, keyrecord_t *record) {
             return false;
         }
 
-        unicode_input_start();
+        uint8_t i;
+        bool    symbol_found = false;
         for (i = 0; ucis_symbol_table[i].symbol; i++) {
             if (is_uni_seq(ucis_symbol_table[i].symbol)) {
                 symbol_found = true;
-                register_ucis(ucis_symbol_table[i].code + 2);
+                register_ucis(ucis_symbol_table[i].code_points);
                 break;
             }
         }
-        if (!symbol_found) {
-            qk_ucis_symbol_fallback();
-        }
-        unicode_input_finish();
-
         if (symbol_found) {
             qk_ucis_success(i);
+        } else {
+            qk_ucis_symbol_fallback();
         }
 
         qk_ucis_state.in_progress = false;
         return false;
-    }
+
+    default:
         return true;
+    }
 }

quantum/process_keycode/process_ucis.h

@@ -22,10 +22,13 @@
 #ifndef UCIS_MAX_SYMBOL_LENGTH
 #    define UCIS_MAX_SYMBOL_LENGTH 32
 #endif
+#ifndef UCIS_MAX_CODE_POINTS
+#    define UCIS_MAX_CODE_POINTS 3
+#endif
 
 typedef struct {
-    char *symbol;
+    char *   symbol;
-    char *code;
+    uint32_t code_points[UCIS_MAX_CODE_POINTS];
 } qk_ucis_symbol_t;
 
 typedef struct {
@@ -36,12 +39,17 @@ typedef struct {
 
 extern qk_ucis_state_t qk_ucis_state;
 
+// clang-format off
+
 #define UCIS_TABLE(...) \
     {                   \
-        __VA_ARGS__, { NULL, NULL } \
+        __VA_ARGS__,    \
+        { NULL, {} }    \
     }
-#define UCIS_SYM(name, code) \
+#define UCIS_SYM(name, ...) \
-    { name, #code }
+    { name, {__VA_ARGS__} }
+
+// clang-format on
 
 extern const qk_ucis_symbol_t ucis_symbol_table[];
 
@@ -49,5 +57,7 @@ void qk_ucis_start(void);
 void qk_ucis_start_user(void);
 void qk_ucis_symbol_fallback(void);
 void qk_ucis_success(uint8_t symbol_index);
-void register_ucis(const char *hex);
+
+void register_ucis(const uint32_t *code_points);
+
 bool process_ucis(uint16_t keycode, keyrecord_t *record);

quantum/process_keycode/process_unicode_common.c

@@ -171,6 +171,25 @@ void register_hex32(uint32_t hex) {
     }
 }
 
+void register_unicode(uint32_t code_point) {
+    if (code_point > 0x10FFFF || (code_point > 0xFFFF && unicode_config.input_mode == UC_WIN)) {
+        // Code point out of range, do nothing
+        return;
+    }
+
+    unicode_input_start();
+    if (code_point > 0xFFFF && unicode_config.input_mode == UC_MAC) {
+        // Convert code point to UTF-16 surrogate pair on macOS
+        code_point -= 0x10000;
+        uint32_t lo = code_point & 0x3FF, hi = (code_point & 0xFFC00) >> 10;
+        register_hex32(hi + 0xD800);
+        register_hex32(lo + 0xDC00);
+    } else {
+        register_hex32(code_point);
+    }
+    unicode_input_finish();
+}
+
 // clang-format off
 
 void send_unicode_hex_string(const char *str) {
@@ -236,14 +255,12 @@ void send_unicode_string(const char *str) {
         return;
     }
 
-    int32_t code_point = 0;
     while (*str) {
+        int32_t code_point = 0;
         str = decode_utf8(str, &code_point);
 
         if (code_point >= 0) {
-            unicode_input_start();
+            register_unicode(code_point);
-            register_hex32(code_point);
-            unicode_input_finish();
         }
     }
 }

quantum/process_keycode/process_unicode_common.h

@@ -89,6 +89,8 @@ void unicode_input_cancel(void);
 
 void register_hex(uint16_t hex);
 void register_hex32(uint32_t hex);
+void register_unicode(uint32_t code_point);
+
 void send_unicode_hex_string(const char *str);
 void send_unicode_string(const char *str);
 

quantum/process_keycode/process_unicodemap.c

@@ -36,25 +36,8 @@ __attribute__((weak)) uint16_t unicodemap_index(uint16_t keycode) {
 
 bool process_unicodemap(uint16_t keycode, keyrecord_t *record) {
     if (keycode >= QK_UNICODEMAP && keycode <= QK_UNICODEMAP_PAIR_MAX && record->event.pressed) {
-        unicode_input_start();
+        uint32_t code_point = pgm_read_dword(unicode_map + unicodemap_index(keycode));
-
+        register_unicode(code_point);
-        uint32_t code       = pgm_read_dword(unicode_map + unicodemap_index(keycode));
-        uint8_t  input_mode = get_unicode_input_mode();
-
-        if (code > 0x10FFFF || (code > 0xFFFF && input_mode == UC_WIN)) {
-            // Character is out of range supported by the platform
-            unicode_input_cancel();
-        } else if (code > 0xFFFF && input_mode == UC_MAC) {
-            // Convert to UTF-16 surrogate pair on Mac
-            code -= 0x10000;
-            uint32_t lo = code & 0x3FF, hi = (code & 0xFFC00) >> 10;
-            register_hex32(hi + 0xD800);
-            register_hex32(lo + 0xDC00);
-            unicode_input_finish();
-        } else {
-            register_hex32(code);
-            unicode_input_finish();
-        }
     }
     return true;
 }