1
0
Fork 0

[CLI] Add c2json (#8817)

* Basic keymap parsing finally works

* Add 'keymap.json' creation to the qmk.keymap module

* Add tests and fix formatting

* Fix/exclude flake8 errors

* Convert keymap.c to valid keymap.json

* Fix some errors

* Add tests

* Finalize keymap.json creation, add json template

* Add docs

* Move pygments to the standard requirements

* Add support for nameless layers, fix tests

* Fix things after rebase

* Add missing 'keymap' value.

* Fix missing layer numbers from advanced keycodes

Buckwich noticed that if the advanced keycode / layer toggling key
contains a number, it goes missing.
Now we properly handle them.
Thx for noticing!

* Apply suggestions from code review

* fixup tests

Co-authored-by: Zach White <skullydazed@drpepper.org>
Co-authored-by: skullY <skullydazed@gmail.com>
This commit is contained in:
Erovia 2020-10-07 01:10:19 +01:00 committed by GitHub
parent c9a06965c9
commit 058737f116
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 389 additions and 31 deletions

View file

@ -167,6 +167,17 @@ Creates a keymap.c from a QMK Configurator export.
qmk json2c [-o OUTPUT] filename qmk json2c [-o OUTPUT] filename
``` ```
## `qmk c2json`
Creates a keymap.json from a keymap.c.
**Note:** Parsing C source files is not easy, therefore this subcommand may not work your keymap. In some cases not using the C pre-processor helps.
**Usage**:
```
qmk c2json [--no-cpp] [-o OUTPUT] filename
```
## `qmk list-keyboards` ## `qmk list-keyboards`
This command lists all the keyboards currently defined in `qmk_firmware` This command lists all the keyboards currently defined in `qmk_firmware`

View file

@ -0,0 +1,23 @@
#include QMK_KEYBOARD_H
#include "audio.h"
/* THIS FILE WAS GENERATED AND IS EXPERIMENTAL!
*
* This file was generated by qmk-compile-json. You may or may not want to
* edit it directly.
*/
const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
[0] = LAYOUT(KC_ENTER)
};
void encoder_update_user(uint8_t index, bool clockwise) {
if (index == 0) {
if (clockwise) {
tap_code(KC_UP);
} else {
tap_code(KC_DOWN);
}
}
};

View file

@ -0,0 +1,3 @@
{
"documentation": "This file is a keymap.json file for handwired/onekey/pytest"
}

View file

@ -6,6 +6,7 @@ import sys
from milc import cli from milc import cli
from . import c2json
from . import cformat from . import cformat
from . import compile from . import compile
from . import config from . import config

View file

@ -0,0 +1,62 @@
"""Generate a keymap.json from a keymap.c file.
"""
import json
import sys
from milc import cli
import qmk.keymap
import qmk.path
@cli.argument('--no-cpp', arg_only=True, action='store_false', help='Do not use \'cpp\' on keymap.c')
@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
@cli.argument('-kb', '--keyboard', arg_only=True, required=True, help='The keyboard\'s name')
@cli.argument('-km', '--keymap', arg_only=True, required=True, help='The keymap\'s name')
@cli.argument('filename', arg_only=True, help='keymap.c file')
@cli.subcommand('Creates a keymap.json from a keymap.c file.')
def c2json(cli):
"""Generate a keymap.json from a keymap.c file.
This command uses the `qmk.keymap` module to generate a keymap.json from a keymap.c file. The generated keymap is written to stdout, or to a file if -o is provided.
"""
cli.args.filename = qmk.path.normpath(cli.args.filename)
# Error checking
if not cli.args.filename.exists():
cli.log.error('C file does not exist!')
cli.print_usage()
exit(1)
if str(cli.args.filename) == '-':
# TODO(skullydazed/anyone): Read file contents from STDIN
cli.log.error('Reading from STDIN is not (yet) supported.')
cli.print_usage()
exit(1)
# Environment processing
if cli.args.output == ('-'):
cli.args.output = None
# Parse the keymap.c
keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp)
# Generate the keymap.json
try:
keymap_json = qmk.keymap.generate(keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'], type='json', keymap=keymap_json['keymap'])
except KeyError:
cli.log.error('Something went wrong. Try to use --no-cpp.')
sys.exit(1)
if cli.args.output:
cli.args.output.parent.mkdir(parents=True, exist_ok=True)
if cli.args.output.exists():
cli.args.output.replace(cli.args.output.name + '.bak')
cli.args.output.write_text(json.dumps(keymap_json))
if not cli.args.quiet:
cli.log.info('Wrote keymap to %s.', cli.args.output)
else:
print(json.dumps(keymap_json))

View file

@ -58,7 +58,7 @@ def parse_gcc_version(version):
return { return {
'major': int(m.group(1)), 'major': int(m.group(1)),
'minor': int(m.group(2)) if m.group(2) else 0, 'minor': int(m.group(2)) if m.group(2) else 0,
'patch': int(m.group(3)) if m.group(3) else 0 'patch': int(m.group(3)) if m.group(3) else 0,
} }

View file

@ -7,7 +7,6 @@ import subprocess
import shlex import shlex
import shutil import shutil
from milc import cli
import qmk.keymap import qmk.keymap
@ -84,6 +83,4 @@ def run(command, *args, **kwargs):
safecmd = ' '.join(safecmd) safecmd = ' '.join(safecmd)
command = [os.environ['SHELL'], '-c', safecmd] command = [os.environ['SHELL'], '-c', safecmd]
cli.log.debug('Running command: %s', command)
return subprocess.run(command, *args, **kwargs) return subprocess.run(command, *args, **kwargs)

View file

@ -1,11 +1,18 @@
"""Functions that help you work with QMK keymaps. """Functions that help you work with QMK keymaps.
""" """
from pathlib import Path from pathlib import Path
import json
import subprocess
from pygments.lexers.c_cpp import CLexer
from pygments.token import Token
from pygments import lex
from milc import cli from milc import cli
from qmk.keyboard import rules_mk from qmk.keyboard import rules_mk
import qmk.path import qmk.path
import qmk.commands
# The `keymap.c` template to use when a keyboard doesn't have its own # The `keymap.c` template to use when a keyboard doesn't have its own
DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@ -22,22 +29,35 @@ __KEYMAP_GOES_HERE__
""" """
def template(keyboard): def template(keyboard, type='c'):
"""Returns the `keymap.c` template for a keyboard. """Returns the `keymap.c` or `keymap.json` template for a keyboard.
If a template exists in `keyboards/<keyboard>/templates/keymap.c` that If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
text will be used instead of `DEFAULT_KEYMAP_C`. text will be used instead of `DEFAULT_KEYMAP_C`.
If a template exists in `keyboards/<keyboard>/templates/keymap.json` that
text will be used instead of an empty dictionary.
Args: Args:
keyboard keyboard
The keyboard to return a template for. The keyboard to return a template for.
type
'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
""" """
template_file = Path('keyboards/%s/templates/keymap.c' % keyboard) if type == 'json':
template_file = Path('keyboards/%s/templates/keymap.json' % keyboard)
template = {'keyboard': keyboard}
if template_file.exists(): if template_file.exists():
return template_file.read_text() template.update(json.loads(template_file.read_text()))
else:
template_file = Path('keyboards/%s/templates/keymap.c' % keyboard)
if template_file.exists():
template = template_file.read_text()
else:
template = DEFAULT_KEYMAP_C
return DEFAULT_KEYMAP_C return template
def _strip_any(keycode): def _strip_any(keycode):
@ -57,8 +77,8 @@ def is_keymap_dir(keymap):
return True return True
def generate(keyboard, layout, layers): def generate(keyboard, layout, layers, type='c', keymap=None):
"""Returns a keymap.c for the specified keyboard, layout, and layers. """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
Args: Args:
keyboard keyboard
@ -69,24 +89,30 @@ def generate(keyboard, layout, layers):
layers layers
An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode. An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
"""
layer_txt = []
type
'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
"""
new_keymap = template(keyboard, type)
if type == 'json':
new_keymap['keymap'] = keymap
new_keymap['layout'] = layout
new_keymap['layers'] = layers
else:
layer_txt = []
for layer_num, layer in enumerate(layers): for layer_num, layer in enumerate(layers):
if layer_num != 0: if layer_num != 0:
layer_txt[-1] = layer_txt[-1] + ',' layer_txt[-1] = layer_txt[-1] + ','
layer = map(_strip_any, layer)
layer_keys = ', '.join(layer) layer_keys = ', '.join(layer)
layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys)) layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
keymap = '\n'.join(layer_txt) keymap = '\n'.join(layer_txt)
keymap_c = template(keyboard) new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)
return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap) return new_keymap
def write(keyboard, keymap, layout, layers): def write(keyboard, keymap, layout, layers, type='c'):
"""Generate the `keymap.c` and write it to disk. """Generate the `keymap.c` and write it to disk.
Returns the filename written to. Returns the filename written to.
@ -103,12 +129,19 @@ def write(keyboard, keymap, layout, layers):
layers layers
An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode. An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
type
'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
""" """
keymap_c = generate(keyboard, layout, layers) keymap_content = generate(keyboard, layout, layers, type)
if type == 'json':
keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'
keymap_content = json.dumps(keymap_content)
else:
keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c' keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
keymap_file.parent.mkdir(parents=True, exist_ok=True) keymap_file.parent.mkdir(parents=True, exist_ok=True)
keymap_file.write_text(keymap_c) keymap_file.write_text(keymap_content)
cli.log.info('Wrote keymap to {fg_cyan}%s', keymap_file) cli.log.info('Wrote keymap to {fg_cyan}%s', keymap_file)
@ -188,3 +221,192 @@ def list_keymaps(keyboard):
names = names.union([keymap.name for keymap in cl_path.iterdir() if is_keymap_dir(keymap)]) names = names.union([keymap.name for keymap in cl_path.iterdir() if is_keymap_dir(keymap)])
return sorted(names) return sorted(names)
def _c_preprocess(path):
""" Run a file through the C pre-processor
Args:
path: path of the keymap.c file
Returns:
the stdout of the pre-processor
"""
pre_processed_keymap = qmk.commands.run(['cpp', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
return pre_processed_keymap.stdout
def _get_layers(keymap): # noqa C901 : until someone has a good idea how to simplify/split up this code
""" Find the layers in a keymap.c file.
Args:
keymap: the content of the keymap.c file
Returns:
a dictionary containing the parsed keymap
"""
layers = list()
opening_braces = '({['
closing_braces = ')}]'
keymap_certainty = brace_depth = 0
is_keymap = is_layer = is_adv_kc = False
layer = dict(name=False, layout=False, keycodes=list())
for line in lex(keymap, CLexer()):
if line[0] is Token.Name:
if is_keymap:
# If we are inside the keymap array
# we know the keymap's name and the layout macro will come,
# followed by the keycodes
if not layer['name']:
if line[1].startswith('LAYOUT') or line[1].startswith('KEYMAP'):
# This can happen if the keymap array only has one layer,
# for macropads and such
layer['name'] = '0'
layer['layout'] = line[1]
else:
layer['name'] = line[1]
elif not layer['layout']:
layer['layout'] = line[1]
elif is_layer:
# If we are inside a layout macro,
# collect all keycodes
if line[1] == '_______':
kc = 'KC_TRNS'
elif line[1] == 'XXXXXXX':
kc = 'KC_NO'
else:
kc = line[1]
if is_adv_kc:
# If we are inside an advanced keycode
# collect everything and hope the user
# knew what he/she was doing
layer['keycodes'][-1] += kc
else:
layer['keycodes'].append(kc)
# The keymaps array's signature:
# const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS]
#
# Only if we've found all 6 keywords in this specific order
# can we know for sure that we are inside the keymaps array
elif line[1] == 'PROGMEM' and keymap_certainty == 2:
keymap_certainty = 3
elif line[1] == 'keymaps' and keymap_certainty == 3:
keymap_certainty = 4
elif line[1] == 'MATRIX_ROWS' and keymap_certainty == 4:
keymap_certainty = 5
elif line[1] == 'MATRIX_COLS' and keymap_certainty == 5:
keymap_certainty = 6
elif line[0] is Token.Keyword:
if line[1] == 'const' and keymap_certainty == 0:
keymap_certainty = 1
elif line[0] is Token.Keyword.Type:
if line[1] == 'uint16_t' and keymap_certainty == 1:
keymap_certainty = 2
elif line[0] is Token.Punctuation:
if line[1] in opening_braces:
brace_depth += 1
if is_keymap:
if is_layer:
# We found the beginning of a non-basic keycode
is_adv_kc = True
layer['keycodes'][-1] += line[1]
elif line[1] == '(' and brace_depth == 2:
# We found the beginning of a layer
is_layer = True
elif line[1] == '{' and keymap_certainty == 6:
# We found the beginning of the keymaps array
is_keymap = True
elif line[1] in closing_braces:
brace_depth -= 1
if is_keymap:
if is_adv_kc:
layer['keycodes'][-1] += line[1]
if brace_depth == 2:
# We found the end of a non-basic keycode
is_adv_kc = False
elif line[1] == ')' and brace_depth == 1:
# We found the end of a layer
is_layer = False
layers.append(layer)
layer = dict(name=False, layout=False, keycodes=list())
elif line[1] == '}' and brace_depth == 0:
# We found the end of the keymaps array
is_keymap = False
keymap_certainty = 0
elif is_adv_kc:
# Advanced keycodes can contain other punctuation
# e.g.: MT(MOD_LCTL | MOD_LSFT, KC_ESC)
layer['keycodes'][-1] += line[1]
elif line[0] is Token.Literal.Number.Integer and is_keymap and not is_adv_kc:
# If the pre-processor finds the 'meaning' of the layer names,
# they will be numbers
if not layer['name']:
layer['name'] = line[1]
else:
# We only care about
# operators and such if we
# are inside an advanced keycode
# e.g.: MT(MOD_LCTL | MOD_LSFT, KC_ESC)
if is_adv_kc:
layer['keycodes'][-1] += line[1]
return layers
def parse_keymap_c(keymap_file, use_cpp=True):
""" Parse a keymap.c file.
Currently only cares about the keymaps array.
Args:
keymap_file: path of the keymap.c file
use_cpp: if True, pre-process the file with the C pre-processor
Returns:
a dictionary containing the parsed keymap
"""
if use_cpp:
keymap_file = _c_preprocess(keymap_file)
else:
keymap_file = keymap_file.read_text()
keymap = dict()
keymap['layers'] = _get_layers(keymap_file)
return keymap
def c2json(keyboard, keymap, keymap_file, use_cpp=True):
""" Convert keymap.c to keymap.json
Args:
keyboard: The name of the keyboard
keymap: The name of the keymap
layout: The LAYOUT macro this keymap uses.
keymap_file: path of the keymap.c file
use_cpp: if True, pre-process the file with the C pre-processor
Returns:
a dictionary in keymap.json format
"""
keymap_json = parse_keymap_c(keymap_file, use_cpp)
dirty_layers = keymap_json.pop('layers', None)
keymap_json['layers'] = list()
for layer in dirty_layers:
layer.pop('name')
layout = layer.pop('layout')
if not keymap_json.get('layout', False):
keymap_json['layout'] = layout
keymap_json['layers'].append(layer.pop('keycodes'))
keymap_json['keyboard'] = keyboard
keymap_json['keymap'] = keymap
return keymap_json

View file

@ -1,10 +1,11 @@
import subprocess from subprocess import STDOUT, PIPE
from qmk.commands import run from qmk.commands import run
def check_subcommand(command, *args): def check_subcommand(command, *args):
cmd = ['bin/qmk', command] + list(args) cmd = ['bin/qmk', command] + list(args)
result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) result = run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
return result return result
@ -28,6 +29,11 @@ def test_compile():
check_returncode(result) check_returncode(result)
def test_compile_json():
result = check_subcommand('compile', '-kb', 'handwired/onekey/pytest', '-km', 'default_json')
check_returncode(result)
def test_flash(): def test_flash():
result = check_subcommand('flash', '-kb', 'handwired/onekey/pytest', '-km', 'default', '-n') result = check_subcommand('flash', '-kb', 'handwired/onekey/pytest', '-km', 'default', '-n')
check_returncode(result) check_returncode(result)
@ -153,3 +159,15 @@ def test_info_matrix_render():
assert 'LAYOUT_ortho_1x1' in result.stdout assert 'LAYOUT_ortho_1x1' in result.stdout
assert '│0A│' in result.stdout assert '│0A│' in result.stdout
assert 'Matrix for "LAYOUT_ortho_1x1"' in result.stdout assert 'Matrix for "LAYOUT_ortho_1x1"' in result.stdout
def test_c2json():
result = check_subcommand("c2json", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/default/keymap.c")
check_returncode(result)
assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}'
def test_c2json_nocpp():
result = check_subcommand("c2json", "--no-cpp", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/pytest_nocpp/keymap.c")
check_returncode(result)
assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_ENTER"]]}'

View file

@ -6,14 +6,34 @@ def test_template_onekey_proton_c():
assert templ == qmk.keymap.DEFAULT_KEYMAP_C assert templ == qmk.keymap.DEFAULT_KEYMAP_C
def test_template_onekey_proton_c_json():
templ = qmk.keymap.template('handwired/onekey/proton_c', type='json')
assert templ == {'keyboard': 'handwired/onekey/proton_c'}
def test_template_onekey_pytest(): def test_template_onekey_pytest():
templ = qmk.keymap.template('handwired/onekey/pytest') templ = qmk.keymap.template('handwired/onekey/pytest')
assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n' assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n'
def test_template_onekey_pytest_json():
templ = qmk.keymap.template('handwired/onekey/pytest', type='json')
assert templ == {'keyboard': 'handwired/onekey/pytest', "documentation": "This file is a keymap.json file for handwired/onekey/pytest"}
def test_generate_onekey_pytest(): def test_generate_onekey_pytest():
templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']]) templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n' assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n'
def test_generate_onekey_pytest_json():
templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']], type='json', keymap='default')
assert templ == {"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]}
def test_parse_keymap_c():
parsed_keymap_c = qmk.keymap.parse_keymap_c('keyboards/handwired/onekey/keymaps/default/keymap.c')
assert parsed_keymap_c == {'layers': [{'name': '0', 'layout': 'LAYOUT_ortho_1x1', 'keycodes': ['KC_A']}]}
# FIXME(skullydazed): Add a test for qmk.keymap.write that mocks up an FD. # FIXME(skullydazed): Add a test for qmk.keymap.write that mocks up an FD.

View file

@ -4,3 +4,4 @@ appdirs
argcomplete argcomplete
colorama colorama
hjson hjson
pygments