1
0
Fork 0

Improve the compile and flash subcommands (#11334)

* add support for --clean to compile and flash

* compile standalone JSON keymaps without polluting the tree

* Add support for passing environment vars to make

* make flake8 happy

* document changes to qmk compile and flash

* add -e support to json export compiling

* Fix python 3.6

* honor $MAKE

* add support for parallel builds
This commit is contained in:
Zach White 2021-01-16 15:13:04 -08:00 committed by GitHub
parent c628408688
commit d9785ec313
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 137 additions and 30 deletions

View file

@ -11,13 +11,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K
**Usage for Configurator Exports**: **Usage for Configurator Exports**:
``` ```
qmk compile <configuratorExport.json> qmk compile [-c] <configuratorExport.json>
``` ```
**Usage for Keymaps**: **Usage for Keymaps**:
``` ```
qmk compile -kb <keyboard_name> -km <keymap_name> qmk compile [-c] [-e <var>=<value>] -kb <keyboard_name> -km <keymap_name>
``` ```
**Usage in Keyboard Directory**: **Usage in Keyboard Directory**:
@ -82,13 +82,13 @@ This command is directory aware. It will automatically fill in KEYBOARD and/or K
**Usage for Configurator Exports**: **Usage for Configurator Exports**:
``` ```
qmk flash <configuratorExport.json> -bl <bootloader> qmk flash [-bl <bootloader>] [-c] [-e <var>=<value>] <configuratorExport.json>
``` ```
**Usage for Keymaps**: **Usage for Keymaps**:
``` ```
qmk flash -kb <keyboard_name> -km <keymap_name> -bl <bootloader> qmk flash -kb <keyboard_name> -km <keymap_name> [-bl <bootloader>] [-c] [-e <var>=<value>]
``` ```
**Listing the Bootloaders** **Listing the Bootloaders**

View file

@ -13,7 +13,7 @@ def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs) print(*args, file=sys.stderr, **kwargs)
fileHeader = """\ file_header = """\
/* Copyright 2020 QMK /* Copyright 2020 QMK
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
@ -77,7 +77,7 @@ def check_diffs(input_defs, reference_defs):
def migrate_chconf_h(to_override, outfile): def migrate_chconf_h(to_override, outfile):
print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile) print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
for override in to_override: for override in to_override:
print("#define %s %s" % (override[0], override[1]), file=outfile) print("#define %s %s" % (override[0], override[1]), file=outfile)
@ -87,7 +87,7 @@ def migrate_chconf_h(to_override, outfile):
def migrate_halconf_h(to_override, outfile): def migrate_halconf_h(to_override, outfile):
print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile) print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
for override in to_override: for override in to_override:
print("#define %s %s" % (override[0], override[1]), file=outfile) print("#define %s %s" % (override[0], override[1]), file=outfile)
@ -97,7 +97,7 @@ def migrate_halconf_h(to_override, outfile):
def migrate_mcuconf_h(to_override, outfile): def migrate_mcuconf_h(to_override, outfile):
print(fileHeader.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile) print(file_header.format(cli.args.input.relative_to(QMK_FIRMWARE), cli.args.reference.relative_to(QMK_FIRMWARE)), file=outfile)
print("#include_next <mcuconf.h>\n", file=outfile) print("#include_next <mcuconf.h>\n", file=outfile)

View file

@ -2,7 +2,6 @@
You can compile a keymap already in the repo or using a QMK Configurator export. You can compile a keymap already in the repo or using a QMK Configurator export.
""" """
import subprocess
from argparse import FileType from argparse import FileType
from milc import cli from milc import cli
@ -15,6 +14,9 @@ from qmk.commands import compile_configurator_json, create_make_command, parse_c
@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.subcommand('Compile a QMK Firmware.') @cli.subcommand('Compile a QMK Firmware.')
@automagic_keyboard @automagic_keyboard
@automagic_keymap @automagic_keymap
@ -25,18 +27,32 @@ def compile(cli):
If a keyboard and keymap are provided this command will build a firmware based on that. If a keyboard and keymap are provided this command will build a firmware based on that.
""" """
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean')
# FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
cli.run(command, capture_output=False, text=False)
# Build the environment vars
envs = {}
for env in cli.args.env:
if '=' in env:
key, value = env.split('=', 1)
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
# Determine the compile command
command = None command = None
if cli.args.filename: if cli.args.filename:
# If a configurator JSON was provided generate a keymap and compile it # If a configurator JSON was provided generate a keymap and compile it
# FIXME(skullydazed): add code to check and warn if the keymap already exists when compiling a json keymap.
user_keymap = parse_configurator_json(cli.args.filename) user_keymap = parse_configurator_json(cli.args.filename)
command = compile_configurator_json(user_keymap) command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)
else: else:
if cli.config.compile.keyboard and cli.config.compile.keymap: if cli.config.compile.keyboard and cli.config.compile.keymap:
# Generate the make command for a specific keyboard/keymap. # Generate the make command for a specific keyboard/keymap.
command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap) command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs)
elif not cli.config.compile.keyboard: elif not cli.config.compile.keyboard:
cli.log.error('Could not determine keyboard!') cli.log.error('Could not determine keyboard!')
@ -48,7 +64,8 @@ def compile(cli):
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run: if not cli.args.dry_run:
cli.echo('\n') cli.echo('\n')
compile = subprocess.run(command) # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere.
compile = cli.run(command, capture_output=False, text=False)
return compile.returncode return compile.returncode
else: else:

View file

@ -3,7 +3,6 @@
You can compile a keymap already in the repo or using a QMK Configurator export. You can compile a keymap already in the repo or using a QMK Configurator export.
A bootloader must be specified. A bootloader must be specified.
""" """
import subprocess
from argparse import FileType from argparse import FileType
from milc import cli from milc import cli
@ -37,6 +36,9 @@ def print_bootloader_help():
@cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-kb', '--keyboard', help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.')
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.")
@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.")
@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
@cli.subcommand('QMK Flash.') @cli.subcommand('QMK Flash.')
@automagic_keyboard @automagic_keyboard
@automagic_keymap @automagic_keymap
@ -50,6 +52,20 @@ def flash(cli):
If bootloader is omitted the make system will use the configured bootloader for that keyboard. If bootloader is omitted the make system will use the configured bootloader for that keyboard.
""" """
if cli.args.clean and not cli.args.filename and not cli.args.dry_run:
command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean')
cli.run(command, capture_output=False)
# Build the environment vars
envs = {}
for env in cli.args.env:
if '=' in env:
key, value = env.split('=', 1)
envs[key] = value
else:
cli.log.warning('Invalid environment variable: %s', env)
# Determine the compile command
command = '' command = ''
if cli.args.bootloaders: if cli.args.bootloaders:
@ -60,16 +76,16 @@ def flash(cli):
if cli.args.filename: if cli.args.filename:
# Handle compiling a configurator JSON # Handle compiling a configurator JSON
user_keymap = parse_configurator_json(cli.args.filename) user_keymap = parse_configurator_json(cli.args.filename, parallel=cli.config.flash.parallel)
keymap_path = qmk.path.keymap(user_keymap['keyboard']) keymap_path = qmk.path.keymap(user_keymap['keyboard'])
command = compile_configurator_json(user_keymap, cli.args.bootloader) command = compile_configurator_json(user_keymap, cli.args.bootloader, **envs)
cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap']) cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
else: else:
if cli.config.flash.keyboard and cli.config.flash.keymap: if cli.config.flash.keyboard and cli.config.flash.keymap:
# Generate the make command for a specific keyboard/keymap. # Generate the make command for a specific keyboard/keymap.
command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader) command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs)
elif not cli.config.flash.keyboard: elif not cli.config.flash.keyboard:
cli.log.error('Could not determine keyboard!') cli.log.error('Could not determine keyboard!')
@ -81,7 +97,7 @@ def flash(cli):
cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command))
if not cli.args.dry_run: if not cli.args.dry_run:
cli.echo('\n') cli.echo('\n')
compile = subprocess.run(command) compile = cli.run(command, capture_output=False, text=True)
return compile.returncode return compile.returncode
else: else:

View file

@ -6,11 +6,26 @@ import platform
import subprocess import subprocess
import shlex import shlex
import shutil import shutil
from pathlib import Path
from milc import cli
import qmk.keymap import qmk.keymap
from qmk.constants import KEYBOARD_OUTPUT_PREFIX
def create_make_command(keyboard, keymap, target=None): def _find_make():
"""Returns the correct make command for this environment.
"""
make_cmd = os.environ.get('MAKE')
if not make_cmd:
make_cmd = 'gmake' if shutil.which('gmake') else 'make'
return make_cmd
def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars):
"""Create a make compile command """Create a make compile command
Args: Args:
@ -24,41 +39,95 @@ def create_make_command(keyboard, keymap, target=None):
target target
Usually a bootloader. Usually a bootloader.
parallel
The number of make jobs to run in parallel
**env_vars
Environment variables to be passed to make.
Returns: Returns:
A command that can be run to make the specified keyboard and keymap A command that can be run to make the specified keyboard and keymap
""" """
env = []
make_args = [keyboard, keymap] make_args = [keyboard, keymap]
make_cmd = 'gmake' if shutil.which('gmake') else 'make' make_cmd = _find_make()
if target: if target:
make_args.append(target) make_args.append(target)
return [make_cmd, ':'.join(make_args)] for key, value in env_vars.items():
env.append(f'{key}={value}')
return [make_cmd, '-j', str(parallel), *env, ':'.join(make_args)]
def compile_configurator_json(user_keymap, bootloader=None): def compile_configurator_json(user_keymap, parallel=1, **env_vars):
"""Convert a configurator export JSON file into a C file """Convert a configurator export JSON file into a C file and then compile it.
Args: Args:
configurator_filename user_keymap
The configurator JSON export file A deserialized keymap export
bootloader bootloader
A bootloader to flash A bootloader to flash
parallel
The number of make jobs to run in parallel
Returns: Returns:
A command to run to compile and flash the C file. A command to run to compile and flash the C file.
""" """
# Write the keymap C file # Write the keymap.c file
qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers']) keyboard_filesafe = user_keymap['keyboard'].replace('/', '_')
target = f'{keyboard_filesafe}_{user_keymap["keymap"]}'
keyboard_output = Path(f'{KEYBOARD_OUTPUT_PREFIX}{keyboard_filesafe}')
keymap_output = Path(f'{keyboard_output}_{user_keymap["keymap"]}')
c_text = qmk.keymap.generate_c(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
keymap_dir = keymap_output / 'src'
keymap_c = keymap_dir / 'keymap.c'
keymap_dir.mkdir(exist_ok=True, parents=True)
keymap_c.write_text(c_text)
# Return a command that can be run to make the keymap and flash if given # Return a command that can be run to make the keymap and flash if given
if bootloader is None: verbose = 'true' if cli.config.general.verbose else 'false'
return create_make_command(user_keymap['keyboard'], user_keymap['keymap']) color = 'true' if cli.config.general.color else 'false'
return create_make_command(user_keymap['keyboard'], user_keymap['keymap'], bootloader) make_command = [
_find_make(),
'-j',
str(parallel),
'-r',
'-R',
'-f',
'build_keyboard.mk',
]
for key, value in env_vars.items():
make_command.append(f'{key}={value}')
make_command.extend([
f'KEYBOARD={user_keymap["keyboard"]}',
f'KEYMAP={user_keymap["keymap"]}',
f'KEYBOARD_FILESAFE={keyboard_filesafe}',
f'TARGET={target}',
f'KEYBOARD_OUTPUT={keyboard_output}',
f'KEYMAP_OUTPUT={keymap_output}',
f'MAIN_KEYMAP_PATH_1={keymap_output}',
f'MAIN_KEYMAP_PATH_2={keymap_output}',
f'MAIN_KEYMAP_PATH_3={keymap_output}',
f'MAIN_KEYMAP_PATH_4={keymap_output}',
f'MAIN_KEYMAP_PATH_5={keymap_output}',
f'KEYMAP_C={keymap_c}',
f'KEYMAP_PATH={keymap_dir}',
f'VERBOSE={verbose}',
f'COLOR={color}',
'SILENT=false',
])
return make_command
def parse_configurator_json(configurator_file): def parse_configurator_json(configurator_file):

View file

@ -1,5 +1,6 @@
"""Information that should be available to the python library. """Information that should be available to the python library.
""" """
from os import environ
from pathlib import Path from pathlib import Path
# The root of the qmk_firmware tree. # The root of the qmk_firmware tree.
@ -17,3 +18,7 @@ VUSB_PROCESSORS = 'atmega32a', 'atmega328p', 'atmega328', 'attiny85'
DATE_FORMAT = '%Y-%m-%d' DATE_FORMAT = '%Y-%m-%d'
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z' DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
TIME_FORMAT = '%H:%M:%S' TIME_FORMAT = '%H:%M:%S'
# Constants that should match their counterparts in make
BUILD_DIR = environ.get('BUILD_DIR', '.build')
KEYBOARD_OUTPUT_PREFIX = f'{BUILD_DIR}/obj_'