From 1426ffc0ee64c1d6de072ffc9d7fd9d1291f4f86 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Mon, 10 May 2021 01:06:44 +1000 Subject: [PATCH] Add script to perform parallel builds. (#12497) Co-authored-by: Erovia --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/info.py | 14 +++++- lib/python/qmk/cli/multibuild.py | 75 ++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 lib/python/qmk/cli/multibuild.py diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index f7df908119..008e57f762 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -24,6 +24,7 @@ from . import json2c from . import lint from . import list from . import kle2json +from . import multibuild from . import new from . import pyformat from . import pytest diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py index 6115e8f87d..0d08d242cd 100755 --- a/lib/python/qmk/cli/info.py +++ b/lib/python/qmk/cli/info.py @@ -10,7 +10,7 @@ from milc import cli from qmk.json_encoders import InfoJSONEncoder from qmk.constants import COL_LETTERS, ROW_LETTERS from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.keyboard import keyboard_completer, keyboard_folder, render_layouts, render_layout +from qmk.keyboard import keyboard_completer, keyboard_folder, render_layouts, render_layout, rules_mk from qmk.keymap import locate_keymap from qmk.info import info_json from qmk.path import is_keyboard @@ -124,12 +124,20 @@ def print_text_output(kb_info_json): show_keymap(kb_info_json, False) +def print_parsed_rules_mk(keyboard_name): + rules = rules_mk(keyboard_name) + for k in sorted(rules.keys()): + print('%s = %s' % (k, rules[k])) + return + + @cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to show info for.') @cli.argument('-km', '--keymap', help='Show the layers for a JSON keymap too.') @cli.argument('-l', '--layouts', action='store_true', help='Render the layouts.') @cli.argument('-m', '--matrix', action='store_true', help='Render the layouts with matrix information.') @cli.argument('-f', '--format', default='friendly', arg_only=True, help='Format to display the data in (friendly, text, json) (Default: friendly).') @cli.argument('--ascii', action='store_true', default=not UNICODE_SUPPORT, help='Render layout box drawings in ASCII only.') +@cli.argument('-r', '--rules-mk', action='store_true', help='Render the parsed values of the keyboard\'s rules.mk file.') @cli.subcommand('Keyboard information.') @automagic_keyboard @automagic_keymap @@ -146,6 +154,10 @@ def info(cli): cli.log.error('Invalid keyboard: "%s"', cli.config.info.keyboard) return False + if bool(cli.args.rules_mk): + print_parsed_rules_mk(cli.config.info.keyboard) + return False + # Build the info.json file kb_info_json = info_json(cli.config.info.keyboard) diff --git a/lib/python/qmk/cli/multibuild.py b/lib/python/qmk/cli/multibuild.py new file mode 100755 index 0000000000..1f3a097704 --- /dev/null +++ b/lib/python/qmk/cli/multibuild.py @@ -0,0 +1,75 @@ +"""Compile all keyboards. + +This will compile everything in parallel, for testing purposes. +""" +import re +from pathlib import Path + +from milc import cli + +from qmk.constants import QMK_FIRMWARE +from qmk.commands import _find_make +import qmk.keyboard + + +def _make_rules_mk_filter(key, value): + def _rules_mk_filter(keyboard_name): + rules_mk = qmk.keyboard.rules_mk(keyboard_name) + return True if key in rules_mk and rules_mk[key].lower() == str(value).lower() else False + return _rules_mk_filter + + +def _is_split(keyboard_name): + rules_mk = qmk.keyboard.rules_mk(keyboard_name) + return True if 'SPLIT_KEYBOARD' in rules_mk and rules_mk['SPLIT_KEYBOARD'].lower() == 'yes' else False + + +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") +@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on the supplied value in rules.mk. Supported format is 'SPLIT_KEYBOARD=yes'. May be passed multiple times.") +@cli.subcommand('Compile QMK Firmware for all keyboards.', hidden=False if cli.config.user.developer else True) +def multibuild(cli): + """Compile QMK Firmware against all keyboards. + """ + + make_cmd = _find_make() + if cli.args.clean: + cli.run([make_cmd, 'clean'], capture_output=False, text=False) + + builddir = Path(QMK_FIRMWARE) / '.build' + makefile = builddir / 'parallel_kb_builds.mk' + + keyboard_list = qmk.keyboard.list_keyboards() + + filter_re = re.compile(r'^(?P[A-Z0-9_]+)\s*=\s*(?P[^#]+)$') + for filter_txt in cli.args.filter: + f = filter_re.match(filter_txt) + if f is not None: + keyboard_list = filter(_make_rules_mk_filter(f.group('key'), f.group('value')), keyboard_list) + + keyboard_list = list(sorted(keyboard_list)) + + if len(keyboard_list) == 0: + return + + builddir.mkdir(parents=True, exist_ok=True) + with open(makefile, "w") as f: + for keyboard_name in keyboard_list: + keyboard_safe = keyboard_name.replace('/', '_') + f.write( + f"""\ +all: {keyboard_safe}_binary +{keyboard_safe}_binary: + @rm -f "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" || true + +@$(MAKE) -C "{QMK_FIRMWARE}" -f "{QMK_FIRMWARE}/build_keyboard.mk" KEYBOARD="{keyboard_name}" KEYMAP="default" REQUIRE_PLATFORM_KEY= COLOR=true SILENT=false \\ + >>"{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" 2>&1 \\ + || cp "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" "{QMK_FIRMWARE}/.build/failed.log.{keyboard_safe}" + @{{ grep '\[ERRORS\]' "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;31m[ERRORS]\e[0m\\n" "{keyboard_name}:default" ; }} \\ + || {{ grep '\[WARNINGS\]' "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" >/dev/null 2>&1 && printf "Build %-64s \e[1;33m[WARNINGS]\e[0m\\n" "{keyboard_name}:default" ; }} \\ + || printf "Build %-64s \e[1;32m[OK]\e[0m\\n" "{keyboard_name}:default" + @rm -f "{QMK_FIRMWARE}/.build/build.log.{keyboard_safe}" || true + +""" # noqa: yapf should not care about the formatting of the Makefile + ) + + cli.run([make_cmd, '-j', str(cli.args.parallel), '-f', makefile, 'all'], capture_output=False, text=False)