QMK CLI and JSON keymap support (#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

.editorconfig

@@ -16,6 +16,10 @@ insert_final_newline = true
 trim_trailing_whitespace = false
 indent_size = 4
 
+[{qmk,*.py}]
+charset = utf-8
+max_line_length = 200
+
 # Make these match what we have in .gitattributes
 [*.mk]
 end_of_line = lf

.gitignore

@@ -70,3 +70,6 @@ util/Win_Check_Output.txt
 secrets.tar
 id_rsa_*
 /.vs
+
+# python things
+__pycache__

bin/qmk

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""CLI wrapper for running QMK commands.
+"""
+import os
+import subprocess
+import sys
+from glob import glob
+from time import strftime
+from importlib import import_module
+from importlib.util import find_spec
+
+# Add the QMK python libs to our path
+script_dir = os.path.dirname(os.path.realpath(__file__))
+qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
+python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
+sys.path.append(python_lib_dir)
+
+# Change to the root of our checkout
+os.environ['ORIG_CWD'] = os.getcwd()
+os.chdir(qmk_dir)
+
+# Make sure our modules have been setup
+with open('requirements.txt', 'r') as fd:
+    for line in fd.readlines():
+        line = line.strip().replace('<', '=').replace('>', '=')
+
+        if line[0] == '#':
+            continue
+
+        if '#' in line:
+            line = line.split('#')[0]
+
+        module = line.split('=')[0] if '=' in line else line
+        if not find_spec(module):
+            print('Your QMK build environment is not fully setup!\n')
+            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            exit(255)
+
+# Figure out our version
+command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
+result = subprocess.run(command, text=True, capture_output=True)
+
+if result.returncode == 0:
+    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+else:
+    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+
+# Setup the CLI
+import milc
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}ψ{style_reset_all}'
+
+# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
+# This means we can't accept arguments to the qmk script itself.
+script_name = os.path.basename(sys.argv[0])
+if script_name == 'qmk':
+    if len(sys.argv) == 1:
+        milc.cli.log.error('No subcommand specified!\n')
+
+    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
+        milc.cli.echo('usage: qmk <subcommand> [...]')
+        milc.cli.echo('\nsubcommands:')
+        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
+        for subcommand in sorted(subcommands):
+            subcommand = os.path.basename(subcommand).split('-', 1)[1]
+            milc.cli.echo('\t%s', subcommand)
+        milc.cli.echo('\nqmk <subcommand> --help for more information')
+        exit(1)
+
+    if sys.argv[1] in ['-V', '--version']:
+        milc.cli.echo(os.environ['QMK_VERSION'])
+        exit(0)
+
+    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
+    del sys.argv[1]
+
+# Look for which module to import
+if script_name == 'qmk':
+    milc.cli.print_help()
+    exit(0)
+elif not script_name.startswith('qmk-'):
+    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
+else:
+    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
+    subcommand.insert(1, 'cli')
+    subcommand = '.'.join(subcommand)
+
+    try:
+        import_module(subcommand)
+    except ModuleNotFoundError as e:
+        if e.__class__.__name__ != subcommand:
+            raise
+
+        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
+        exit(1)
+
+if __name__ == '__main__':
+    milc.cli()

bin/qmk-compile-json

@@ -0,0 +1 @@
+qmk

bin/qmk-doctor

@@ -0,0 +1 @@
+qmk

bin/qmk-hello

@@ -0,0 +1 @@
+qmk

bin/qmk-json-keymap

@@ -0,0 +1 @@
+qmk

build_json.mk

@@ -0,0 +1,27 @@
+# Look for a json keymap file
+ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+    KEYMAP_C := $(KEYBOARD_OUTPUT)/src/keymap.c
+    KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
+    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+endif
+
+# Generate the keymap.c
+ifneq ("$(KEYMAP_JSON)","")
+    _ = $(shell bin/qmk-json-keymap -f $(KEYMAP_JSON) -o $(KEYMAP_C))
+endif

build_keyboard.mk

@@ -98,6 +98,11 @@ MAIN_KEYMAP_PATH_3 := $(KEYBOARD_PATH_3)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
+# Check for keymap.json first, so we can regenerate keymap.c
+include build_json.mk
+
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
     ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
         -include $(MAIN_KEYMAP_PATH_5)/rules.mk
         KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
@@ -119,11 +124,13 @@ else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
         KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
         KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
     else ifneq ($(LAYOUTS),)
+        # If we haven't found a keymap yet fall back to community layouts
         include build_layout.mk
     else
         $(error Could not find keymap)
         # this state should never be reached
     endif
+endif
 
 ifeq ($(strip $(CTPC)), yes)
   CONVERT_TO_PROTON_C=yes
@@ -313,7 +320,6 @@ ifneq ("$(wildcard $(USER_PATH)/config.h)","")
     CONFIG_H += $(USER_PATH)/config.h
 endif
 
-
 # Object files directory
 #     To put object files in current directory, use a dot (.), do NOT make
 #     this an empty or blank macro!
@@ -323,7 +329,7 @@ ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
 
-# # project specific files
+# project specific files
 SRC += $(KEYBOARD_SRC) \
     $(KEYMAP_C) \
     $(QUANTUM_SRC)

docs/_summary.md

@@ -8,6 +8,7 @@
 
 * [QMK Basics](README.md)
   * [QMK Introduction](getting_started_introduction.md)
+  * [QMK CLI](cli.md)
   * [Contributing to QMK](contributing.md)
   * [How to Use Github](getting_started_github.md)
   * [Getting Help](getting_started_getting_help.md)
@@ -34,6 +35,8 @@
   * [Keyboard Guidelines](hardware_keyboard_guidelines.md)
   * [Config Options](config_options.md)
   * [Keycodes](keycodes.md)
+  * [Coding Conventions - C](coding_conventions_c.md)
+  * [Coding Conventions - Python](coding_conventions_python.md)
   * [Documentation Best Practices](documentation_best_practices.md)
   * [Documentation Templates](documentation_templates.md)
   * [Glossary](reference_glossary.md)
@@ -41,6 +44,7 @@
   * [Useful Functions](ref_functions.md)
   * [Configurator Support](reference_configurator_support.md)
   * [info.json Format](reference_info_json.md)
+  * [Python Development](python_development.md)
 
 * [Features](features.md)
   * [Basic Keycodes](keycodes_basic.md)

docs/cli.md

@@ -0,0 +1,31 @@
+# QMK CLI
+
+This page describes how to setup and use the QMK CLI.
+
+# Overview
+
+The QMK CLI makes building and working with QMK keyboards easier. We have provided a number of commands to help you work with QMK:
+
+* `qmk compile-json`
+
+# Setup
+
+Simply add the `qmk_firmware/bin` directory to your `PATH`. You can run the `qmk` commands from any directory.
+
+```
+export PATH=$PATH:$HOME/qmk_firmware/bin
+```
+
+You may want to add this to your `.profile`, `.bash_profile`, `.zsh_profile`, or other shell startup scripts.
+
+# Commands
+
+## `qmk compile-json`
+
+This command allows you to compile JSON files you have downloaded from <https://config.qmk.fm>.
+
+**Usage**:
+
+```
+qmk compile-json mine.json
+```

docs/coding_conventions_c.md

@@ -0,0 +1,58 @@
+# Coding Conventions (C)
+
+Most of our style is pretty easy to pick up on, but right now it's not entirely consistent. You should match the style of the code surrounding your change, but if that code is inconsistent or unclear use the following guidelines:
+
+* We indent using four (4) spaces (soft tabs)
+* We use a modified One True Brace Style
+  * Opening Brace: At the end of the same line as the statement that opens the block
+  * Closing Brace: Lined up with the first character of the statement that opens the block
+  * Else If: Place the closing brace at the beginning of the line and the next opening brace at the end of the same line.
+  * Optional Braces: Always include optional braces.
+    * Good: if (condition) { return false; }
+    * Bad: if (condition) return false;
+* We encourage use of C style comments: `/* */`
+  * Think of them as a story describing the feature
+  * Use them liberally to explain why particular decisions were made.
+  * Do not write obvious comments
+  * If you not sure if a comment is obvious, go ahead and include it.
+* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
+* We use `#pragma once` at the start of header files rather than old-style include guards (`#ifndef THIS_FILE_H`, `#define THIS_FILE_H`, ..., `#endif`)
+* We accept both forms of preprocessor if's: `#ifdef DEFINED` and `#if defined(DEFINED)`
+  * If you are not sure which to prefer use the `#if defined(DEFINED)` form.
+  * Do not change existing code from one style to the other, except when moving to a multiple condition `#if`.
+  * Do not put whitespace between `#` and `if`.
+  * When deciding how (or if) to indent directives keep these points in mind:
+    * Readability is more important than consistency.
+    * Follow the file's existing style. If the file is mixed follow the style that makes sense for the section you are modifying.
+    * When choosing to indent you can follow the indention level of the surrounding C code, or preprocessor directives can have their own indent level. Choose the style that best communicates the intent of your code.
+
+Here is an example for easy reference:
+
+```c
+/* Enums for foo */
+enum foo_state {
+  FOO_BAR,
+  FOO_BAZ,
+};
+
+/* Returns a value */
+int foo(void) {
+  if (some_condition) {
+    return FOO_BAR;
+  } else {
+    return -1;
+  }
+}
+```
+
+# Auto-formatting with clang-format
+
+[Clang-format](https://clang.llvm.org/docs/ClangFormat.html) is part of LLVM and can automatically format your code for you, because ain't nobody got time to do it manually. We supply a configuration file for it that applies most of the coding conventions listed above. It will only change whitespace and newlines, so you will still have to remember to include optional braces yourself.
+
+Use the [full LLVM installer](http://llvm.org/builds/) to get clang-format on Windows, or use `sudo apt install clang-format` on Ubuntu.
+
+If you run it from the command-line, pass `-style=file` as an option and it will automatically find the .clang-format configuration file in the QMK root directory.
+
+If you use VSCode, the standard C/C++ plugin supports clang-format, alternatively there is a [separate extension](https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat) for it.
+
+Some things (like LAYOUT macros) are destroyed by clang-format, so either don't run it on those files, or wrap the sensitive code in `// clang-format off` and `// clang-format on`.

docs/coding_conventions_python.md

@@ -0,0 +1,314 @@
+# Coding Conventions (Python)
+
+Most of our style follows PEP8 with some local modifications to make things less nit-picky. 
+
+* We target Python 3.5 for compatability with all supported platforms.
+* We indent using four (4) spaces (soft tabs)
+* We encourage liberal use of comments
+  * Think of them as a story describing the feature
+  * Use them liberally to explain why particular decisions were made.
+  * Do not write obvious comments
+  * If you not sure if a comment is obvious, go ahead and include it.
+* We require useful docstrings for all functions.
+* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
+* Some of our practices conflict with the wider python community to make our codebase more approachable to non-pythonistas.
+
+# YAPF
+
+You can use [yapf](https://github.com/google/yapf) to style your code. We provide a config in [setup.cfg](setup.cfg).
+
+# Imports
+
+We don't have a hard and fast rule for when to use `import ...` vs `from ... import ...`. Understandability and maintainability is our ultimate goal.
+
+Generally we prefer to import specific function and class names from a module to keep code shorter and easier to understand. Sometimes this results in a name that is ambiguous, and in such cases we prefer to import the module instead. You should avoid using the "as" keyword when importing, unless you are importing a compatability module.
+
+Imports should be one line per module. We group import statements together using the standard python rules- system, 3rd party, local.
+
+Do not use `from foo import *`. Supply a list of objects you want to import instead, or import the whole module.
+
+## Import Examples
+
+Good:
+
+```
+from qmk import effects
+
+effects.echo()
+```
+
+Bad:
+
+```
+from qmk.effects import echo
+
+echo()  # It's unclear where echo comes from
+```
+
+Good:
+
+```
+from qmk.keymap import compile_firmware
+
+compile_firmware()
+```
+
+OK, but the above is better:
+
+```
+import qmk.keymap
+
+qmk.keymap.compile_firmware()
+```
+
+# Statements
+
+One statement per line.
+
+Even when allowed (EG `if foo: bar`) we do not combine 2 statements onto a single line.
+
+# Naming
+
+`module_name`, `package_name`, `ClassName`, `method_name`, `ExceptionName`, `function_name`, `GLOBAL_CONSTANT_NAME`, `global_var_name`, `instance_var_name`, `function_parameter_name`, `local_var_name`.
+
+Function names, variable names, and filenames should be descriptive; eschew abbreviation. In particular, do not use abbreviations that are ambiguous or unfamiliar to readers outside your project, and do not abbreviate by deleting letters within a word.
+
+Always use a .py filename extension. Never use dashes.
+
+## Names to Avoid
+
+* single character names except for counters or iterators. You may use "e" as an exception identifier in try/except statements.
+* dashes (-) in any package/module name
+* __double_leading_and_trailing_underscore__ names (reserved by Python)
+
+# Docstrings
+
+To maintain consistency with our docstrings we've set out the following guidelines.
+
+* Use markdown formatting
+* Always use triple-dquote docstrings with at least one linebreak: `"""\n"""`
+* First line is a short (< 70 char) description of what the function does
+* If you need more in your docstring leave a blank line between the description and the rest.
+* Start indented lines at the same indent level as the opening triple-dquote
+* Document all function arguments using the format described below
+* If present, Args:, Returns:, and Raises: should be the last three things in the docstring, separated by a blank line each.
+
+## Simple docstring example
+
+```
+def my_awesome_function():
+    """Return the number of seconds since 1970 Jan 1 00:00 UTC.
+    """
+    return int(time.time())
+```
+
+## Complex docstring example
+
+```
+def my_awesome_function():
+    """Return the number of seconds since 1970 Jan 1 00:00 UTC.
+
+    This function always returns an integer number of seconds.
+    """
+    return int(time.time())
+```
+
+## Function arguments docstring example
+
+```
+def my_awesome_function(start=None, offset=0):
+    """Return the number of seconds since 1970 Jan 1 00:00 UTC.
+
+    This function always returns an integer number of seconds.
+
+
+    Args:
+        start
+            The time to start at instead of 1970 Jan 1 00:00 UTC
+
+        offset
+            Return an answer that has this number of seconds subtracted first
+
+    Returns:
+        An integer describing a number of seconds.
+
+    Raises:
+        ValueError
+            When `start` or `offset` are not positive numbers
+    """
+    if start < 0 or offset < 0:
+        raise ValueError('start and offset must be positive numbers.')
+
+    if not start:
+        start = time.time()
+
+    return int(start - offset)
+```
+
+# Exceptions
+
+Exceptions are used to handle exceptional situations. They should not be used for flow control. This is a break from the python norm of "ask for forgiveness." If you are catching an exception it should be to handle a situation that is unusual.
+
+If you use a catch-all exception for any reason you must log the exception and stacktrace using cli.log.
+
+Make your try/except blocks as short as possible. If you need a lot of try statements you may need to restructure your code.
+
+# Tuples
+
+When defining one-item tuples always include a trailing comma so that it is obvious you are using a tuple. Do not rely on implicit one-item tuple unpacking. Better still use a list which is unambiguous.
+
+This is particularly important when using the printf-style format strings that are commonly used.
+
+# Lists and Dictionaries
+
+We have configured YAPF to differentiate between sequence styles with a trailing comma. When a trailing comma is omitted YAPF will format the sequence as a single line. When a trailing comma is included YAPF will format the sequence with one item per line.
+
+You should generally prefer to keep short definition on a single line. Break out to multiple lines sooner rather than later to aid readability and maintainability.
+
+# Parentheses
+
+Avoid excessive parentheses, but do use parentheses to make code easier to understand. Do not use them in return statements unless you are explicitly returning a tuple, or it is part of a math expression.
+
+# Format Strings
+
+We generally prefer printf-style format strings. Example:
+
+```
+name = 'World'
+print('Hello, %s!' % (name,))
+```
+
+This style is used by the logging module, which we make use of extensively, and we have adopted it in other places for consistency. It is also more familiar to C programmers, who are a big part of our casual audience.
+
+Our included CLI module has support for using these without using the percent (%) operator. Look at `cli.echo()` and the various `cli.log` functions (EG, `cli.log.info()`) for more details.
+
+# Comprehensions & Generator Expressions
+
+We encourage the liberal use of comprehensions and generators, but do not let them get too complex. If you need complexity fall back to a for loop that is easier to understand.
+
+# Lambdas
+
+OK to use but probably should be avoided. With comprehensions and generators the need for lambdas is not as strong as it once was.
+
+# Conditional Expressions
+
+OK in variable assignment, but otherwise should be avoided.
+
+Conditional expressions are if statements that are in line with code. For example:
+
+```
+x = 1 if cond else 2
+```
+
+It's generally not a good idea to use these as function arguments, sequence items, etc. It's too easy to overlook.
+
+# Default Argument Values
+
+Encouraged, but values must be immutable objects.
+
+When specifying default values in argument lists always be careful to specify objects that can't be modified in place. If you use a mutable object the changes you make will persist between calls, which is usually not what you want. Even if that is what you intend to do it is confusing for others and will hinder understanding.
+
+Bad:
+
+```
+def my_func(foo={}):
+    pass
+```
+
+Good:
+
+```
+def my_func(foo=None):
+    if not foo:
+        foo = {}
+```
+
+# Properties
+
+Always use properties instead of getter and setter functions.
+
+```
+class Foo(object):
+    def __init__(self):
+        self._bar = None
+
+    @property
+    def bar(self):
+        return self._bar
+
+    @bar.setter
+    def bar(self, bar):
+        self._bar = bar
+```
+
+# True/False Evaluations
+
+You should generally prefer the implicit True/False evaluation in if statements, rather than checking equivalency.
+
+Bad:
+
+```
+if foo == True:
+    pass
+
+if bar == False:
+    pass
+```
+
+Good:
+
+```
+if foo:
+    pass
+
+if not bar:
+    pass
+```
+
+# Decorators
+
+Use when appropriate. Try to avoid too much magic unless it helps with understanding.
+
+# Threading and Multiprocessing
+
+Should be avoided. If you need this you will have to make a strong case before we merge your code.
+
+# Power Features
+
+Python is an extremely flexible language and gives you many fancy features such as custom metaclasses, access to bytecode, on-the-fly compilation, dynamic inheritance, object reparenting, import hacks, reflection, modification of system internals, etc.
+
+Don't use these.
+
+Performance is not a critical concern for us, and code understandability is. We want our codebase to be approachable by someone who only has a day or two to play with it. These features generally come with a cost to easy understanding, and we would prefer to have code that can be readily understood over faster or more compact code.
+
+Note that some standard library modules use these techniques and it is ok to make use of those modules. But please keep readability and understandability in mind when using them.
+
+# Type Annotated Code
+
+For now we are not using any type annotation system, and would prefer that code remain unannotated. We may revisit this in the future.
+
+# Function length
+
+Prefer small and focused functions.
+
+We recognize that long functions are sometimes appropriate, so no hard limit is placed on function length. If a function exceeds about 40 lines, think about whether it can be broken up without harming the structure of the program.
+
+Even if your long function works perfectly now, someone modifying it in a few months may add new behavior. This could result in bugs that are hard to find. Keeping your functions short and simple makes it easier for other people to read and modify your code.
+
+You could find long and complicated functions when working with some code. Do not be intimidated by modifying existing code: if working with such a function proves to be difficult, you find that errors are hard to debug, or you want to use a piece of it in several different contexts, consider breaking up the function into smaller and more manageable pieces.
+
+# FIXMEs
+
+It is OK to leave FIXMEs in code. Why? Encouraging people to at least document parts of code that need to be thought out more (or that are confusing) is better than leaving this code undocumented.
+
+All FIXMEs should be formatted like:
+
+```
+FIXME(username): Revisit this code when the frob feature is done.
+```
+
+...where username is your GitHub username.
+
+# Unit Tests
+
+These are good. We should have some one day.

docs/contributing.md

@@ -54,62 +54,10 @@ Never made an open source contribution before? Wondering how contributions work
 
 # Coding Conventions
 
-Most of our style is pretty easy to pick up on, but right now it's not entirely consistent. You should match the style of the code surrounding your change, but if that code is inconsistent or unclear use the following guidelines:
+Most of our style is pretty easy to pick up on. If you are familiar with either C or Python you should not have too much trouble with our local styles.
 
-* We indent using four (4) spaces (soft tabs)
+* [Coding Conventions - C](coding_conventions_c.md)
-* We use a modified One True Brace Style
+* [Coding Conventions - Python](coding_conventions_python.md)
-  * Opening Brace: At the end of the same line as the statement that opens the block
-  * Closing Brace: Lined up with the first character of the statement that opens the block
-  * Else If: Place the closing brace at the beginning of the line and the next opening brace at the end of the same line.
-  * Optional Braces: Always include optional braces.
-    * Good: if (condition) { return false; }
-    * Bad: if (condition) return false;
-* We encourage use of C style comments: `/* */`
-  * Think of them as a story describing the feature
-  * Use them liberally to explain why particular decisions were made.
-  * Do not write obvious comments
-  * If you not sure if a comment is obvious, go ahead and include it.
-* In general we don't wrap lines, they can be as long as needed. If you do choose to wrap lines please do not wrap any wider than 76 columns.
-* We use `#pragma once` at the start of header files rather than old-style include guards (`#ifndef THIS_FILE_H`, `#define THIS_FILE_H`, ..., `#endif`)
-* We accept both forms of preprocessor if's: `#ifdef DEFINED` and `#if defined(DEFINED)`
-  * If you are not sure which to prefer use the `#if defined(DEFINED)` form.
-  * Do not change existing code from one style to the other, except when moving to a multiple condition `#if`.
-  * Do not put whitespace between `#` and `if`.
-  * When deciding how (or if) to indent directives keep these points in mind:
-    * Readability is more important than consistency.
-    * Follow the file's existing style. If the file is mixed follow the style that makes sense for the section you are modifying.
-    * When choosing to indent you can follow the indention level of the surrounding C code, or preprocessor directives can have their own indent level. Choose the style that best communicates the intent of your code.
-
-Here is an example for easy reference:
-
-```c
-/* Enums for foo */
-enum foo_state {
-  FOO_BAR,
-  FOO_BAZ,
-};
-
-/* Returns a value */
-int foo(void) {
-  if (some_condition) {
-    return FOO_BAR;
-  } else {
-    return -1;
-  }
-}
-```
-
-# Auto-formatting with clang-format
-
-[Clang-format](https://clang.llvm.org/docs/ClangFormat.html) is part of LLVM and can automatically format your code for you, because ain't nobody got time to do it manually. We supply a configuration file for it that applies most of the coding conventions listed above. It will only change whitespace and newlines, so you will still have to remember to include optional braces yourself.
-
-Use the [full LLVM installer](http://llvm.org/builds/) to get clang-format on Windows, or use `sudo apt install clang-format` on Ubuntu.
-
-If you run it from the command-line, pass `-style=file` as an option and it will automatically find the .clang-format configuration file in the QMK root directory.
-
-If you use VSCode, the standard C/C++ plugin supports clang-format, alternatively there is a [separate extension](https://marketplace.visualstudio.com/items?itemName=LLVMExtensions.ClangFormat) for it.
-
-Some things (like LAYOUT macros) are destroyed by clang-format, so either don't run it on those files, or wrap the sensitive code in `// clang-format off` and `// clang-format on`.
 
 # General Guidelines
 

docs/python_development.md

@@ -0,0 +1,45 @@
+# Python Development in QMK
+
+This document gives an overview of how QMK has structured its python code. You should read this before working on any of the python code.
+
+## Script directories
+
+There are two places scripts live in QMK: `qmk_firmware/bin` and `qmk_firmware/util`. You should use `bin` for any python scripts that utilize the `qmk` wrapper. Scripts that are standalone and not run very often live in `util`.
+
+We discourage putting anything into `bin` that does not utilize the `qmk` wrapper. If you think you have a good reason for doing so please talk to us about your use case.
+
+## Python Modules
+
+Most of the QMK python modules can be found in `qmk_firmware/lib/python`. This is the path that we append to `sys.path`.
+
+We have a module hierarchy under that path:
+
+* `qmk_firmware/lib/python`
+    * `milc.py` - The CLI library we use. Will be pulled out into its own module in the future.
+    * `qmk` - Code associated with QMK
+        * `cli` - Modules that will be imported for CLI commands.
+        * `errors.py` - Errors that can be raised within QMK apps
+        * `keymap.py` - Functions for working with keymaps
+
+## CLI Scripts
+
+We have a CLI wrapper that you should utilize for any user facing scripts. We think it's pretty easy to use and it gives you a lot of nice things for free.
+
+To use the wrapper simply place a module into `qmk_firmware/lib/python/qmk/cli`, and create a symlink to `bin/qmk` named after your module. Dashes in command names will be converted into dots so you can use hierarchy to manage commands.
+
+When `qmk` is run it checks to see how it was invoked. If it was invoked as `qmk` the module name is take from `sys.argv[1]`. If it was invoked as `qmk-<module-name>` then everything after the first dash is taken as the module name. Dashes and underscores are converted to dots, and then `qmk.cli` is prepended before the module is imported.
+
+The module uses `@cli.entrypoint()` and `@cli.argument()` decorators to define an entrypoint, which is where execution starts.
+
+## Example CLI Script
+
+We have provided a QMK Hello World script you can use as an example. To run it simply run `qmk hello` or `qmk-hello`. The source code is listed below.
+
+```
+from milc import cli
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Python Hello World.')
+def main(cli):
+    cli.echo('Hello, %s!', cli.config.general.name)
+```

keyboards/clueboard/66_hotswap/keymaps/json/keymap.json

@@ -0,0 +1 @@
+{"keyboard":"clueboard/66_hotswap/gen1","keymap":"default_66","layout":"LAYOUT","layers":[["KC_GESC","KC_1","KC_2","KC_3","KC_4","KC_5","KC_6","KC_7","KC_8","KC_9","KC_0","KC_MINS","KC_EQL","KC_BSPC","KC_PGUP","KC_TAB","KC_Q","KC_W","KC_E","KC_R","KC_T","KC_Y","KC_U","KC_I","KC_O","KC_P","KC_LBRC","KC_RBRC","KC_BSLS","KC_PGDN","KC_CAPS","KC_A","KC_S","KC_D","KC_F","KC_G","KC_H","KC_J","KC_K","KC_L","KC_SCLN","KC_QUOT","KC_ENT","KC_LSFT","KC_Z","KC_X","KC_C","KC_V","KC_B","KC_N","KC_M","KC_COMM","KC_DOT","KC_SLSH","KC_RSFT","KC_UP","KC_LCTL","KC_LGUI","KC_LALT","KC_SPC","KC_SPC","KC_RALT","KC_RGUI","MO(1)","KC_RCTL","KC_LEFT","KC_DOWN","KC_RGHT"],["KC_GRV","KC_F1","KC_F2","KC_F3","KC_F4","KC_F5","KC_F6","KC_F7","KC_F8","KC_F9","KC_F10","KC_F11","KC_F12","KC_DEL","BL_INC","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_MPRV","KC_MPLY","KC_MNXT","KC_NO","KC_MUTE","BL_DEC","KC_NO","KC_NO","MO(2)","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_PGUP","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","MO(1)","KC_NO","KC_HOME","KC_PGDN","KC_END"],["KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_TOGG","BL_INC","KC_NO","KC_NO","KC_NO","KC_NO","RESET","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_DEC","KC_NO","KC_NO","MO(2)","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","KC_NO","BL_STEP","KC_NO","KC_NO","MO(1)","KC_NO","KC_NO","KC_NO","KC_NO"]],"author":"","notes":""}

lib/python/milc.py

@@ -0,0 +1,716 @@
+#!/usr/bin/env python3
+# coding=utf-8
+"""MILC - A CLI Framework
+
+PYTHON_ARGCOMPLETE_OK
+
+MILC is an opinionated framework for writing CLI apps. It optimizes for the
+most common unix tool pattern- small tools that are run from the command
+line but generally do not feature any user interaction while they run.
+
+For more details see the MILC documentation:
+
+    <https://github.com/clueboard/milc/tree/master/docs>
+"""
+from __future__ import division, print_function, unicode_literals
+import argparse
+import logging
+import os
+import re
+import sys
+from decimal import Decimal
+from tempfile import NamedTemporaryFile
+from time import sleep
+
+try:
+    from ConfigParser import RawConfigParser
+except ImportError:
+    from configparser import RawConfigParser
+
+try:
+    import thread
+    import threading
+except ImportError:
+    thread = None
+
+import argcomplete
+import colorama
+
+# Log Level Representations
+EMOJI_LOGLEVELS = {
+    'CRITICAL': '{bg_red}{fg_white}¬_¬{style_reset_all}',
+    'ERROR': '{fg_red}☒{style_reset_all}',
+    'WARNING': '{fg_yellow}⚠{style_reset_all}',
+    'INFO': '{fg_blue}ℹ{style_reset_all}',
+    'DEBUG': '{fg_cyan}☐{style_reset_all}',
+    'NOTSET': '{style_reset_all}¯\\_(o_o)_/¯'
+}
+EMOJI_LOGLEVELS['FATAL'] = EMOJI_LOGLEVELS['CRITICAL']
+EMOJI_LOGLEVELS['WARN'] = EMOJI_LOGLEVELS['WARNING']
+
+# ANSI Color setup
+# Regex was gratefully borrowed from kfir on stackoverflow:
+# https://stackoverflow.com/a/45448194
+ansi_regex = r'\x1b(' \
+             r'(\[\??\d+[hl])|' \
+             r'([=<>a-kzNM78])|' \
+             r'([\(\)][a-b0-2])|' \
+             r'(\[\d{0,2}[ma-dgkjqi])|' \
+             r'(\[\d+;\d+[hfy]?)|' \
+             r'(\[;?[hf])|' \
+             r'(#[3-68])|' \
+             r'([01356]n)|' \
+             r'(O[mlnp-z]?)|' \
+             r'(/Z)|' \
+             r'(\d+)|' \
+             r'(\[\?\d;\d0c)|' \
+             r'(\d;\dR))'
+ansi_escape = re.compile(ansi_regex, flags=re.IGNORECASE)
+ansi_styles = (
+    ('fg', colorama.ansi.AnsiFore()),
+    ('bg', colorama.ansi.AnsiBack()),
+    ('style', colorama.ansi.AnsiStyle()),
+)
+ansi_colors = {}
+
+for prefix, obj in ansi_styles:
+    for color in [x for x in obj.__dict__ if not x.startswith('_')]:
+        ansi_colors[prefix + '_' + color.lower()] = getattr(obj, color)
+
+
+def format_ansi(text):
+    """Return a copy of text with certain strings replaced with ansi.
+    """
+    # Avoid .format() so we don't have to worry about the log content
+    for color in ansi_colors:
+        text = text.replace('{%s}' % color, ansi_colors[color])
+    return text + ansi_colors['style_reset_all']
+
+
+class ANSIFormatter(logging.Formatter):
+    """A log formatter that inserts ANSI color.
+    """
+
+    def format(self, record):
+        msg = super(ANSIFormatter, self).format(record)
+        return format_ansi(msg)
+
+
+class ANSIEmojiLoglevelFormatter(ANSIFormatter):
+    """A log formatter that makes the loglevel an emoji.
+    """
+
+    def format(self, record):
+        record.levelname = EMOJI_LOGLEVELS[record.levelname].format(**ansi_colors)
+        return super(ANSIEmojiLoglevelFormatter, self).format(record)
+
+
+class ANSIStrippingFormatter(ANSIFormatter):
+    """A log formatter that strips ANSI.
+    """
+
+    def format(self, record):
+        msg = super(ANSIStrippingFormatter, self).format(record)
+        return ansi_escape.sub('', msg)
+
+
+class Configuration(object):
+    """Represents the running configuration.
+
+    This class never raises IndexError, instead it will return None if a
+    section or option does not yet exist.
+    """
+
+    def __contains__(self, key):
+        return self._config.__contains__(key)
+
+    def __iter__(self):
+        return self._config.__iter__()
+
+    def __len__(self):
+        return self._config.__len__()
+
+    def __repr__(self):
+        return self._config.__repr__()
+
+    def keys(self):
+        return self._config.keys()
+
+    def items(self):
+        return self._config.items()
+
+    def values(self):
+        return self._config.values()
+
+    def __init__(self, *args, **kwargs):
+        self._config = {}
+        self.default_container = ConfigurationOption
+
+    def __getitem__(self, key):
+        """Returns a config section, creating it if it doesn't exist yet.
+        """
+        if key not in self._config:
+            self.__dict__[key] = self._config[key] = ConfigurationOption()
+
+        return self._config[key]
+
+    def __setitem__(self, key, value):
+        self.__dict__[key] = value
+        self._config[key] = value
+
+    def __delitem__(self, key):
+        if key in self.__dict__ and key[0] != '_':
+            del self.__dict__[key]
+        del self._config[key]
+
+
+class ConfigurationOption(Configuration):
+    def __init__(self, *args, **kwargs):
+        super(ConfigurationOption, self).__init__(*args, **kwargs)
+        self.default_container = dict
+
+    def __getitem__(self, key):
+        """Returns a config section, creating it if it doesn't exist yet.
+        """
+        if key not in self._config:
+            self.__dict__[key] = self._config[key] = None
+
+        return self._config[key]
+
+
+def handle_store_boolean(self, *args, **kwargs):
+    """Does the add_argument for action='store_boolean'.
+    """
+    kwargs['add_dest'] = False
+    disabled_args = None
+    disabled_kwargs = kwargs.copy()
+    disabled_kwargs['action'] = 'store_false'
+    disabled_kwargs['help'] = 'Disable ' + kwargs['help']
+    kwargs['action'] = 'store_true'
+    kwargs['help'] = 'Enable ' + kwargs['help']
+
+    for flag in args:
+        if flag[:2] == '--':
+            disabled_args = ('--no-' + flag[2:],)
+            break
+
+    self.add_argument(*args, **kwargs)
+    self.add_argument(*disabled_args, **disabled_kwargs)
+
+    return (args, kwargs, disabled_args, disabled_kwargs)
+
+
+class SubparserWrapper(object):
+    """Wrap subparsers so we can populate the normal and the shadow parser.
+    """
+
+    def __init__(self, cli, submodule, subparser):
+        self.cli = cli
+        self.submodule = submodule
+        self.subparser = subparser
+
+        for attr in dir(subparser):
+            if not hasattr(self, attr):
+                setattr(self, attr, getattr(subparser, attr))
+
+    def completer(self, completer):
+        """Add an arpcomplete completer to this subcommand.
+        """
+        self.subparser.completer = completer
+
+    def add_argument(self, *args, **kwargs):
+        if kwargs.get('add_dest', True):
+            kwargs['dest'] = self.submodule + '_' + self.cli.get_argument_name(*args, **kwargs)
+        if 'add_dest' in kwargs:
+            del kwargs['add_dest']
+
+        if 'action' in kwargs and kwargs['action'] == 'store_boolean':
+            return handle_store_boolean(self, *args, **kwargs)
+
+        self.cli.acquire_lock()
+        self.subparser.add_argument(*args, **kwargs)
+
+        if 'default' in kwargs:
+            del kwargs['default']
+        if 'action' in kwargs and kwargs['action'] == 'store_false':
+            kwargs['action'] == 'store_true'
+        self.cli.subcommands_default[self.submodule].add_argument(*args, **kwargs)
+        self.cli.release_lock()
+
+
+class MILC(object):
+    """MILC - An Opinionated Batteries Included Framework
+    """
+
+    def __init__(self):
+        """Initialize the MILC object.
+        """
+        # Setup a lock for thread safety
+        self._lock = threading.RLock() if thread else None
+
+        # Define some basic info
+        self.acquire_lock()
+        self._description = None
+        self._entrypoint = None
+        self._inside_context_manager = False
+        self.ansi = ansi_colors
+        self.config = Configuration()
+        self.config_file = None
+        self.prog_name = sys.argv[0][:-3] if sys.argv[0].endswith('.py') else sys.argv[0]
+        self.version = os.environ.get('QMK_VERSION', 'unknown')
+        self.release_lock()
+
+        # Initialize all the things
+        self.initialize_argparse()
+        self.initialize_logging()
+
+    @property
+    def description(self):
+        return self._description
+
+    @description.setter
+    def description(self, value):
+        self._description = self._arg_parser.description = self._arg_defaults.description = value
+
+    def echo(self, text, *args, **kwargs):
+        """Print colorized text to stdout, as long as stdout is a tty.
+
+        ANSI color strings (such as {fg-blue}) will be converted into ANSI
+        escape sequences, and the ANSI reset sequence will be added to all
+        strings.
+
+        If *args or **kwargs are passed they will be used to %-format the strings.
+        """
+        if args and kwargs:
+            raise RuntimeError('You can only specify *args or **kwargs, not both!')
+
+        if sys.stdout.isatty():
+            args = args or kwargs
+            text = format_ansi(text)
+
+            print(text % args)
+
+    def initialize_argparse(self):
+        """Prepare to process arguments from sys.argv.
+        """
+        kwargs = {
+            'fromfile_prefix_chars': '@',
+            'conflict_handler': 'resolve',
+        }
+
+        self.acquire_lock()
+        self.subcommands = {}
+        self.subcommands_default = {}
+        self._subparsers = None
+        self._subparsers_default = None
+        self.argwarn = argcomplete.warn
+        self.args = None
+        self._arg_defaults = argparse.ArgumentParser(**kwargs)
+        self._arg_parser = argparse.ArgumentParser(**kwargs)
+        self.set_defaults = self._arg_parser.set_defaults
+        self.print_usage = self._arg_parser.print_usage
+        self.print_help = self._arg_parser.print_help
+        self.release_lock()
+
+    def completer(self, completer):
+        """Add an arpcomplete completer to this subcommand.
+        """
+        self._arg_parser.completer = completer
+
+    def add_argument(self, *args, **kwargs):
+        """Wrapper to add arguments to both the main and the shadow argparser.
+        """
+        if kwargs.get('add_dest', True) and args[0][0] == '-':
+            kwargs['dest'] = 'general_' + self.get_argument_name(*args, **kwargs)
+        if 'add_dest' in kwargs:
+            del kwargs['add_dest']
+
+        if 'action' in kwargs and kwargs['action'] == 'store_boolean':
+            return handle_store_boolean(self, *args, **kwargs)
+
+        self.acquire_lock()
+        self._arg_parser.add_argument(*args, **kwargs)
+
+        # Populate the shadow parser
+        if 'default' in kwargs:
+            del kwargs['default']
+        if 'action' in kwargs and kwargs['action'] == 'store_false':
+            kwargs['action'] == 'store_true'
+        self._arg_defaults.add_argument(*args, **kwargs)
+        self.release_lock()
+
+    def initialize_logging(self):
+        """Prepare the defaults for the logging infrastructure.
+        """
+        self.acquire_lock()
+        self.log_file = None
+        self.log_file_mode = 'a'
+        self.log_file_handler = None
+        self.log_print = True
+        self.log_print_to = sys.stderr
+        self.log_print_level = logging.INFO
+        self.log_file_level = logging.DEBUG
+        self.log_level = logging.INFO
+        self.log = logging.getLogger(self.__class__.__name__)
+        self.log.setLevel(logging.DEBUG)
+        logging.root.setLevel(logging.DEBUG)
+        self.release_lock()
+
+        self.add_argument('-V', '--version', version=self.version, action='version', help='Display the version and exit')
+        self.add_argument('-v', '--verbose', action='store_true', help='Make the logging more verbose')
+        self.add_argument('--datetime-fmt', default='%Y-%m-%d %H:%M:%S', help='Format string for datetimes')
+        self.add_argument('--log-fmt', default='%(levelname)s %(message)s', help='Format string for printed log output')
+        self.add_argument('--log-file-fmt', default='[%(levelname)s] [%(asctime)s] [file:%(pathname)s] [line:%(lineno)d] %(message)s', help='Format string for log file.')
+        self.add_argument('--log-file', help='File to write log messages to')
+        self.add_argument('--color', action='store_boolean', default=True, help='color in output')
+        self.add_argument('-c', '--config-file', help='The config file to read and/or write')
+        self.add_argument('--save-config', action='store_true', help='Save the running configuration to the config file')
+
+    def add_subparsers(self, title='Sub-commands', **kwargs):
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        self.acquire_lock()
+        self._subparsers_default = self._arg_defaults.add_subparsers(title=title, dest='subparsers', **kwargs)
+        self._subparsers = self._arg_parser.add_subparsers(title=title, dest='subparsers', **kwargs)
+        self.release_lock()
+
+    def acquire_lock(self):
+        """Acquire the MILC lock for exclusive access to properties.
+        """
+        if self._lock:
+            self._lock.acquire()
+
+    def release_lock(self):
+        """Release the MILC lock.
+        """
+        if self._lock:
+            self._lock.release()
+
+    def find_config_file(self):
+        """Locate the config file.
+        """
+        if self.config_file:
+            return self.config_file
+
+        if self.args and self.args.general_config_file:
+            return self.args.general_config_file
+
+        return os.path.abspath(os.path.expanduser('~/.%s.ini' % self.prog_name))
+
+    def get_argument_name(self, *args, **kwargs):
+        """Takes argparse arguments and returns the dest name.
+        """
+        try:
+            return self._arg_parser._get_optional_kwargs(*args, **kwargs)['dest']
+        except ValueError:
+            return self._arg_parser._get_positional_kwargs(*args, **kwargs)['dest']
+
+    def argument(self, *args, **kwargs):
+        """Decorator to call self.add_argument or self.<subcommand>.add_argument.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        def argument_function(handler):
+            if handler is self._entrypoint:
+                self.add_argument(*args, **kwargs)
+
+            elif handler.__name__ in self.subcommands:
+                self.subcommands[handler.__name__].add_argument(*args, **kwargs)
+
+            else:
+                raise RuntimeError('Decorated function is not entrypoint or subcommand!')
+
+            return handler
+
+        return argument_function
+
+    def arg_passed(self, arg):
+        """Returns True if arg was passed on the command line.
+        """
+        return self.args_passed[arg] in (None, False)
+
+    def parse_args(self):
+        """Parse the CLI args.
+        """
+        if self.args:
+            self.log.debug('Warning: Arguments have already been parsed, ignoring duplicate attempt!')
+            return
+
+        argcomplete.autocomplete(self._arg_parser)
+
+        self.acquire_lock()
+        self.args = self._arg_parser.parse_args()
+        self.args_passed = self._arg_defaults.parse_args()
+
+        if 'entrypoint' in self.args:
+            self._entrypoint = self.args.entrypoint
+
+        if self.args.general_config_file:
+            self.config_file = self.args.general_config_file
+
+        self.release_lock()
+
+    def read_config(self):
+        """Parse the configuration file and determine the runtime configuration.
+        """
+        self.acquire_lock()
+        self.config_file = self.find_config_file()
+
+        if self.config_file and os.path.exists(self.config_file):
+            config = RawConfigParser(self.config)
+            config.read(self.config_file)
+
+            # Iterate over the config file options and write them into self.config
+            for section in config.sections():
+                for option in config.options(section):
+                    value = config.get(section, option)
+
+                    # Coerce values into useful datatypes
+                    if value.lower() in ['1', 'yes', 'true', 'on']:
+                        value = True
+                    elif value.lower() in ['0', 'no', 'false', 'none', 'off']:
+                        value = False
+                    elif value.replace('.', '').isdigit():
+                        if '.' in value:
+                            value = Decimal(value)
+                        else:
+                            value = int(value)
+
+                    self.config[section][option] = value
+
+        # Fold the CLI args into self.config
+        for argument in vars(self.args):
+            if argument in ('subparsers', 'entrypoint'):
+                continue
+
+            if '_' not in argument:
+                continue
+
+            section, option = argument.split('_', 1)
+            if hasattr(self.args_passed, argument):
+                self.config[section][option] = getattr(self.args, argument)
+            else:
+                if option not in self.config[section]:
+                    self.config[section][option] = getattr(self.args, argument)
+
+        self.release_lock()
+
+    def save_config(self):
+        """Save the current configuration to the config file.
+        """
+        self.log.debug("Saving config file to '%s'", self.config_file)
+
+        if not self.config_file:
+            self.log.warning('%s.config_file file not set, not saving config!', self.__class__.__name__)
+            return
+
+        self.acquire_lock()
+
+        config = RawConfigParser()
+        for section_name, section in self.config._config.items():
+            config.add_section(section_name)
+            for option_name, value in section.items():
+                if section_name == 'general':
+                    if option_name in ['save_config']:
+                        continue
+                config.set(section_name, option_name, str(value))
+
+        with NamedTemporaryFile(mode='w', dir=os.path.dirname(self.config_file), delete=False) as tmpfile:
+            config.write(tmpfile)
+
+        # Move the new config file into place atomically
+        if os.path.getsize(tmpfile.name) > 0:
+            os.rename(tmpfile.name, self.config_file)
+        else:
+            self.log.warning('Config file saving failed, not replacing %s with %s.', self.config_file, tmpfile.name)
+
+        self.release_lock()
+
+    def __call__(self):
+        """Execute the entrypoint function.
+        """
+        if not self._inside_context_manager:
+            # If they didn't use the context manager use it ourselves
+            with self:
+                self.__call__()
+                return
+
+        if not self._entrypoint:
+            raise RuntimeError('No entrypoint provided!')
+
+        return self._entrypoint(self)
+
+    def entrypoint(self, description):
+        """Set the entrypoint for when no subcommand is provided.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before cli()!')
+
+        self.acquire_lock()
+        self.description = description
+        self.release_lock()
+
+        def entrypoint_func(handler):
+            self.acquire_lock()
+            self._entrypoint = handler
+            self.release_lock()
+
+            return handler
+
+        return entrypoint_func
+
+    def add_subcommand(self, handler, description, name=None, **kwargs):
+        """Register a subcommand.
+
+        If name is not provided we use `handler.__name__`.
+        """
+        if self._inside_context_manager:
+            raise RuntimeError('You must run this before the with statement!')
+
+        if self._subparsers is None:
+            self.add_subparsers()
+
+        if not name:
+            name = handler.__name__
+
+        self.acquire_lock()
+        kwargs['help'] = description
+        self.subcommands_default[name] = self._subparsers_default.add_parser(name, **kwargs)
+        self.subcommands[name] = SubparserWrapper(self, name, self._subparsers.add_parser(name, **kwargs))
+        self.subcommands[name].set_defaults(entrypoint=handler)
+
+        if name not in self.__dict__:
+            self.__dict__[name] = self.subcommands[name]
+        else:
+            self.log.debug("Could not add subcommand '%s' to attributes, key already exists!", name)
+
+        self.release_lock()
+
+        return handler
+
+    def subcommand(self, description, **kwargs):
+        """Decorator to register a subcommand.
+        """
+
+        def subcommand_function(handler):
+            return self.add_subcommand(handler, description, **kwargs)
+
+        return subcommand_function
+
+    def setup_logging(self):
+        """Called by __enter__() to setup the logging configuration.
+        """
+        if len(logging.root.handlers) != 0:
+            # This is not a design decision. This is what I'm doing for now until I can examine and think about this situation in more detail.
+            raise RuntimeError('MILC should be the only system installing root log handlers!')
+
+        self.acquire_lock()
+
+        if self.config['general']['verbose']:
+            self.log_print_level = logging.DEBUG
+
+        self.log_file = self.config['general']['log_file'] or self.log_file
+        self.log_file_format = self.config['general']['log_file_fmt']
+        self.log_file_format = ANSIStrippingFormatter(self.config['general']['log_file_fmt'], self.config['general']['datetime_fmt'])
+        self.log_format = self.config['general']['log_fmt']
+
+        if self.config.general.color:
+            self.log_format = ANSIEmojiLoglevelFormatter(self.args.general_log_fmt, self.config.general.datetime_fmt)
+        else:
+            self.log_format = ANSIStrippingFormatter(self.args.general_log_fmt, self.config.general.datetime_fmt)
+
+        if self.log_file:
+            self.log_file_handler = logging.FileHandler(self.log_file, self.log_file_mode)
+            self.log_file_handler.setLevel(self.log_file_level)
+            self.log_file_handler.setFormatter(self.log_file_format)
+            logging.root.addHandler(self.log_file_handler)
+
+        if self.log_print:
+            self.log_print_handler = logging.StreamHandler(self.log_print_to)
+            self.log_print_handler.setLevel(self.log_print_level)
+            self.log_print_handler.setFormatter(self.log_format)
+            logging.root.addHandler(self.log_print_handler)
+
+        self.release_lock()
+
+    def __enter__(self):
+        if self._inside_context_manager:
+            self.log.debug('Warning: context manager was entered again. This usually means that self.__call__() was called before the with statement. You probably do not want to do that.')
+            return
+
+        self.acquire_lock()
+        self._inside_context_manager = True
+        self.release_lock()
+
+        colorama.init()
+        self.parse_args()
+        self.read_config()
+        self.setup_logging()
+
+        if self.config.general.save_config:
+            self.save_config()
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.acquire_lock()
+        self._inside_context_manager = False
+        self.release_lock()
+
+        if exc_type is not None and not isinstance(SystemExit(), exc_type):
+            print(exc_type)
+            logging.exception(exc_val)
+            exit(255)
+
+
+cli = MILC()
+
+if __name__ == '__main__':
+
+    @cli.argument('-c', '--comma', help='comma in output', default=True, action='store_boolean')
+    @cli.entrypoint('My useful CLI tool with subcommands.')
+    def main(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{bg_green}{fg_red}Hello%s World!', comma)
+
+    @cli.argument('-n', '--name', help='Name to greet', default='World')
+    @cli.subcommand('Description of hello subcommand here.')
+    def hello(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{fg_blue}Hello%s %s!', comma, cli.config.hello.name)
+
+    def goodbye(cli):
+        comma = ',' if cli.config.general.comma else ''
+        cli.log.info('{bg_red}Goodbye%s %s!', comma, cli.config.goodbye.name)
+
+    @cli.argument('-n', '--name', help='Name to greet', default='World')
+    @cli.subcommand('Think a bit before greeting the user.')
+    def thinking(cli):
+        comma = ',' if cli.config.general.comma else ''
+        spinner = cli.spinner(text='Just a moment...', spinner='earth')
+        spinner.start()
+        sleep(2)
+        spinner.stop()
+
+        with cli.spinner(text='Almost there!', spinner='moon'):
+            sleep(2)
+
+        cli.log.info('{fg_cyan}Hello%s %s!', comma, cli.config.thinking.name)
+
+    @cli.subcommand('Show off our ANSI colors.')
+    def pride(cli):
+        cli.echo('{bg_red}                    ')
+        cli.echo('{bg_lightred_ex}                    ')
+        cli.echo('{bg_lightyellow_ex}                    ')
+        cli.echo('{bg_green}                    ')
+        cli.echo('{bg_blue}                    ')
+        cli.echo('{bg_magenta}                    ')
+
+    # You can register subcommands using decorators as seen above, or using functions like like this:
+    cli.add_subcommand(goodbye, 'This will show up in --help output.')
+    cli.goodbye.add_argument('-n', '--name', help='Name to bid farewell to', default='World')
+
+    cli()  # Automatically picks between main(), hello() and goodbye()
+    print(sorted(ansi_colors.keys()))

lib/python/qmk/cli/compile/json.py

@@ -0,0 +1,44 @@
+"""Create a keymap directory from a configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', help='Configurator JSON export')
+@cli.entrypoint('Compile a QMK Configurator export.')
+def main(cli):
+    """Compile a QMK Configurator export.
+
+    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        exit(1)
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    # Compile the keymap
+    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)

lib/python/qmk/cli/doctor.py

@@ -0,0 +1,47 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import shutil
+import platform
+import os
+
+from milc import cli
+
+
+@cli.entrypoint('Basic QMK environment checks')
+def main(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Run the binaries to make sure they work
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+
+    cli.log.info('QMK Doctor is Checking your environment')
+
+    ok = True
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            ok = False
+
+    OS = platform.system()
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS")
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}linux")
+        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+        if os.system(test) == 0:
+            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows")
+
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')

lib/python/qmk/cli/hello.py

@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Hello World.')
+def main(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.general.name)

lib/python/qmk/cli/json/keymap.py

@@ -0,0 +1,54 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+import sys
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', help='File to write to')
+@cli.argument('filename', help='Configurator JSON file')
+@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
+def main(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)

lib/python/qmk/errors.py

@@ -0,0 +1,6 @@
+class NoSuchKeyboardError(Exception):
+    """Raised when we can't find a keyboard/keymap directory.
+    """
+
+    def __init__(self, message):
+        self.message = message

lib/python/qmk/keymap.py

@@ -0,0 +1,100 @@
+"""Functions that help you work with QMK keymaps.
+"""
+import json
+import logging
+import os
+from traceback import format_exc
+
+import qmk.path
+from qmk.errors import NoSuchKeyboardError
+
+# The `keymap.c` template to use when a keyboard doesn't have its own
+DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+
+/* THIS FILE WAS GENERATED!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+__KEYMAP_GOES_HERE__
+};
+"""
+
+
+def template(keyboard):
+    """Returns the `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
+    text will be used instead of `DEFAULT_KEYMAP_C`.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
+    """
+    template_name = 'keyboards/%s/templates/keymap.c' % keyboard
+
+    if os.path.exists(template_name):
+        with open(template_name, 'r') as fd:
+            return fd.read()
+
+    return DEFAULT_KEYMAP_C
+
+
+def generate(keyboard, layout, layers):
+    """Returns a keymap.c for the specified keyboard, layout, and layers.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    keymap_c = template(keyboard, keymap)
+
+    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        keymap
+            The name of the keymap
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_c = generate(keyboard, layout, layers)
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_dir = os.path.join(keymap_path, keymap)
+    keymap_file = os.path.join(keymap_dir, 'keymap.c')
+
+    if not os.path.exists(keymap_dir):
+        os.makedirs(keymap_dir)
+
+    with open(keymap_file, 'w') as keymap_fd:
+        keymap_fd.write(keymap_c)
+
+    return keymap_file

lib/python/qmk/path.py

@@ -0,0 +1,32 @@
+"""Functions that help us work with files and folders.
+"""
+import os
+
+
+def keymap(keyboard):
+    """Locate the correct directory for storing a keymap.
+
+    Args:
+        keyboard
+            The name of the keyboard. Example: clueboard/66/rev3
+    """
+    for directory in ['.', '..', '../..', '../../..', '../../../..', '../../../../..']:
+        basepath = os.path.normpath(os.path.join('keyboards', keyboard, directory, 'keymaps'))
+
+        if os.path.exists(basepath):
+            return basepath
+
+    logging.error('Could not find keymaps directory!')
+    raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard)
+
+
+def normpath(path):
+    """Returns the fully resolved absolute path to a file.
+
+    This function will return the absolute path to a file as seen from the
+    directory the script was called from.
+    """
+    if path and path[0] == '/':
+        return os.path.normpath(path)
+
+    return os.path.normpath(os.path.join(os.environ['ORIG_CWD'], path))

requirements.txt

@@ -0,0 +1,5 @@
+# Python requirements
+# milc FIXME(skullydazed): Included in the repo for now.
+argcomplete
+colorama
+#halo

setup.cfg

@@ -0,0 +1,330 @@
+# Python settings for QMK
+
+[yapf]
+# Align closing bracket with visual indentation.
+align_closing_bracket_with_visual_indent=True
+
+# Allow dictionary keys to exist on multiple lines. For example:
+#
+#   x = {
+#       ('this is the first element of a tuple',
+#        'this is the second element of a tuple'):
+#            value,
+#   }
+allow_multiline_dictionary_keys=False
+
+# Allow lambdas to be formatted on more than one line.
+allow_multiline_lambdas=False
+
+# Allow splitting before a default / named assignment in an argument list.
+allow_split_before_default_or_named_assigns=True
+
+# Allow splits before the dictionary value.
+allow_split_before_dict_value=True
+
+#   Let spacing indicate operator precedence. For example:
+#
+#     a = 1 * 2 + 3 / 4
+#     b = 1 / 2 - 3 * 4
+#     c = (1 + 2) * (3 - 4)
+#     d = (1 - 2) / (3 + 4)
+#     e = 1 * 2 - 3
+#     f = 1 + 2 + 3 + 4
+#
+# will be formatted as follows to indicate precedence:
+#
+#     a = 1*2 + 3/4
+#     b = 1/2 - 3*4
+#     c = (1+2) * (3-4)
+#     d = (1-2) / (3+4)
+#     e = 1*2 - 3
+#     f = 1 + 2 + 3 + 4
+#
+arithmetic_precedence_indication=True
+
+# Number of blank lines surrounding top-level function and class
+# definitions.
+blank_lines_around_top_level_definition=2
+
+# Insert a blank line before a class-level docstring.
+blank_line_before_class_docstring=False
+
+# Insert a blank line before a module docstring.
+blank_line_before_module_docstring=False
+
+# Insert a blank line before a 'def' or 'class' immediately nested
+# within another 'def' or 'class'. For example:
+#
+#   class Foo:
+#                      # <------ this blank line
+#     def method():
+#       ...
+blank_line_before_nested_class_or_def=False
+
+# Do not split consecutive brackets. Only relevant when
+# dedent_closing_brackets is set. For example:
+#
+#    call_func_that_takes_a_dict(
+#        {
+#            'key1': 'value1',
+#            'key2': 'value2',
+#        }
+#    )
+#
+# would reformat to:
+#
+#    call_func_that_takes_a_dict({
+#        'key1': 'value1',
+#        'key2': 'value2',
+#    })
+coalesce_brackets=True
+
+# The column limit.
+column_limit=256
+
+# The style for continuation alignment. Possible values are:
+#
+# - SPACE: Use spaces for continuation alignment. This is default behavior.
+# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns
+#   (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation
+#   alignment.
+# - VALIGN-RIGHT: Vertically align continuation lines with indent
+#   characters. Slightly right (one more indent character) if cannot
+#   vertically align continuation lines with indent characters.
+#
+# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is
+# enabled.
+continuation_align_style=SPACE
+
+# Indent width used for line continuations.
+continuation_indent_width=4
+
+# Put closing brackets on a separate line, dedented, if the bracketed
+# expression can't fit in a single line. Applies to all kinds of brackets,
+# including function definitions and calls. For example:
+#
+#   config = {
+#       'key1': 'value1',
+#       'key2': 'value2',
+#   }        # <--- this bracket is dedented and on a separate line
+#
+#   time_series = self.remote_client.query_entity_counters(
+#       entity='dev3246.region1',
+#       key='dns.query_latency_tcp',
+#       transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
+#       start_ts=now()-timedelta(days=3),
+#       end_ts=now(),
+#   )        # <--- this bracket is dedented and on a separate line
+dedent_closing_brackets=True
+
+# Disable the heuristic which places each list element on a separate line
+# if the list is comma-terminated.
+disable_ending_comma_heuristic=False
+
+# Place each dictionary entry onto its own line.
+each_dict_entry_on_separate_line=True
+
+# The regex for an i18n comment. The presence of this comment stops
+# reformatting of that line, because the comments are required to be
+# next to the string they translate.
+i18n_comment=
+
+# The i18n function call names. The presence of this function stops
+# reformattting on that line, because the string it has cannot be moved
+# away from the i18n comment.
+i18n_function_call=
+
+# Indent blank lines.
+indent_blank_lines=False
+
+# Indent the dictionary value if it cannot fit on the same line as the
+# dictionary key. For example:
+#
+#   config = {
+#       'key1':
+#           'value1',
+#       'key2': value1 +
+#               value2,
+#   }
+indent_dictionary_value=True
+
+# The number of columns to use for indentation.
+indent_width=4
+
+# Join short lines into one line. E.g., single line 'if' statements.
+join_multiple_lines=False
+
+# Do not include spaces around selected binary operators. For example:
+#
+#   1 + 2 * 3 - 4 / 5
+#
+# will be formatted as follows when configured with "*,/":
+#
+#   1 + 2*3 - 4/5
+no_spaces_around_selected_binary_operators=
+
+# Use spaces around default or named assigns.
+spaces_around_default_or_named_assign=False
+
+# Use spaces around the power operator.
+spaces_around_power_operator=False
+
+# The number of spaces required before a trailing comment.
+# This can be a single value (representing the number of spaces
+# before each trailing comment) or list of values (representing
+# alignment column values; trailing comments within a block will
+# be aligned to the first column value that is greater than the maximum
+# line length within the block). For example:
+#
+# With spaces_before_comment=5:
+#
+#   1 + 1 # Adding values
+#
+# will be formatted as:
+#
+#   1 + 1     # Adding values <-- 5 spaces between the end of the statement and comment
+#
+# With spaces_before_comment=15, 20:
+#
+#   1 + 1 # Adding values
+#   two + two # More adding
+#
+#   longer_statement # This is a longer statement
+#   short # This is a shorter statement
+#
+#   a_very_long_statement_that_extends_beyond_the_final_column # Comment
+#   short # This is a shorter statement
+#
+# will be formatted as:
+#
+#   1 + 1          # Adding values <-- end of line comments in block aligned to col 15
+#   two + two      # More adding
+#
+#   longer_statement    # This is a longer statement <-- end of line comments in block aligned to col 20
+#   short               # This is a shorter statement
+#
+#   a_very_long_statement_that_extends_beyond_the_final_column  # Comment <-- the end of line comments are aligned based on the line length
+#   short                                                       # This is a shorter statement
+#
+spaces_before_comment=2
+
+# Insert a space between the ending comma and closing bracket of a list,
+# etc.
+space_between_ending_comma_and_closing_bracket=False
+
+# Split before arguments
+split_all_comma_separated_values=False
+
+# Split before arguments if the argument list is terminated by a
+# comma.
+split_arguments_when_comma_terminated=True
+
+# Set to True to prefer splitting before '+', '-', '*', '/', '//', or '@'
+# rather than after.
+split_before_arithmetic_operator=False
+
+# Set to True to prefer splitting before '&', '|' or '^' rather than
+# after.
+split_before_bitwise_operator=True
+
+# Split before the closing bracket if a list or dict literal doesn't fit on
+# a single line.
+split_before_closing_bracket=True
+
+# Split before a dictionary or set generator (comp_for). For example, note
+# the split before the 'for':
+#
+#   foo = {
+#       variable: 'Hello world, have a nice day!'
+#       for variable in bar if variable != 42
+#   }
+split_before_dict_set_generator=True
+
+# Split before the '.' if we need to split a longer expression:
+#
+#   foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d))
+#
+# would reformat to something like:
+#
+#   foo = ('This is a really long string: {}, {}, {}, {}'
+#          .format(a, b, c, d))
+split_before_dot=False
+
+# Split after the opening paren which surrounds an expression if it doesn't
+# fit on a single line.
+split_before_expression_after_opening_paren=False
+
+# If an argument / parameter list is going to be split, then split before
+# the first argument.
+split_before_first_argument=False
+
+# Set to True to prefer splitting before 'and' or 'or' rather than
+# after.
+split_before_logical_operator=False
+
+# Split named assignments onto individual lines.
+split_before_named_assigns=True
+
+# Set to True to split list comprehensions and generators that have
+# non-trivial expressions and multiple clauses before each of these
+# clauses. For example:
+#
+#   result = [
+#       a_long_var + 100 for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+#
+# would reformat to something like:
+#
+#   result = [
+#       a_long_var + 100
+#       for a_long_var in xrange(1000)
+#       if a_long_var % 10]
+split_complex_comprehension=True
+
+# The penalty for splitting right after the opening bracket.
+split_penalty_after_opening_bracket=300
+
+# The penalty for splitting the line after a unary operator.
+split_penalty_after_unary_operator=10000
+
+# The penalty of splitting the line around the '+', '-', '*', '/', '//',
+# ``%``, and '@' operators.
+split_penalty_arithmetic_operator=300
+
+# The penalty for splitting right before an if expression.
+split_penalty_before_if_expr=0
+
+# The penalty of splitting the line around the '&', '|', and '^'
+# operators.
+split_penalty_bitwise_operator=300
+
+# The penalty for splitting a list comprehension or generator
+# expression.
+split_penalty_comprehension=80
+
+# The penalty for characters over the column limit.
+split_penalty_excess_character=7000
+
+# The penalty incurred by adding a line split to the unwrapped line. The
+# more line splits added the higher the penalty.
+split_penalty_for_added_line_split=30
+
+# The penalty of splitting a list of "import as" names. For example:
+#
+#   from a_very_long_or_indented_module_name_yada_yad import (long_argument_1,
+#                                                             long_argument_2,
+#                                                             long_argument_3)
+#
+# would reformat to something like:
+#
+#   from a_very_long_or_indented_module_name_yada_yad import (
+#       long_argument_1, long_argument_2, long_argument_3)
+split_penalty_import_names=0
+
+# The penalty of splitting the line around the 'and' and 'or'
+# operators.
+split_penalty_logical_operator=300
+
+# Use the Tab character for indentation.
+use_tabs=False
+

util/freebsd_install.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+util_dir=$(dirname "$0")
 pkg update
 pkg install -y \
 	git \
@@ -17,3 +18,4 @@ pkg install -y \
 	arm-none-eabi-newlib \
 	diffutils \
 	python3
+pip3 install -r ${util_dir}/../requirements.txt

util/linux_install.sh

@@ -8,6 +8,8 @@ SLACKWARE_WARNING="You will need the following packages from slackbuilds.org:\n\
 
 SOLUS_INFO="Your tools are now installed. To start using them, open new terminal or source these scripts:\n\t/usr/share/defaults/etc/profile.d/50-arm-toolchain-path.sh\n\t/usr/share/defaults/etc/profile.d/50-avr-toolchain-path.sh"
 
+util_dir=$(dirname "$0")
+
 if grep ID /etc/os-release | grep -qE "fedora"; then
 	sudo dnf install \
 		arm-none-eabi-binutils-cs \
@@ -183,3 +185,6 @@ else
 	echo
 	echo "https://docs.qmk.fm/#/contributing"
 fi
+
+# Global install tasks
+pip3 install -r ${util_dir}/../requirements.txt

util/macos_install.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+util_dir=$(dirname "$0")
+
 if ! brew --version 2>&1 > /dev/null; then
 	echo "Error! Homebrew not installed or broken!"
 	echo -n "Would you like to install homebrew now? [y/n] "
@@ -24,3 +26,4 @@ brew tap PX4/homebrew-px4
 brew update
 brew install avr-gcc@8 gcc-arm-none-eabi dfu-programmer avrdude dfu-util python3
 brew link --force avr-gcc@8
+pip3 install -r ${util_dir}/../requirements.txt

util/msys2_install.sh

@@ -5,6 +5,7 @@ download_dir=~/qmk_utils
 avrtools=avr8-gnu-toolchain
 armtools=gcc-arm-none-eabi
 installflip=false
+util_dir=$(dirname "$0")
 
 echo "Installing dependencies needed for the installation (quazip)"
 pacman --needed -S base-devel mingw-w64-x86_64-toolchain msys/git msys/p7zip msys/python3 msys/unzip
@@ -92,6 +93,8 @@ else
 fi
 popd
 
+pip3 install -r ${util_dir}/../requirements.txt
+
 cp -f "$dir/activate_msys2.sh" "$download_dir/"
 
 if grep "^source ~/qmk_utils/activate_msys2.sh$" ~/.bashrc

util/wsl_install.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+util_dir=$(dirname "$0")
+dir=$(cd -P -- "$util_dir" && pwd -P)
 pushd "$dir";
 
 if [[ $dir != /mnt/* ]];
@@ -28,6 +29,8 @@ download_dir=wsl_downloaded
 
 source "$dir/win_shared_install.sh"
 
+pip3 install -r ${util_dir}/../requirements.txt
+
 pushd "$download_dir"
 while true; do
     echo