From 7a25dcacffcadf541da5107a35856b66e770bcaf Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 8 May 2021 20:56:07 -0700 Subject: [PATCH] New command: qmk console (#12828) * stash poc * stash * tidy up implementation * Tidy up slightly for review * Tidy up slightly for review * Bodge environment to make tests pass * Refactor away from asyncio due to windows issues * Filter devices * align vid/pid printing * Add hidapi to the installers * start preparing for multiple hid_listeners * udev rules for hid_listen * refactor to move closer to end state * very basic implementation of the threaded model * refactor how vid/pid/index are supplied and parsed * windows improvements * read the report directly when usage page isn't available * add per-device colors, the choice to show names or numbers, and refactor * add timestamps * Add support for showing bootloaders * tweak the color for bootloaders * Align bootloader disconnect with connect color * add support for showing all bootloaders * fix the pyusb check * tweaks * fix exception * hide a stack trace behind -v * add --no-bootloaders option * add documentation for qmk console * Apply suggestions from code review Co-authored-by: Ryan * pyformat * clean up and flesh out KNOWN_BOOTLOADERS Co-authored-by: zvecr Co-authored-by: Ryan --- .github/workflows/cli.yml | 2 +- bin/qmk | 2 + docs/cli_commands.md | 48 ++++++ lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/console.py | 302 +++++++++++++++++++++++++++++++++ requirements-dev.txt | 2 + util/install/arch.sh | 10 +- util/install/debian.sh | 7 +- util/install/fedora.sh | 7 +- util/install/gentoo.sh | 7 +- util/install/msys2.sh | 9 +- util/udev/50-qmk.rules | 3 + 12 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 lib/python/qmk/cli/console.py diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 28c6bb3679..df727518e5 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -23,6 +23,6 @@ jobs: with: submodules: recursive - name: Install dependencies - run: pip3 install -r requirements.txt + run: pip3 install -r requirements-dev.txt - name: Run tests run: bin/qmk pytest diff --git a/bin/qmk b/bin/qmk index a2af2951c9..4b5fd5bbce 100755 --- a/bin/qmk +++ b/bin/qmk @@ -33,6 +33,8 @@ def _check_modules(requirements): # Not every module is importable by its own name. if module['name'] == "pep8-naming": module['import'] = "pep8ext_naming" + elif module['name'] == 'pyusb': + module['import'] = 'usb.core' if not find_spec(module['import']): print('Could not find module %s!' % module['name']) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 05e9306070..581342093a 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -107,6 +107,54 @@ This command lets you configure the behavior of QMK. For the full `qmk config` d qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN] ``` +## `qmk console` + +This command lets you connect to keyboard consoles to get debugging messages. It only works if your keyboard firmware has been compiled with `CONSOLE_ENABLED=yes`. + +**Usage**: + +``` +qmk console [-d :[:]] [-l] [-n] [-t] [-w ] +``` + +**Examples**: + +Connect to all available keyboards and show their console messages: + +``` +qmk console +``` + +List all devices: + +``` +qmk console -l +``` + +Show only messages from clueboard/66/rev3 keyboards: + +``` +qmk console -d C1ED:2370 +``` + +Show only messages from the second clueboard/66/rev3: + +``` +qmk console -d C1ED:2370:2 +``` + +Show timestamps and VID:PID instead of names: + +``` +qmk console -n -t +``` + +Disable bootloader messages: + +``` +qmk console --no-bootloaders +``` + ## `qmk doctor` This command examines your environment and alerts you to potential build or flash problems. It can fix many of them if you want it to. diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index f7df908119..cfb6e6ea59 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -12,6 +12,7 @@ from . import chibios from . import clean from . import compile from . import config +from . import console from . import docs from . import doctor from . import fileformat diff --git a/lib/python/qmk/cli/console.py b/lib/python/qmk/cli/console.py new file mode 100644 index 0000000000..45ff0c8bee --- /dev/null +++ b/lib/python/qmk/cli/console.py @@ -0,0 +1,302 @@ +"""Acquire debugging information from usb hid devices + +cli implementation of https://www.pjrc.com/teensy/hid_listen.html +""" +from pathlib import Path +from threading import Thread +from time import sleep, strftime + +import hid +import usb.core + +from milc import cli + +LOG_COLOR = { + 'next': 0, + 'colors': [ + '{fg_blue}', + '{fg_cyan}', + '{fg_green}', + '{fg_magenta}', + '{fg_red}', + '{fg_yellow}', + ], +} + +KNOWN_BOOTLOADERS = { + # VID , PID + ('03EB', '2FEF'): 'atmel-dfu: ATmega16U2', + ('03EB', '2FF0'): 'atmel-dfu: ATmega32U2', + ('03EB', '2FF3'): 'atmel-dfu: ATmega16U4', + ('03EB', '2FF4'): 'atmel-dfu: ATmega32U4', + ('03EB', '2FF9'): 'atmel-dfu: AT90USB64', + ('03EB', '2FFA'): 'atmel-dfu: AT90USB162', + ('03EB', '2FFB'): 'atmel-dfu: AT90USB128', + ('03EB', '6124'): 'Microchip SAM-BA', + ('0483', 'DF11'): 'stm32-dfu: STM32 BOOTLOADER', + ('16C0', '05DC'): 'USBasp: USBaspLoader', + ('16C0', '05DF'): 'bootloadHID: HIDBoot', + ('16C0', '0478'): 'halfkay: Teensy Halfkay', + ('1B4F', '9203'): 'caterina: Pro Micro 3.3V', + ('1B4F', '9205'): 'caterina: Pro Micro 5V', + ('1B4F', '9207'): 'caterina: LilyPadUSB', + ('1C11', 'B007'): 'kiibohd: Kiibohd DFU Bootloader', + ('1EAF', '0003'): 'stm32duino: Maple 003', + ('1FFB', '0101'): 'caterina: Polou A-Star 32U4 Bootloader', + ('2341', '0036'): 'caterina: Arduino Leonardo', + ('2341', '0037'): 'caterina: Arduino Micro', + ('239A', '000C'): 'caterina: Adafruit Feather 32U4', + ('239A', '000D'): 'caterina: Adafruit ItsyBitsy 32U4 3v', + ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v', + ('239A', '000E'): 'caterina: Adafruit ItsyBitsy 32U4 5v', + ('2A03', '0036'): 'caterina: Arduino Leonardo', + ('2A03', '0037'): 'caterina: Arduino Micro', + ('314B', '0106'): 'apm32-dfu: APM32 DFU ISP Mode' +} + + +class MonitorDevice(object): + def __init__(self, hid_device, numeric): + self.hid_device = hid_device + self.numeric = numeric + self.device = hid.Device(path=hid_device['path']) + self.current_line = '' + + cli.log.info('Console Connected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', hid_device) + + def read(self, size, encoding='ascii', timeout=1): + """Read size bytes from the device. + """ + return self.device.read(size, timeout).decode(encoding) + + def read_line(self): + """Read from the device's console until we get a \n. + """ + while '\n' not in self.current_line: + self.current_line += self.read(32).replace('\x00', '') + + lines = self.current_line.split('\n', 1) + self.current_line = lines[1] + + return lines[0] + + def run_forever(self): + while True: + try: + message = {**self.hid_device, 'text': self.read_line()} + identifier = (int2hex(message['vendor_id']), int2hex(message['product_id'])) if self.numeric else (message['manufacturer_string'], message['product_string']) + message['identifier'] = ':'.join(identifier) + message['ts'] = '{style_dim}{fg_green}%s{style_reset_all} ' % (strftime(cli.config.general.datetime_fmt),) if cli.args.timestamp else '' + + cli.echo('%(ts)s%(color)s%(identifier)s:%(index)d{style_reset_all}: %(text)s' % message) + + except hid.HIDException: + break + + +class FindDevices(object): + def __init__(self, vid, pid, index, numeric): + self.vid = vid + self.pid = pid + self.index = index + self.numeric = numeric + + def run_forever(self): + """Process messages from our queue in a loop. + """ + live_devices = {} + live_bootloaders = {} + + while True: + try: + for device in list(live_devices): + if not live_devices[device]['thread'].is_alive(): + cli.log.info('Console Disconnected: %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s%(vendor_id)04X:%(product_id)04X:%(index)d{style_reset_all})', live_devices[device]) + del live_devices[device] + + for device in self.find_devices(): + if device['path'] not in live_devices: + device['color'] = LOG_COLOR['colors'][LOG_COLOR['next']] + LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors']) + live_devices[device['path']] = device + + try: + monitor = MonitorDevice(device, self.numeric) + device['thread'] = Thread(target=monitor.run_forever, daemon=True) + + device['thread'].start() + except Exception as e: + device['e'] = e + device['e_name'] = e.__class__.__name__ + cli.log.error("Could not connect to %(color)s%(manufacturer_string)s %(product_string)s{style_reset_all} (%(color)s:%(vendor_id)04X:%(product_id)04X:%(index)d): %(e_name)s: %(e)s", device) + if cli.config.general.verbose: + cli.log.exception(e) + del live_devices[device['path']] + + if cli.args.bootloaders: + for device in self.find_bootloaders(): + if device.address in live_bootloaders: + live_bootloaders[device.address]._qmk_found = True + else: + name = KNOWN_BOOTLOADERS[(int2hex(device.idVendor), int2hex(device.idProduct))] + cli.log.info('Bootloader Connected: {style_bright}{fg_magenta}%s', name) + device._qmk_found = True + live_bootloaders[device.address] = device + + for device in list(live_bootloaders): + if live_bootloaders[device]._qmk_found: + live_bootloaders[device]._qmk_found = False + else: + name = KNOWN_BOOTLOADERS[(int2hex(live_bootloaders[device].idVendor), int2hex(live_bootloaders[device].idProduct))] + cli.log.info('Bootloader Disconnected: {style_bright}{fg_magenta}%s', name) + del live_bootloaders[device] + + sleep(.1) + + except KeyboardInterrupt: + break + + def is_bootloader(self, hid_device): + """Returns true if the device in question matches a known bootloader vid/pid. + """ + return (int2hex(hid_device.idVendor), int2hex(hid_device.idProduct)) in KNOWN_BOOTLOADERS + + def is_console_hid(self, hid_device): + """Returns true when the usage page indicates it's a teensy-style console. + """ + return hid_device['usage_page'] == 0xFF31 and hid_device['usage'] == 0x0074 + + def is_filtered_device(self, hid_device): + """Returns True if the device should be included in the list of available consoles. + """ + return int2hex(hid_device['vendor_id']) == self.vid and int2hex(hid_device['product_id']) == self.pid + + def find_devices_by_report(self, hid_devices): + """Returns a list of available teensy-style consoles by doing a brute-force search. + + Some versions of linux don't report usage and usage_page. In that case we fallback to reading the report (possibly inaccurately) ourselves. + """ + devices = [] + + for device in hid_devices: + path = device['path'].decode('utf-8') + + if path.startswith('/dev/hidraw'): + number = path[11:] + report = Path(f'/sys/class/hidraw/hidraw{number}/device/report_descriptor') + + if report.exists(): + rp = report.read_bytes() + + if rp[1] == 0x31 and rp[3] == 0x09: + devices.append(device) + + return devices + + def find_bootloaders(self): + """Returns a list of available bootloader devices. + """ + return list(filter(self.is_bootloader, usb.core.find(find_all=True))) + + def find_devices(self): + """Returns a list of available teensy-style consoles. + """ + hid_devices = hid.enumerate() + devices = list(filter(self.is_console_hid, hid_devices)) + + if not devices: + devices = self.find_devices_by_report(hid_devices) + + if self.vid and self.pid: + devices = list(filter(self.is_filtered_device, devices)) + + # Add index numbers + device_index = {} + for device in devices: + id = ':'.join((int2hex(device['vendor_id']), int2hex(device['product_id']))) + + if id not in device_index: + device_index[id] = 0 + + device_index[id] += 1 + device['index'] = device_index[id] + + return devices + + +def int2hex(number): + """Returns a string representation of the number as hex. + """ + return "%04X" % number + + +def list_devices(device_finder): + """Show the user a nicely formatted list of devices. + """ + devices = device_finder.find_devices() + + if devices: + cli.log.info('Available devices:') + for dev in devices: + color = LOG_COLOR['colors'][LOG_COLOR['next']] + LOG_COLOR['next'] = (LOG_COLOR['next'] + 1) % len(LOG_COLOR['colors']) + cli.log.info("\t%s%s:%s:%d{style_reset_all}\t%s %s", color, int2hex(dev['vendor_id']), int2hex(dev['product_id']), dev['index'], dev['manufacturer_string'], dev['product_string']) + + if cli.args.bootloaders: + bootloaders = device_finder.find_bootloaders() + + if bootloaders: + cli.log.info('Available Bootloaders:') + + for dev in bootloaders: + cli.log.info("\t%s:%s\t%s", int2hex(dev.idVendor), int2hex(dev.idProduct), KNOWN_BOOTLOADERS[(int2hex(dev.idVendor), int2hex(dev.idProduct))]) + + +@cli.argument('--bootloaders', arg_only=True, default=True, action='store_boolean', help='displaying bootloaders.') +@cli.argument('-d', '--device', help='Device to select - uses format :[:].') +@cli.argument('-l', '--list', arg_only=True, action='store_true', help='List available hid_listen devices.') +@cli.argument('-n', '--numeric', arg_only=True, action='store_true', help='Show VID/PID instead of names.') +@cli.argument('-t', '--timestamp', arg_only=True, action='store_true', help='Print the timestamp for received messages as well.') +@cli.argument('-w', '--wait', type=int, default=1, help="How many seconds to wait between checks (Default: 1)") +@cli.subcommand('Acquire debugging information from usb hid devices.', hidden=False if cli.config.user.developer else True) +def console(cli): + """Acquire debugging information from usb hid devices + """ + vid = None + pid = None + index = 1 + + if cli.config.console.device: + device = cli.config.console.device.split(':') + + if len(device) == 2: + vid, pid = device + + elif len(device) == 3: + vid, pid, index = device + + if not index.isdigit(): + cli.log.error('Device index must be a number! Got "%s" instead.', index) + exit(1) + + index = int(index) + + if index < 1: + cli.log.error('Device index must be greater than 0! Got %s', index) + exit(1) + + else: + cli.log.error('Invalid format for device, expected ":[:]" but got "%s".', cli.config.console.device) + cli.print_help() + exit(1) + + vid = vid.upper() + pid = pid.upper() + + device_finder = FindDevices(vid, pid, index, cli.args.numeric) + + if cli.args.list: + return list_devices(device_finder) + + print('Looking for devices...', flush=True) + device_finder.run_forever() diff --git a/requirements-dev.txt b/requirements-dev.txt index 1db3b6d733..12d570e70c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,5 +4,7 @@ # Python development requirements nose2 flake8 +hid pep8-naming +pyusb yapf diff --git a/util/install/arch.sh b/util/install/arch.sh index 7442e2f136..eac4ad64ef 100755 --- a/util/install/arch.sh +++ b/util/install/arch.sh @@ -4,13 +4,13 @@ _qmk_install() { echo "Installing dependencies" sudo pacman --needed --noconfirm -S \ - base-devel clang diffutils gcc git unzip wget zip \ - python-pip \ - avr-binutils \ - arm-none-eabi-binutils arm-none-eabi-gcc arm-none-eabi-newlib \ - avrdude dfu-programmer dfu-util + base-devel clang diffutils gcc git unzip wget zip python-pip \ + avr-binutils arm-none-eabi-binutils arm-none-eabi-gcc \ + arm-none-eabi-newlib avrdude dfu-programmer dfu-util sudo pacman --needed --noconfirm -U https://archive.archlinux.org/packages/a/avr-gcc/avr-gcc-8.3.0-1-x86_64.pkg.tar.xz sudo pacman --needed --noconfirm -S avr-libc # Must be installed after the above, or it will bring in the latest avr-gcc instead + sudo pacman --needed --noconfirm -S hidapi # This will fail if the community repo isn't enabled + python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt } diff --git a/util/install/debian.sh b/util/install/debian.sh index 0ae9764a33..ef87c41b51 100755 --- a/util/install/debian.sh +++ b/util/install/debian.sh @@ -13,10 +13,9 @@ _qmk_install() { sudo apt-get -yq install \ build-essential clang-format diffutils gcc git unzip wget zip \ - python3-pip \ - binutils-avr gcc-avr avr-libc \ - binutils-arm-none-eabi gcc-arm-none-eabi libnewlib-arm-none-eabi \ - avrdude dfu-programmer dfu-util teensy-loader-cli libusb-dev + python3-pip binutils-avr gcc-avr avr-libc binutils-arm-none-eabi \ + gcc-arm-none-eabi libnewlib-arm-none-eabi avrdude dfu-programmer \ + dfu-util teensy-loader-cli libhidapi-hidraw0 python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt } diff --git a/util/install/fedora.sh b/util/install/fedora.sh index 44b71b98bf..10fc7c8ad8 100755 --- a/util/install/fedora.sh +++ b/util/install/fedora.sh @@ -5,11 +5,10 @@ _qmk_install() { # TODO: Check whether devel/headers packages are really needed sudo dnf -y install \ - clang diffutils git gcc glibc-headers kernel-devel kernel-headers make unzip wget zip \ - python3 \ - avr-binutils avr-gcc avr-libc \ + clang diffutils git gcc glibc-headers kernel-devel kernel-headers \ + make unzip wget zip python3 avr-binutils avr-gcc avr-libc \ arm-none-eabi-binutils-cs arm-none-eabi-gcc-cs arm-none-eabi-newlib \ - avrdude dfu-programmer dfu-util libusb-devel + avrdude dfu-programmer dfu-util hidapi python3 -m pip install --user -r $QMK_FIRMWARE_DIR/requirements.txt } diff --git a/util/install/gentoo.sh b/util/install/gentoo.sh index 97eb5df07f..604d07bf84 100755 --- a/util/install/gentoo.sh +++ b/util/install/gentoo.sh @@ -22,9 +22,10 @@ _qmk_install() { echo "sys-devel/gcc multilib" | sudo tee --append /etc/portage/package.use/qmkfirmware >/dev/null sudo emerge -auN sys-devel/gcc sudo emerge -au --noreplace \ - app-arch/unzip app-arch/zip net-misc/wget sys-devel/clang sys-devel/crossdev \ - \>=dev-lang/python-3.7 \ - dev-embedded/avrdude dev-embedded/dfu-programmer app-mobilephone/dfu-util + app-arch/unzip app-arch/zip net-misc/wget sys-devel/clang \ + sys-devel/crossdev \>=dev-lang/python-3.7 dev-embedded/avrdude \ + dev-embedded/dfu-programmer app-mobilephone/dfu-util sys-apps/hwloc \ + dev-libs/hidapi sudo crossdev -s4 --stable --g \<9 --portage --verbose --target avr sudo crossdev -s4 --stable --g \<9 --portage --verbose --target arm-none-eabi diff --git a/util/install/msys2.sh b/util/install/msys2.sh index c8598a60fa..9b8343aed0 100755 --- a/util/install/msys2.sh +++ b/util/install/msys2.sh @@ -9,11 +9,10 @@ _qmk_install() { pacman --needed --noconfirm --disable-download-timeout -S pactoys-git pacboy sync --needed --noconfirm --disable-download-timeout \ - base-devel: toolchain:x clang:x git: unzip: \ - python3-pip:x \ - avr-binutils:x avr-gcc:x avr-libc:x \ - arm-none-eabi-binutils:x arm-none-eabi-gcc:x arm-none-eabi-newlib:x \ - avrdude:x bootloadhid:x dfu-programmer:x dfu-util:x teensy-loader-cli:x + base-devel: toolchain:x clang:x git: unzip: python3-pip:x \ + avr-binutils:x avr-gcc:x avr-libc:x arm-none-eabi-binutils:x \ + arm-none-eabi-gcc:x arm-none-eabi-newlib:x avrdude:x bootloadhid:x \ + dfu-programmer:x dfu-util:x teensy-loader-cli:x hidapi:x _qmk_install_drivers diff --git a/util/udev/50-qmk.rules b/util/udev/50-qmk.rules index acaa7dcc58..679fe4ced3 100644 --- a/util/udev/50-qmk.rules +++ b/util/udev/50-qmk.rules @@ -60,3 +60,6 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="239a", ATTRS{idProduct}=="000e", TAG+="uacc SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0036", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1" ### Micro SUBSYSTEMS=="usb", ATTRS{idVendor}=="2a03", ATTRS{idProduct}=="0037", TAG+="uaccess", ENV{ID_MM_DEVICE_IGNORE}="1" + +# hid_listen +KERNEL=="hidraw*", MODE="0660", GROUP="plugdev", TAG+="uaccess", TAG+="udev-acl"