Add support for producing UF2-format binaries. (#12435)

* First stab at enabling builds of UF2-format binaries.

* Add description on producing a UF2 file.

.gitignore

@@ -5,6 +5,7 @@
 *.eep
 *.elf
 *.hex
+*.uf2
 *.qmk
 !util/bootloader.hex
 !quantum/tools/eeprom_reset.hex

Makefile

@@ -93,8 +93,8 @@ clean:
 
 .PHONY: distclean
 distclean: clean
-	echo -n 'Deleting *.bin and *.hex ... '
+	echo -n 'Deleting *.bin, *.hex, and *.uf2 ... '
-	rm -f *.bin *.hex
+	rm -f *.bin *.hex *.uf2
 	echo 'done.'
 
 #Compatibility with the old make variables, anything you specify directly on the command line

message.mk

@@ -47,10 +47,12 @@ MSG_SIZE_AFTER = Size after:
 MSG_COFF = Converting to AVR COFF:
 MSG_EXTENDED_COFF = Converting to AVR Extended COFF:
 MSG_FLASH = Creating load file for flashing:
+MSG_UF2 = Creating UF2 file for deployment:
 MSG_EEPROM = Creating load file for EEPROM:
 MSG_BIN = Creating binary load file for flashing:
 MSG_EXTENDED_LISTING = Creating Extended Listing:
 MSG_SYMBOL_TABLE = Creating Symbol Table:
+MSG_EXECUTING = Executing:
 MSG_LINKING = Linking:
 MSG_COMPILING = Compiling:
 MSG_COMPILING_CXX = Compiling:

quantum/mcu_selection.mk

@@ -139,6 +139,9 @@ ifneq ($(findstring STM32F042, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F0
 endif
 
 ifneq ($(findstring STM32F072, $(MCU)),)
@@ -172,6 +175,9 @@ ifneq ($(findstring STM32F072, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F0
 endif
 
 ifneq ($(findstring STM32F103, $(MCU)),)
@@ -205,6 +211,9 @@ ifneq ($(findstring STM32F103, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F1
 endif
 
 ifneq ($(findstring STM32F303, $(MCU)),)
@@ -238,6 +247,9 @@ ifneq ($(findstring STM32F303, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F3
 endif
 
 ifneq ($(findstring STM32F401, $(MCU)),)
@@ -271,6 +283,9 @@ ifneq ($(findstring STM32F401, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F4
 endif
 
 ifneq ($(findstring STM32F411, $(MCU)),)
@@ -304,6 +319,9 @@ ifneq ($(findstring STM32F411, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32F4
 endif
 
 ifneq ($(findstring STM32G431, $(MCU)),)
@@ -337,6 +355,9 @@ ifneq ($(findstring STM32G431, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32G4
 endif
 
 ifneq ($(findstring STM32G474, $(MCU)),)
@@ -370,6 +391,9 @@ ifneq ($(findstring STM32G474, $(MCU)),)
   # Options to pass to dfu-util when flashing
   DFU_ARGS ?= -d 0483:DF11 -a 0 -s 0x08000000:leave
   DFU_SUFFIX_ARGS ?= -v 0483 -p DF11
+
+  # UF2 settings
+  UF2_FAMILY ?= STM32G4
 endif
 
 ifneq (,$(filter $(MCU),at90usb162 atmega16u2 atmega32u2 atmega16u4 atmega32u4 at90usb646 at90usb647 at90usb1286 at90usb1287))

tmk_core/rules.mk

@@ -223,6 +223,12 @@ ifneq ($(filter Darwin FreeBSD,$(shell uname -s)),)
   MD5SUM = md5
 endif
 
+# UF2 format settings
+# To produce a UF2 file in your build, add to your keyboard's rules.mk:
+#      FIRMWARE_FORMAT = uf2
+UF2CONV = $(TOP_DIR)/util/uf2conv.py
+UF2_FAMILY ?= 0x0
+
 # Compiler flags to generate dependency files.
 #GENDEPFLAGS = -MMD -MP -MF .dep/$(@F).d
 GENDEPFLAGS = -MMD -MP -MF $(patsubst %.o,%.td,$@)
@@ -255,6 +261,7 @@ DFU_SUFFIX_ARGS ?=
 
 elf: $(BUILD_DIR)/$(TARGET).elf
 hex: $(BUILD_DIR)/$(TARGET).hex
+uf2: $(BUILD_DIR)/$(TARGET).uf2
 cpfirmware: $(FIRMWARE_FORMAT)
 	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to qmk_firmware folder" | $(AWK_CMD)
 	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
@@ -283,32 +290,44 @@ gccversion :
 
 # Create final output files (.hex, .eep) from ELF output file.
 %.hex: %.elf
-	@$(SILENT) || printf "$(MSG_FLASH) $@" | $(AWK_CMD)
 	$(eval CMD=$(HEX) $< $@)
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_FLASH) $@" | $(AWK_CMD)
+	@$(BUILD_CMD)
+
+%.uf2: %.hex
+	$(eval CMD=$(UF2CONV) $(BUILD_DIR)/$(TARGET).hex -o $(BUILD_DIR)/$(TARGET).uf2 -c -f $(UF2_FAMILY) >/dev/null 2>&1)
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_UF2) $@" | $(AWK_CMD)
 	@$(BUILD_CMD)
 
 %.eep: %.elf
-	@$(SILENT) || printf "$(MSG_EEPROM) $@" | $(AWK_CMD)
 	$(eval CMD=$(EEP) $< $@ || exit 0)
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_EEPROM) $@" | $(AWK_CMD)
 	@$(BUILD_CMD)
 
 # Create extended listing file from ELF output file.
 %.lss: %.elf
-	@$(SILENT) || printf "$(MSG_EXTENDED_LISTING) $@" | $(AWK_CMD)
 	$(eval CMD=$(OBJDUMP) -h -S -z $< > $@)
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_EXTENDED_LISTING) $@" | $(AWK_CMD)
 	@$(BUILD_CMD)
 
 # Create a symbol table from ELF output file.
 %.sym: %.elf
-	@$(SILENT) || printf "$(MSG_SYMBOL_TABLE) $@" | $(AWK_CMD)
 	$(eval CMD=$(NM) -n $< > $@ )
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_SYMBOL_TABLE) $@" | $(AWK_CMD)
 	@$(BUILD_CMD)
 
 %.bin: %.elf
-	@$(SILENT) || printf "$(MSG_BIN) $@" | $(AWK_CMD)
 	$(eval CMD=$(BIN) $< $@ || exit 0)
+	#@$(SILENT) || printf "$(MSG_EXECUTING) '$(CMD)':\n"
+	@$(SILENT) || printf "$(MSG_BIN) $@" | $(AWK_CMD)
 	@$(BUILD_CMD)
 	if [ ! -z "$(DFU_SUFFIX_ARGS)" ]; then \
+		#$(SILENT) || printf "$(MSG_EXECUTING) '$(DFU_SUFFIX) $(DFU_SUFFIX_ARGS) -a $(BUILD_DIR)/$(TARGET).bin 1>/dev/null':\n" ;\
 		$(DFU_SUFFIX) $(DFU_SUFFIX_ARGS) -a $(BUILD_DIR)/$(TARGET).bin 1>/dev/null ;\
 	fi
 	$(COPY) $(BUILD_DIR)/$(TARGET).bin $(TARGET).bin;
@@ -476,7 +495,7 @@ $(eval $(foreach OUTPUT,$(OUTPUTS),$(shell mkdir -p $(OUTPUT) 2>/dev/null)))
 
 # Listing of phony targets.
 .PHONY : all dump_vars finish sizebefore sizeafter qmkversion \
-gccversion build elf hex eep lss sym coff extcoff \
+gccversion build elf hex uf2 eep lss sym coff extcoff \
 clean clean_list debug gdb-config show_path \
 program teensy dfu dfu-ee dfu-start \
 flash dfu-split-left dfu-split-right \

util/uf2conv.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+import sys
+import struct
+import subprocess
+import re
+import os
+import os.path
+import argparse
+
+
+UF2_MAGIC_START0 = 0x0A324655 # "UF2\n"
+UF2_MAGIC_START1 = 0x9E5D5157 # Randomly selected
+UF2_MAGIC_END    = 0x0AB16F30 # Ditto
+
+families = {
+    'SAMD21': 0x68ed2b88,
+    'SAML21': 0x1851780a,
+    'SAMD51': 0x55114460,
+    'NRF52': 0x1b57745f,
+    'STM32F0': 0x647824b6,
+    'STM32F1': 0x5ee21072,
+    'STM32F2': 0x5d1a0a2e,
+    'STM32F3': 0x6b846188,
+    'STM32F4': 0x57755a57,
+    'STM32F7': 0x53b80f00,
+    'STM32G0': 0x300f5633,
+    'STM32G4': 0x4c71240a,
+    'STM32H7': 0x6db66082,
+    'STM32L0': 0x202e3a91,
+    'STM32L1': 0x1e1f432d,
+    'STM32L4': 0x00ff6919,
+    'STM32L5': 0x04240bdf,
+    'STM32WB': 0x70d16653,
+    'STM32WL': 0x21460ff0,
+    'ATMEGA32': 0x16573617,
+    'MIMXRT10XX': 0x4FB2D5BD,
+    'LPC55': 0x2abc77ec,
+    'GD32F350': 0x31D228C6,
+    'ESP32S2': 0xbfdd4eee,
+    'RP2040': 0xe48bff56
+}
+
+INFO_FILE = "/INFO_UF2.TXT"
+
+appstartaddr = 0x2000
+familyid = 0x0
+
+
+def is_uf2(buf):
+    w = struct.unpack("<II", buf[0:8])
+    return w[0] == UF2_MAGIC_START0 and w[1] == UF2_MAGIC_START1
+
+def is_hex(buf):
+    try:
+        w = buf[0:30].decode("utf-8")
+    except UnicodeDecodeError:
+        return False
+    if w[0] == ':' and re.match(b"^[:0-9a-fA-F\r\n]+$", buf):
+        return True
+    return False
+
+def convert_from_uf2(buf):
+    global appstartaddr
+    numblocks = len(buf) // 512
+    curraddr = None
+    outp = []
+    for blockno in range(numblocks):
+        ptr = blockno * 512
+        block = buf[ptr:ptr + 512]
+        hd = struct.unpack(b"<IIIIIIII", block[0:32])
+        if hd[0] != UF2_MAGIC_START0 or hd[1] != UF2_MAGIC_START1:
+            print("Skipping block at " + ptr + "; bad magic")
+            continue
+        if hd[2] & 1:
+            # NO-flash flag set; skip block
+            continue
+        datalen = hd[4]
+        if datalen > 476:
+            assert False, "Invalid UF2 data size at " + ptr
+        newaddr = hd[3]
+        if curraddr == None:
+            appstartaddr = newaddr
+            curraddr = newaddr
+        padding = newaddr - curraddr
+        if padding < 0:
+            assert False, "Block out of order at " + ptr
+        if padding > 10*1024*1024:
+            assert False, "More than 10M of padding needed at " + ptr
+        if padding % 4 != 0:
+            assert False, "Non-word padding size at " + ptr
+        while padding > 0:
+            padding -= 4
+            outp += b"\x00\x00\x00\x00"
+        outp.append(block[32 : 32 + datalen])
+        curraddr = newaddr + datalen
+    return b"".join(outp)
+
+def convert_to_carray(file_content):
+    outp = "const unsigned long bindata_len = %d;\n" % len(file_content)
+    outp += "const unsigned char bindata[] __attribute__((aligned(16))) = {"
+    for i in range(len(file_content)):
+        if i % 16 == 0:
+            outp += "\n"
+        outp += "0x%02x, " % file_content[i]
+    outp += "\n};\n"
+    return bytes(outp, "utf-8")
+
+def convert_to_uf2(file_content):
+    global familyid
+    datapadding = b""
+    while len(datapadding) < 512 - 256 - 32 - 4:
+        datapadding += b"\x00\x00\x00\x00"
+    numblocks = (len(file_content) + 255) // 256
+    outp = []
+    for blockno in range(numblocks):
+        ptr = 256 * blockno
+        chunk = file_content[ptr:ptr + 256]
+        flags = 0x0
+        if familyid:
+            flags |= 0x2000
+        hd = struct.pack(b"<IIIIIIII",
+            UF2_MAGIC_START0, UF2_MAGIC_START1,
+            flags, ptr + appstartaddr, 256, blockno, numblocks, familyid)
+        while len(chunk) < 256:
+            chunk += b"\x00"
+        block = hd + chunk + datapadding + struct.pack(b"<I", UF2_MAGIC_END)
+        assert len(block) == 512
+        outp.append(block)
+    return b"".join(outp)
+
+class Block:
+    def __init__(self, addr):
+        self.addr = addr
+        self.bytes = bytearray(256)
+
+    def encode(self, blockno, numblocks):
+        global familyid
+        flags = 0x0
+        if familyid:
+            flags |= 0x2000
+        hd = struct.pack("<IIIIIIII",
+            UF2_MAGIC_START0, UF2_MAGIC_START1,
+            flags, self.addr, 256, blockno, numblocks, familyid)
+        hd += self.bytes[0:256]
+        while len(hd) < 512 - 4:
+            hd += b"\x00"
+        hd += struct.pack("<I", UF2_MAGIC_END)
+        return hd
+
+def convert_from_hex_to_uf2(buf):
+    global appstartaddr
+    appstartaddr = None
+    upper = 0
+    currblock = None
+    blocks = []
+    for line in buf.split('\n'):
+        if line[0] != ":":
+            continue
+        i = 1
+        rec = []
+        while i < len(line) - 1:
+            rec.append(int(line[i:i+2], 16))
+            i += 2
+        tp = rec[3]
+        if tp == 4:
+            upper = ((rec[4] << 8) | rec[5]) << 16
+        elif tp == 2:
+            upper = ((rec[4] << 8) | rec[5]) << 4
+            assert (upper & 0xffff) == 0
+        elif tp == 1:
+            break
+        elif tp == 0:
+            addr = upper | (rec[1] << 8) | rec[2]
+            if appstartaddr == None:
+                appstartaddr = addr
+            i = 4
+            while i < len(rec) - 1:
+                if not currblock or currblock.addr & ~0xff != addr & ~0xff:
+                    currblock = Block(addr & ~0xff)
+                    blocks.append(currblock)
+                currblock.bytes[addr & 0xff] = rec[i]
+                addr += 1
+                i += 1
+    numblocks = len(blocks)
+    resfile = b""
+    for i in range(0, numblocks):
+        resfile += blocks[i].encode(i, numblocks)
+    return resfile
+
+def to_str(b):
+    return b.decode("utf-8")
+
+def get_drives():
+    drives = []
+    if sys.platform == "win32":
+        r = subprocess.check_output(["wmic", "PATH", "Win32_LogicalDisk",
+                                     "get", "DeviceID,", "VolumeName,",
+                                     "FileSystem,", "DriveType"])
+        for line in to_str(r).split('\n'):
+            words = re.split('\s+', line)
+            if len(words) >= 3 and words[1] == "2" and words[2] == "FAT":
+                drives.append(words[0])
+    else:
+        rootpath = "/media"
+        if sys.platform == "darwin":
+            rootpath = "/Volumes"
+        elif sys.platform == "linux":
+            tmp = rootpath + "/" + os.environ["USER"]
+            if os.path.isdir(tmp):
+                rootpath = tmp
+        for d in os.listdir(rootpath):
+            drives.append(os.path.join(rootpath, d))
+
+
+    def has_info(d):
+        try:
+            return os.path.isfile(d + INFO_FILE)
+        except:
+            return False
+
+    return list(filter(has_info, drives))
+
+
+def board_id(path):
+    with open(path + INFO_FILE, mode='r') as file:
+        file_content = file.read()
+    return re.search("Board-ID: ([^\r\n]*)", file_content).group(1)
+
+
+def list_drives():
+    for d in get_drives():
+        print(d, board_id(d))
+
+
+def write_file(name, buf):
+    with open(name, "wb") as f:
+        f.write(buf)
+    print("Wrote %d bytes to %s" % (len(buf), name))
+
+
+def main():
+    global appstartaddr, familyid
+    def error(msg):
+        print(msg)
+        sys.exit(1)
+    parser = argparse.ArgumentParser(description='Convert to UF2 or flash directly.')
+    parser.add_argument('input', metavar='INPUT', type=str, nargs='?',
+                        help='input file (HEX, BIN or UF2)')
+    parser.add_argument('-b' , '--base', dest='base', type=str,
+                        default="0x2000",
+                        help='set base address of application for BIN format (default: 0x2000)')
+    parser.add_argument('-o' , '--output', metavar="FILE", dest='output', type=str,
+                        help='write output to named file; defaults to "flash.uf2" or "flash.bin" where sensible')
+    parser.add_argument('-d' , '--device', dest="device_path",
+                        help='select a device path to flash')
+    parser.add_argument('-l' , '--list', action='store_true',
+                        help='list connected devices')
+    parser.add_argument('-c' , '--convert', action='store_true',
+                        help='do not flash, just convert')
+    parser.add_argument('-D' , '--deploy', action='store_true',
+                        help='just flash, do not convert')
+    parser.add_argument('-f' , '--family', dest='family', type=str,
+                        default="0x0",
+                        help='specify familyID - number or name (default: 0x0)')
+    parser.add_argument('-C' , '--carray', action='store_true',
+                        help='convert binary file to a C array, not UF2')
+    args = parser.parse_args()
+    appstartaddr = int(args.base, 0)
+
+    if args.family.upper() in families:
+        familyid = families[args.family.upper()]
+    else:
+        try:
+            familyid = int(args.family, 0)
+        except ValueError:
+            error("Family ID needs to be a number or one of: " + ", ".join(families.keys()))
+
+    if args.list:
+        list_drives()
+    else:
+        if not args.input:
+            error("Need input file")
+        with open(args.input, mode='rb') as f:
+            inpbuf = f.read()
+        from_uf2 = is_uf2(inpbuf)
+        ext = "uf2"
+        if args.deploy:
+            outbuf = inpbuf
+        elif from_uf2:
+            outbuf = convert_from_uf2(inpbuf)
+            ext = "bin"
+        elif is_hex(inpbuf):
+            outbuf = convert_from_hex_to_uf2(inpbuf.decode("utf-8"))
+        elif args.carray:
+            outbuf = convert_to_carray(inpbuf)
+            ext = "h"
+        else:
+            outbuf = convert_to_uf2(inpbuf)
+        print("Converting to %s, output size: %d, start address: 0x%x" %
+              (ext, len(outbuf), appstartaddr))
+        if args.convert or ext != "uf2":
+            drives = []
+            if args.output == None:
+                args.output = "flash." + ext
+        else:
+            drives = get_drives()
+
+        if args.output:
+            write_file(args.output, outbuf)
+        else:
+            if len(drives) == 0:
+                error("No drive to deploy.")
+        for d in drives:
+            print("Flashing %s (%s)" % (d, board_id(d)))
+            write_file(d + "/NEW.UF2", outbuf)
+
+
+if __name__ == "__main__":
+    main()