diff --git a/README.md b/README.md index f0d8bd5..543c1ac 100644 --- a/README.md +++ b/README.md @@ -283,3 +283,32 @@ Start GDB and connect to OpenOCD for debugging. ```bash cmake --build . --target debug ``` + +### `upload-files` / `install-all` +Send files the firmware does not hold (views, configurations, anything for +the SD card) to the device over its UART console, with YMODEM: dmell's +`ymodem recv` command receives them, `scripts/ymodem_send.py` (Python 3, +no pyserial needed) types it and sends the files - their paths kept under +the destination directory. + +- `upload-files` - to a device that is running +- `install-all` - `install-firmware`, then the files once the device has started + +```bash +cmake --build . --target upload-files +``` + +| Variable | Default | Description | +|----------|---------|-------------| +| `DMBOOT_UPLOAD_PORT` | `/dev/ttyACM0` | The device's console | +| `DMBOOT_UPLOAD_BAUD` | `921600` | Its baud rate | +| `DMBOOT_UPLOAD_DEST` | `/mnt/dmsdio0_0p1` | Directory on the device | +| `DMBOOT_UPLOAD_SOURCES` | `build/sdcard` | Files and directories (`SRC=NAME` sends SRC as NAME) | +| `DMBOOT_UPLOAD_BLOCK` | `1024` | YMODEM block size (128 or 1024) | +| `DMBOOT_UPLOAD_WAIT` | `60` | `install-all`: seconds the device may take to start | + +The script works on its own too: + +```bash +scripts/ymodem_send.py --dest /mnt/dmsdio0_0p1/views view.dmv fonts/ +``` diff --git a/scripts/targets.cmake b/scripts/targets.cmake index c760955..b6a2e47 100644 --- a/scripts/targets.cmake +++ b/scripts/targets.cmake @@ -137,3 +137,37 @@ add_custom_target(debug WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Starting GDB and connecting to OpenOCD..." ) + +# ====================================================================== +# Uploading Files (YMODEM over the console) +# ====================================================================== +# Files the firmware does not hold - the sdcard modules' views, anything +# for the SD card - sent to the device over its UART console: dmell's +# `ymodem` command receives them (scripts/ymodem_send.py types it). +# upload-files the files, to the device running (its console on DMBOOT_UPLOAD_PORT) +# install-all the firmware, then the files (waiting for the device to start) +set(DMBOOT_UPLOAD_PORT "/dev/ttyACM0" CACHE STRING "Serial port of the device's dmell console the files are sent over (upload-files)") +set(DMBOOT_UPLOAD_BAUD "921600" CACHE STRING "Baud rate of DMBOOT_UPLOAD_PORT") +set(DMBOOT_UPLOAD_DEST "/mnt/dmsdio0_0p1" CACHE STRING "Directory on the device the uploaded files go to (their paths under it)") +set(DMBOOT_UPLOAD_SOURCES "${CMAKE_BINARY_DIR}/sdcard" CACHE STRING "Files and directories uploaded (semicolon-separated; SRC=NAME puts SRC as NAME); default: build/sdcard") +set(DMBOOT_UPLOAD_BLOCK "1024" CACHE STRING "YMODEM block size, 128 or 1024") +set(DMBOOT_UPLOAD_WAIT "60" CACHE STRING "install-all: seconds the device may take to start before it receives the files") + +find_package(Python3 COMPONENTS Interpreter QUIET) +if(Python3_Interpreter_FOUND) + set(DMBOOT_UPLOAD_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/ymodem_send.py + --port ${DMBOOT_UPLOAD_PORT} --baud ${DMBOOT_UPLOAD_BAUD} --dest ${DMBOOT_UPLOAD_DEST} --block ${DMBOOT_UPLOAD_BLOCK}) + add_custom_target(upload-files + COMMAND ${DMBOOT_UPLOAD_COMMAND} ${DMBOOT_UPLOAD_SOURCES} + USES_TERMINAL + COMMENT "Uploading files to ${DMBOOT_UPLOAD_DEST} over ${DMBOOT_UPLOAD_PORT} (YMODEM)..." + ) + add_custom_target(install-all + COMMAND ${DMBOOT_UPLOAD_COMMAND} --wait ${DMBOOT_UPLOAD_WAIT} ${DMBOOT_UPLOAD_SOURCES} + DEPENDS install-firmware + USES_TERMINAL + COMMENT "Uploading files to ${DMBOOT_UPLOAD_DEST} once ${TARGET} has started..." + ) +else() + message(STATUS "Python 3 not found - the upload-files and install-all targets are not available") +endif() diff --git a/scripts/ymodem_send.py b/scripts/ymodem_send.py new file mode 100755 index 0000000..5d31b5a --- /dev/null +++ b/scripts/ymodem_send.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Send files to a dmod-boot device over its UART console (YMODEM batch). + +The device's dmell receives them with its `ymodem` command: this script +types `ymodem recv -d DEST` into the console, waits for the receiver's +'C' and sends every file - its name (with directories) under DEST: + + ymodem_send.py --port /dev/ttyACM0 --dest /mnt/dmsdio0_0p1 build/sdcard + ymodem_send.py --dest /mnt/dmsdio0_0p1/views view.dmv fonts/ + ymodem_send.py --dest /mnt/sd local/dir=remote/dir + +A FILE is sent by its name, a DIR with everything in it (names relative +to it), SRC=NAME as NAME (a directory: under NAME). Only the standard +library (termios): no pyserial needed. +""" + +import argparse +import os +import select +import sys +import termios +import time +import tty + +SOH, STX, EOT, ACK, NAK, CAN, CRC = 0x01, 0x02, 0x04, 0x06, 0x15, 0x18, ord('C') +BAUDS = { + 9600: termios.B9600, 19200: termios.B19200, 38400: termios.B38400, 57600: termios.B57600, + 115200: termios.B115200, 230400: termios.B230400, 460800: getattr(termios, 'B460800', None), + 921600: getattr(termios, 'B921600', None), +} + + +class Failure(Exception): + pass + + +class Link: + """Bytes in and out: a serial port, or stdin / stdout (--stdio, tests)""" + + retries = 0 + + def __init__(self, port, baud): + if port is None: + self.rfd, self.wfd, self.saved = sys.stdin.fileno(), sys.stdout.fileno(), None + return + fd = os.open(port, os.O_RDWR | os.O_NOCTTY) + self.saved = termios.tcgetattr(fd) + tty.setraw(fd) + attrs = termios.tcgetattr(fd) + speed = BAUDS.get(baud) + if speed is None: + raise Failure(f'unsupported baud rate {baud}') + attrs[4] = attrs[5] = speed + attrs[2] = (attrs[2] & ~termios.CRTSCTS) | termios.CLOCAL | termios.CREAD + termios.tcsetattr(fd, termios.TCSANOW, attrs) + termios.tcflush(fd, termios.TCIOFLUSH) + self.rfd = self.wfd = fd + + def close(self): + if self.saved is not None: + termios.tcsetattr(self.rfd, termios.TCSADRAIN, self.saved) + os.close(self.rfd) + + def write(self, data): + view = memoryview(data) + while view: + n = os.write(self.wfd, view) + view = view[n:] + + def read(self, timeout): + """One byte, None after `timeout` seconds""" + ready, _, _ = select.select([self.rfd], [], [], timeout) + if not ready: + return None + b = os.read(self.rfd, 1) + if not b: + raise Failure('the device closed the connection') + return b[0] + + def drain(self, quiet=0.2): + """What the device says until it is quiet""" + out = bytearray() + try: + while (b := self.read(quiet)) is not None: + out.append(b) + except Failure: + pass # (closed: what it said is all) + return bytes(out) + + +def crc16(data): + crc = 0 + for byte in data: + crc ^= byte << 8 + for _ in range(8): + crc = ((crc << 1) ^ 0x1021) if crc & 0x8000 else (crc << 1) + crc &= 0xFFFF + return crc + + +def packet(number, data, size): + data = data.ljust(size, b'\x1a' if number else b'\x00') + c = crc16(data) + return bytes([SOH if size == 128 else STX, number & 0xFF, 0xFF - (number & 0xFF)]) + data + bytes([c >> 8, c & 0xFF]) + + +def wait_for(link, wanted, timeout, what): + """One of `wanted` (bytes) - others skipped; CAN CAN: the device gave up""" + end = time.monotonic() + timeout + cans = 0 + while time.monotonic() < end: + b = link.read(max(0.0, end - time.monotonic())) + if b is None: + break + if b == CAN: + cans += 1 + if cans >= 2: + raise Failure(f'the device cancelled ({what})') + continue + cans = 0 + if b in wanted: + return b + raise Failure(f'no answer from the device ({what})') + + +def send_packet(link, number, data, size, what, retries=10): + p = packet(number, data, size) + for attempt in range(retries): + if attempt > 0: + link.retries += 1 + link.write(p) + try: + if wait_for(link, (ACK, NAK), 3.0, what) == ACK: + return + except Failure as e: + if 'cancelled' in str(e): + raise + raise Failure(f'{what}: not acknowledged') + + +def send_file(link, path, name, block, progress): + size = os.path.getsize(path) + header = name.encode() + b'\x00' + str(size).encode() + b'\x00' + if len(header) > 128: + raise Failure(f'{name}: name too long for YMODEM') + send_packet(link, 0, header, 128, f'{name}: header') + wait_for(link, (CRC,), 10.0, f'{name}: data') + sent, number = 0, 1 + with open(path, 'rb') as f: + while chunk := f.read(block): + send_packet(link, number, chunk, 128 if len(chunk) <= 128 else block, f'{name}: block {number}') + number += 1 + sent += len(chunk) + progress(name, sent, size) + for attempt in range(10): + link.write(bytes([EOT])) + if wait_for(link, (ACK, NAK), 3.0, f'{name}: end') == ACK: + break + wait_for(link, (CRC,), 10.0, f'{name}: next') + return size + + +def receiver_ready(link, command, timeout): + """The command's echo, then the receiver's 'C' - twice in a row (it sends one a second): not a 'C' of a log""" + end = time.monotonic() + timeout + seen, last = bytearray(), None + while time.monotonic() < end: + b = link.read(max(0.0, end - time.monotonic())) + if b is None: + return False + if command not in seen: + seen.append(b) + if len(seen) > 4096: + del seen[:2048] + continue + if b == CRC and last == CRC: + return True + last = b + return False + + +def files_of(sources): + """(local path, name sent) of every source""" + out = [] + for source in sources: + local, _, name = source.partition('=') + if os.path.isdir(local): + for root, dirs, names in os.walk(local): + dirs[:] = sorted(d for d in dirs if not d.startswith('.')) + for n in sorted(names): + if n.startswith('.'): + continue # (.download_complete, ...) + full = os.path.join(root, n) + rel = os.path.relpath(full, local).replace(os.sep, '/') + out.append((full, f'{name.rstrip("/")}/{rel}' if name else rel)) + elif os.path.isfile(local): + out.append((local, name or os.path.basename(local))) + else: + raise Failure(f'{local}: no such file or directory') + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('sources', nargs='+', help='FILE, DIR or SRC=NAME') + ap.add_argument('--port', default='/dev/ttyACM0', help='the device console (default /dev/ttyACM0)') + ap.add_argument('--baud', type=int, default=921600) + ap.add_argument('--dest', default='', help='the directory on the device the names are under') + ap.add_argument("--block", type=int, choices=(128, 1024), default=1024, + help='YMODEM block size (default 1024)') + ap.add_argument('--no-command', action='store_true', help='ymodem recv is already running') + ap.add_argument('--wait', type=float, default=0.0, metavar='SECONDS', + help='the device may still be starting: type the command again until it answers (after a flash)') + ap.add_argument('--stdio', action='store_true', help='stdin / stdout instead of a port (tests)') + ap.add_argument('--quiet', action='store_true') + args = ap.parse_args() + + files = files_of(args.sources) + log = (lambda *a: None) if args.quiet else (lambda *a: print(*a, file=sys.stderr, flush=True)) + + def progress(name, sent, size): + if not args.quiet: + print(f'\r {name}: {sent * 100 // max(size, 1)}%', end='', file=sys.stderr, flush=True) + + link = Link(None if args.stdio else args.port, args.baud) + try: + if args.no_command: + wait_for(link, (CRC,), 15.0, 'the receiver') + else: + command = f'ymodem recv -d {args.dest}' if args.dest else 'ymodem recv' + until = time.monotonic() + args.wait + while True: + link.write(b'\r') + link.drain() + link.write(command.encode() + b'\r') + if receiver_ready(link, command.encode(), 8.0): + break + if time.monotonic() >= until: + raise Failure('no ymodem receiver answered (is the console on this port, ymodem installed?)') + log(' waiting for the device ...') + start, total = time.monotonic(), 0 + for path, name in files: + total += send_file(link, path, name, args.block, progress) + log(f'\r {name}: {os.path.getsize(path)} bytes') + send_packet(link, 0, b'', 128, 'the end of the batch') + seconds = max(time.monotonic() - start, 0.001) + said = link.drain(0.5).decode(errors='replace').strip() + log(f'{len(files)} file(s), {total} bytes in {seconds:.1f} s ({total / seconds / 1024:.1f} KiB/s)' + + (f', {link.retries} block(s) sent again' if link.retries else '')) + if said: + log(f'device: {said.splitlines()[0] if said.startswith("ymodem") else said}') + return 0 + except Failure as e: + try: + link.write(bytes([CAN] * 4)) + except OSError: + pass + print(f'ymodem_send: {e}', file=sys.stderr) + return 1 + finally: + link.close() + + +if __name__ == '__main__': + sys.exit(main())