Add esptool backend the ESP32-C3, switch to XIP flash layout

default.nix

@@ -36,7 +36,11 @@ in pkgs.python3Packages.buildPythonPackage rec {
     flit
   ];
 
-  propagatedBuildInputs = with python3Packages; [
+  propagatedBuildInputs = [
+    # For interacting with ESP32 boards, makes both the CLI and the `esptool`
+    # Python package available:
+    pkgs.esptool
+  ] ++ (with python3Packages; [
     appdirs
     argcomplete
     colorama
@@ -49,7 +53,7 @@ in pkgs.python3Packages.buildPythonPackage rec {
     six
     toml
     tqdm
-  ];
+  ]);
 
   # Ensure that Tockloader can, at runtime, find the `nrfutil` binary:
   postPatch = lib.optionalString withUnfreePkgs ''

tockloader/board_interface.py

@@ -293,10 +293,17 @@ class BoardInterface:
         "esp32-c3-devkitm-1": {
             "description": "Espressif ESP32-C3-DevKitM-1",
             "arch": "rv32imc",
+            # Size of a sector of the external SPI flash.
+            "page_size": 4096,
             "no_attribute_table": True,
+            # The mask ROM maps the external SPI flash at this address.
+            "flash_address": 0x42000000,
+            "esptool": {
+                "chip": "ESP32-C3",
+            },
             "flash_file": {
-                "flash_address": 0x40380000,
-                "max_size": 0x60000,
+                "flash_address": 0x42000000,
+                "max_size": 0x400000,
             },
         },
         "sma_q3": {

tockloader/esptool.py

@@ -0,0 +1,237 @@
+"""
+Interface for Espressif boards using esptool.
+
+Espressif chips have a serial bootloader in their mask ROM. Tockloader uses
+the Python API of esptool to access the external SPI flash of the chip through
+this bootloader. The chip stays in its ROM bootloader while Tockloader reads
+and writes flash, and is reset into the Tock kernel afterwards.
+"""
+
+import logging
+
+import serial.tools.list_ports
+
+from .board_interface import BoardInterface
+from .exceptions import TockLoaderException
+
+# USB VID:PID pairs that indicate an attached Espressif chip.
+#
+# Espressif's own VID is used by the USB Serial/JTAG controller integrated into
+# newer chips. The other entries are USB-UART bridges used on Espressif's
+# development kits. Many unrelated boards use these bridges as well, so we only
+# select such a port after esptool has confirmed that a supported chip is
+# attached to it.
+ESPRESSIF_USB_VID = 0x303A
+USB_UART_BRIDGES = [
+    (0x10C4, 0xEA60),  # Silicon Labs CP210x (e.g., ESP32-C3-DevKitM-1)
+    (0x1A86, 0x55D3),  # WCH CH343 (e.g., ESP32-C3-DevKitC-02 v1.1)
+    (0x1A86, 0x7523),  # WCH CH340
+]
+
+# Mapping from esptool-reported chip to Tock board name.
+DEFAULT_BOARD_FOR_CHIP = {
+    "ESP32-C3": "esp32-c3-devkitm-1",
+}
+
+
+class EspTool(BoardInterface):
+    def __init__(self, args):
+        # Must call the generic init first.
+        super().__init__(args)
+
+        self.port = getattr(self.args, "port", None)
+
+        # The `ESPLoader` connection to the chip's ROM bootloader. `None` while
+        # the chip is not in its ROM bootloader.
+        self.esp = None
+
+    def _import_esptool(self):
+        """
+        Import esptool lazily, such that it is only required for users of this
+        channel.
+        """
+        try:
+            import esptool.cmds
+            from esptool.logger import log
+        except ImportError:
+            raise TockLoaderException(
+                "Tockloader esptool support requires esptool >= 5.0. "
+                "Install it with `pip install esptool`."
+            )
+
+        # esptool prints to stdout by default. Only show its output when
+        # tockloader is running in debug mode.
+        log.set_verbosity("verbose" if self.args.debug else "silent")
+
+        return esptool.cmds
+
+    def _candidate_ports(self):
+        """
+        Return the serial ports that an Espressif chip may be attached to.
+        """
+        if self.port is not None:
+            return [self.port]
+
+        ports = []
+        for port in serial.tools.list_ports.comports():
+            if (
+                port.vid == ESPRESSIF_USB_VID
+                or (port.vid, port.pid) in USB_UART_BRIDGES
+            ):
+                ports.append(port.device)
+        return ports
+
+    def _connect(self, port):
+        """
+        Reset the chip attached to `port` into its ROM bootloader, and return an
+        `ESPLoader` with the stub flasher running.
+        """
+        cmds = self._import_esptool()
+        logging.debug('Connecting to Espressif chip on "{}".'.format(port))
+        esp = cmds.detect_chip(port=port, connect_attempts=1)
+        logging.debug("Detected {}.".format(esp.CHIP_NAME))
+        esp = cmds.run_stub(esp)
+        cmds.attach_flash(esp)
+        return esp
+
+    def _disconnect(self):
+        """
+        Reset the chip into the Tock kernel and close the serial port.
+        """
+        if self.esp is None:
+            return
+
+        cmds = self._import_esptool()
+        try:
+            cmds.reset_chip(self.esp, "hard-reset")
+        finally:
+            self.esp._port.close()
+            self.esp = None
+
+    def _expected_chip(self):
+        """
+        Return the chip we expect esptool to report for the selected board, or
+        `None` if the board does not specify one.
+        """
+        if self.board and self.board in self.KNOWN_BOARDS:
+            return self.KNOWN_BOARDS[self.board].get("esptool", {}).get("chip")
+        return None
+
+    def attached_board_exists(self):
+        # Probe each candidate port with esptool. This resets the attached chip,
+        # so we keep the connection open for `open_link_to_board()`.
+        try:
+            self._import_esptool()
+        except TockLoaderException:
+            return False
+
+        for port in self._candidate_ports():
+            try:
+                esp = self._connect(port)
+            except Exception as e:
+                logging.debug('No Espressif chip on "{}": {}'.format(port, e))
+                continue
+
+            if esp.CHIP_NAME in DEFAULT_BOARD_FOR_CHIP:
+                self.port = port
+                self.esp = esp
+                return True
+
+            esp._port.close()
+
+        return False
+
+    def open_link_to_board(self):
+        if self.port is None:
+            ports = self._candidate_ports()
+            if len(ports) == 0:
+                raise TockLoaderException(
+                    "No Espressif serial port found. Pass --port to select one."
+                )
+            if len(ports) > 1:
+                raise TockLoaderException(
+                    "Multiple candidate serial ports found ({}). Pass --port to "
+                    "select one.".format(", ".join(ports))
+                )
+            self.port = ports[0]
+
+        self.enter_bootloader_mode()
+
+        # The chip does not tell us which board it is on. If the user did not
+        # specify the board, assume the default board for this chip.
+        if self.board is None:
+            self.board = DEFAULT_BOARD_FOR_CHIP.get(self.esp.CHIP_NAME)
+
+        if self.board and self.board in self.KNOWN_BOARDS:
+            logging.info('Using settings from KNOWN_BOARDS["{}"]'.format(self.board))
+            self._configure_from_known_boards()
+
+        expected_chip = self._expected_chip()
+        if expected_chip is not None and expected_chip != self.esp.CHIP_NAME:
+            self._disconnect()
+            raise TockLoaderException(
+                'Board "{}" requires an {}, but esptool detected an {}.'.format(
+                    self.board, expected_chip, self.esp.CHIP_NAME
+                )
+            )
+
+    def enter_bootloader_mode(self):
+        if self.esp is None:
+            self.esp = self._connect(self.port)
+
+    def exit_bootloader_mode(self):
+        self._disconnect()
+
+    def translate_address(self, address):
+        # esptool addresses the external flash by offset. Tock boards link
+        # against the address where the chip maps the flash.
+        flash_address = self.get_flash_address() or 0
+        if address < flash_address:
+            raise TockLoaderException(
+                "Address {:#x} is not contained in flash".format(address)
+            )
+        return address - flash_address
+
+    def flash_binary(self, address, binary, pad=False):
+        # esptool erases whole flash sectors before writing them. Extend the
+        # write to sector boundaries with the current flash contents, such that
+        # data next to `binary` is preserved.
+        address, binary = self._align_and_stretch_to_page(address, binary)
+
+        cmds = self._import_esptool()
+        cmds.write_flash(
+            self.esp,
+            # esptool accepts `bytes`, but not `bytearray`.
+            [(self.translate_address(address), bytes(binary))],
+            no_progress=not self.args.debug,
+        )
+
+    def read_range(self, address, length):
+        logging.debug(
+            "DEBUG => Read Range, address: {:#010x}, length: {}".format(address, length)
+        )
+        if length == 0:
+            return bytes()
+
+        cmds = self._import_esptool()
+        return cmds.read_flash(
+            self.esp, self.translate_address(address), length, no_progress=True
+        )
+
+    def clear_bytes(self, address):
+        logging.debug("Clearing bytes starting at {:#0x}".format(address))
+
+        binary = bytes([0xFF] * 8)
+        self.flash_binary(address, binary)
+
+    def determine_current_board(self):
+        # We might need to fill in settings if we only know the board name.
+        self._configure_from_known_boards()
+
+        # Check that we learned what we needed to learn.
+        if self.board is None:
+            raise TockLoaderException("Could not determine the current board")
+        if self.arch is None:
+            raise TockLoaderException("Could not determine the current arch")
+        if self.page_size == 0:
+            raise TockLoaderException("Could not determine the current page size")

tockloader/main.py

@@ -923,6 +923,11 @@ def main():
     parent_channel.add_argument(
         "--stlink", action="store_true", help="Use ST-Link tools to flash."
     )
+    parent_channel.add_argument(
+        "--esptool",
+        action="store_true",
+        help="Use esptool to flash an Espressif chip through its ROM bootloader.",
+    )
     parent_channel.add_argument(
         "--probers", action="store_true", help="Use probe-rs to flash."
     )

tockloader/tockloader.py

@@ -35,6 +35,7 @@
 from .openocd import OpenOCD, collect_temp_files
 from .probers import ProbeRs
 from .stlink import STLink
+from .esptool import EspTool
 from .linkserver import LinkServer
 from .flash_file import FlashFile
 from .tickv import TockTicKV
@@ -141,6 +142,7 @@ class TockLoader:
                 "cmd_flags": {"bundle_apps": True},
             },
             "raspberry_pi_pico": {"start_address": 0x10020000},
+            "esp32-c3-devkitm-1": {"start_address": 0x42100000},
             "tickv": {
                 "tickv": {
                     "region_size": 4096,
@@ -197,6 +199,7 @@ def open(self):
                             getattr(self.args, "nrfutil", False),
                             getattr(self.args, "openocd", False),
                             getattr(self.args, "stlink", False),
+                            getattr(self.args, "esptool", False),
                             getattr(self.args, "flash_file") != None,
                             getattr(self.args, "serial", False),
                         ],
@@ -206,7 +209,7 @@ def open(self):
             > 1
         ):
             raise TockLoaderException(
-                "Can only use one of --jlink, --openocd, --stlink, --nrfutil, --flash-file or --serial options"
+                "Can only use one of --jlink, --openocd, --stlink, --esptool, --nrfutil, --flash-file or --serial options"
             )
 
         # Get an object that allows talking to the board.
@@ -222,6 +225,9 @@ def open(self):
         elif hasattr(self.args, "stlink") and self.args.stlink:
             # User passed `--stlink`. Force the STLink channel.
             self.channel = STLink(self.args)
+        elif hasattr(self.args, "esptool") and self.args.esptool:
+            # User passed `--esptool`. Force the esptool channel.
+            self.channel = EspTool(self.args)
         elif hasattr(self.args, "probers") and self.args.probers:
             # User passed `--probers`. Force the probe-rs channel.
             self.channel = ProbeRs(self.args)
@@ -293,6 +299,15 @@ def open(self):
                     logging.info("Using LinkServer channel to comm. with the board.")
                     break
 
+                # Next try esptool. This must come before the serial bootloader,
+                # which would otherwise claim the serial port of an Espressif
+                # chip.
+                esptool_channel = EspTool(self.args)
+                if esptool_channel.attached_board_exists():
+                    self.channel = esptool_channel
+                    logging.info("Using esptool channel to communicate with the board.")
+                    break
+
                 # Try using the serial bootloader. Traditionally, we have
                 # defaulted to this, and if there is a reasonable serial port we
                 # still will, but we no longer unconditionally default to this.