diff --git a/ports/esp32/.gitignore b/ports/esp32/.gitignore index a78ecd0191c9f..9c708bb85c630 100644 --- a/ports/esp32/.gitignore +++ b/ports/esp32/.gitignore @@ -1,2 +1 @@ -dependencies.lock managed_components/ diff --git a/ports/esp32/CMakeLists.txt b/ports/esp32/CMakeLists.txt index 1db374b40a431..9d6be8010ff64 100644 --- a/ports/esp32/CMakeLists.txt +++ b/ports/esp32/CMakeLists.txt @@ -61,5 +61,21 @@ set(SDKCONFIG_DEFAULTS ${CMAKE_BINARY_DIR}/sdkconfig.combined) # Include main IDF cmake file. include($ENV{IDF_PATH}/tools/cmake/project.cmake) +# Generate individual dependencies.lock files based on chip target +idf_build_set_property(DEPENDENCIES_LOCK lockfiles/dependencies.lock.${IDF_TARGET}) + # Define the project. project(micropython) + +# Check for lockfile changes and either warn or error depending on build type +message("Checking lockfile contents...") +execute_process(COMMAND git diff --exit-code lockfiles/ WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE RES) +if (RES) + # Maintainer builds (CI or autobuild runs) should fail if this has happened + if($ENV{MICROPY_MAINTAINER_BUILD}) + message(FATAL_ERROR "Failing build as lockfiles are dirty (see above). Check that ESP-IDF versions match.") + else() + message(WARNING "Component lockfile contents have changed (see above). This may be due to building with a different ESP-IDF version. Please mention this output if reporting an issue with MicroPython.") + endif() +endif() diff --git a/ports/esp32/README.md b/ports/esp32/README.md index dd4584772cf84..2c09ebccd5039 100644 --- a/ports/esp32/README.md +++ b/ports/esp32/README.md @@ -30,8 +30,9 @@ framework, aka SDK). The ESP-IDF includes the libraries and RTOS needed to manage the ESP32 microcontroller, as well as a way to manage the required build environment and toolchains needed to build the firmware. -The ESP-IDF changes quickly and MicroPython only supports certain versions. -Currently MicroPython supports v5.2, v5.2.2, v5.3, v5.4, v5.4.1 and v5.4.2. +The ESP-IDF changes quickly and MicroPython only supports certain versions. The +current recommended version of ESP-IDF for MicroPython is v5.4.2. MicroPython +also supports v5.2, v5.2.2, v5.3, v5.4 and v5.4.1. To install the ESP-IDF the full instructions can be found at the [Espressif Getting Started guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html#installation-step-by-step). diff --git a/ports/esp32/lockfiles/README.md b/ports/esp32/lockfiles/README.md new file mode 100644 index 0000000000000..a957b428f926f --- /dev/null +++ b/ports/esp32/lockfiles/README.md @@ -0,0 +1,12 @@ +# ESP-IDF Component Lockfiles + +This directory contains the exact versions of ESP-IDF components that have been +used to build MicroPython. It is updated by the [component version +solver](https://docs.espressif.com/projects/idf-component-manager/en/latest/guides/version_solver.html). + +Unless you change the `main/idf_component.yml` file for MicroPython ESP32, files +in this directory should only change contents if you build using a different +ESP-IDF version to the last time the file was updated. + +*Please do not commit changes to these files and submit PRs unless the PR needs +to change versions of components or ESP-IDF.* diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32 b/ports/esp32/lockfiles/dependencies.lock.esp32 new file mode 100644 index 0000000000000..5c2b7827544ef --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32 @@ -0,0 +1,35 @@ +dependencies: + espressif/lan867x: + component_hash: 0ff9dae3affeff53811e7c8283e67c6d36dc0c03e3bc5102c0fba629e08bf6c4 + dependencies: + - name: idf + require: private + version: '>=5.3' + source: + registry_url: https://components.espressif.com/ + type: service + targets: + - esp32 + - esp32p4 + version: 1.0.3 + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/lan867x +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32 +version: 2.0.0 diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c2 b/ports/esp32/lockfiles/dependencies.lock.esp32c2 new file mode 100644 index 0000000000000..df6ed2c1d89dc --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32c2 @@ -0,0 +1,21 @@ +dependencies: + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32c2 +version: 2.0.0 diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c3 b/ports/esp32/lockfiles/dependencies.lock.esp32c3 new file mode 100644 index 0000000000000..a4c955d4763e9 --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32c3 @@ -0,0 +1,21 @@ +dependencies: + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32c3 +version: 2.0.0 diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c6 b/ports/esp32/lockfiles/dependencies.lock.esp32c6 new file mode 100644 index 0000000000000..fa81d8ad78dca --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32c6 @@ -0,0 +1,21 @@ +dependencies: + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32c6 +version: 2.0.0 diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32s2 b/ports/esp32/lockfiles/dependencies.lock.esp32s2 new file mode 100644 index 0000000000000..12430428e0979 --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32s2 @@ -0,0 +1,50 @@ +dependencies: + espressif/esp_tinyusb: + component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc + dependencies: + - name: idf + require: private + version: '>=5.0' + - name: espressif/tinyusb + registry_url: https://components.espressif.com + require: public + version: '>=0.14.2' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.7.6~1 + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + espressif/tinyusb: + component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32s2 + - esp32s3 + - esp32p4 + version: 0.18.0~4 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/esp_tinyusb +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32s2 +version: 2.0.0 diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32s3 b/ports/esp32/lockfiles/dependencies.lock.esp32s3 new file mode 100644 index 0000000000000..215346dbd441c --- /dev/null +++ b/ports/esp32/lockfiles/dependencies.lock.esp32s3 @@ -0,0 +1,50 @@ +dependencies: + espressif/esp_tinyusb: + component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc + dependencies: + - name: idf + require: private + version: '>=5.0' + - name: espressif/tinyusb + registry_url: https://components.espressif.com + require: public + version: '>=0.14.2' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.7.6~1 + espressif/mdns: + component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.1.0 + espressif/tinyusb: + component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32s2 + - esp32s3 + - esp32p4 + version: 0.18.0~4 + idf: + source: + type: idf + version: 5.4.2 +direct_dependencies: +- espressif/esp_tinyusb +- espressif/mdns +- idf +manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a +target: esp32s3 +version: 2.0.0 diff --git a/ports/esp32/main/idf_component.yml b/ports/esp32/main/idf_component.yml index f7773f4f4e472..cb14ffde60de8 100644 --- a/ports/esp32/main/idf_component.yml +++ b/ports/esp32/main/idf_component.yml @@ -4,7 +4,7 @@ dependencies: espressif/esp_tinyusb: rules: - if: "target in [esp32s2, esp32s3]" - version: "~1.0.0" + version: "~1.7.6" espressif/lan867x: version: "~1.0.0" rules: diff --git a/py/mpconfig.h b/py/mpconfig.h index 8775b74bb61bc..311d7bba47bfc 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -31,7 +31,7 @@ // are unavailable. #define MICROPY_VERSION_MAJOR 1 #define MICROPY_VERSION_MINOR 26 -#define MICROPY_VERSION_MICRO 0 +#define MICROPY_VERSION_MICRO 1 #define MICROPY_VERSION_PRERELEASE 0 // Combined version as a 32-bit number for convenience to allow version diff --git a/shared/tinyusb/mp_usbd.h b/shared/tinyusb/mp_usbd.h index 5c8f2a6095f30..ed7a248acabe9 100644 --- a/shared/tinyusb/mp_usbd.h +++ b/shared/tinyusb/mp_usbd.h @@ -43,7 +43,18 @@ // Initialise TinyUSB device. static inline void mp_usbd_init_tud(void) { tusb_init(); - tud_cdc_configure_fifo_t cfg = { .rx_persistent = 0, .tx_persistent = 1 }; + tud_cdc_configure_fifo_t cfg = { .rx_persistent = 0, + .tx_persistent = 1, + + // This config flag is unreleased in TinyUSB >v0.18.0 + // but included in Espressif's TinyUSB component since v0.18.0~3 + // + // Versioning issue reported as + // https://github.com/espressif/esp-usb/issues/236 + #if TUSB_VERSION_NUMBER > 1800 || defined(ESP_PLATFORM) + .tx_overwritabe_if_not_connected = 1, + #endif + }; tud_cdc_configure_fifo(&cfg); } diff --git a/shared/tinyusb/mp_usbd_cdc.c b/shared/tinyusb/mp_usbd_cdc.c index b4151f685c41e..9b380acefc060 100644 --- a/shared/tinyusb/mp_usbd_cdc.c +++ b/shared/tinyusb/mp_usbd_cdc.c @@ -98,32 +98,45 @@ mp_uint_t mp_usbd_cdc_tx_strn(const char *str, mp_uint_t len) { if (!tusb_inited()) { return 0; } + mp_uint_t last_write = mp_hal_ticks_ms(); size_t i = 0; while (i < len) { uint32_t n = len - i; - if (n > CFG_TUD_CDC_EP_BUFSIZE) { - n = CFG_TUD_CDC_EP_BUFSIZE; - } + if (tud_cdc_connected()) { - // If CDC port is connected but the buffer is full, wait for up to USC_CDC_TIMEOUT ms. - mp_uint_t t0 = mp_hal_ticks_ms(); - while (n > tud_cdc_write_available() && (mp_uint_t)(mp_hal_ticks_ms() - t0) < MICROPY_HW_USB_CDC_TX_TIMEOUT) { - mp_event_wait_ms(1); - - // Explicitly run the USB stack as the scheduler may be locked (eg we - // are in an interrupt handler), while there is data pending. - mp_usbd_task(); - } // Limit write to available space in tx buffer when connected. + // + // (If not connected then we write everything to the fifo, expecting + // it to overwrite old data so it will have latest data buffered + // when host connects.) n = MIN(n, tud_cdc_write_available()); - if (n == 0) { - break; - } } - // When not connected we always write to usb fifo, ensuring it has latest data. + uint32_t n2 = tud_cdc_write(str + i, n); tud_cdc_write_flush(); i += n2; + + if (i < len) { + if (n2 > 0) { + // reset the timeout each time we successfully write to the FIFO + last_write = mp_hal_ticks_ms(); + } else { + if ((mp_uint_t)(mp_hal_ticks_ms() - last_write) >= MICROPY_HW_USB_CDC_TX_TIMEOUT) { + break; // Timeout + } + + if (tud_cdc_connected()) { + // If we know we're connected then we can wait for host to make + // more space + mp_event_wait_ms(1); + } + } + + // Always explicitly run the USB stack as the scheduler may be + // locked (eg we are in an interrupt handler), while there is data + // or a state change pending. + mp_usbd_task(); + } } return i; } diff --git a/tools/autobuild/autobuild.sh b/tools/autobuild/autobuild.sh index 7073152df7931..c24c4b3a9af4e 100755 --- a/tools/autobuild/autobuild.sh +++ b/tools/autobuild/autobuild.sh @@ -48,6 +48,9 @@ git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}/lib/pico-sdk submodule update --ini ######################################## # Build all firmware +# Fail on some things which are warnings otherwise +export MICROPY_MAINTAINER_BUILD=1 + pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} # build cross compiler diff --git a/tools/ci.sh b/tools/ci.sh index 8f045639b80c9..2fa4f7e52d3d1 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -9,6 +9,9 @@ fi # Ensure known OPEN_MAX (NO_FILES) limit. ulimit -n 1024 +# Fail on some things which are warnings otherwise +export MICROPY_MAINTAINER_BUILD=1 + ######################################################################################## # general helper functions @@ -168,14 +171,17 @@ function ci_cc3200_build { ######################################################################################## # ports/esp32 -# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch) -IDF_VER=v5.4.2 +# GitHub tag of ESP-IDF to use for CI, extracted from the esp32 dependency lockfile +# This should end up as a tag name like vX.Y.Z +# (note: This hacky parsing can be replaced with 'yq' once Ubuntu >=24.04 is in use) +IDF_VER=v$(grep -A10 "idf:" ports/esp32/lockfiles/dependencies.lock.esp32 | grep "version:" | head -n1 | sed -E 's/ +version: //') PYTHON=$(command -v python3 2> /dev/null) PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2) export IDF_CCACHE_ENABLE=1 function ci_esp32_idf_setup { + echo "Using ESP-IDF version $IDF_VER" git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git # doing a treeless clone isn't quite as good as --shallow-submodules, but it # is smaller than full clones and works when the submodule commit isn't a head. diff --git a/tools/mpremote/mpremote/transport_serial.py b/tools/mpremote/mpremote/transport_serial.py index daeff02b594e2..461313adb3e4c 100644 --- a/tools/mpremote/mpremote/transport_serial.py +++ b/tools/mpremote/mpremote/transport_serial.py @@ -36,11 +36,37 @@ # as a command line tool and a library for interacting with devices. import ast, io, os, re, struct, sys, time +import serial +import serial.tools.list_ports from errno import EPERM from .console import VT_ENABLED from .transport import TransportError, TransportExecError, Transport VID_ESPRESSIF = 0x303A # Espressif Incorporated +PID_ESPRESSIF_SERIAL_JTAG = 0x1001 # Serial/JTAG peripheral of ESP32-S3,C3,C6 + + +def has_espressif_dtr_quirk(devicename): + """ESP8266 and ESP32 dev boards use the DTR and RTS lines to trigger reset & + reset into bootloader mode. This can causes spurious reset issues on Windows. + + Apply the quirk to any USB/Serial chip on Windows that isn't using the + Microsoft CDC-ACM driver, or to the integrated Espressif Serial/JTAG device. + + Don't apply it to Espressif boards running TinyUSB, as TinyUSB uses DTR + to determine if the CDC port is open (and there's no spurious reset issue). + """ + portinfo = list(serial.tools.list_ports.grep(devicename)) # type: ignore + if not portinfo: + return False + + def port_attr(name): + return getattr(portinfo[0], name, None) + + return (port_attr("vid"), port_attr("pid")) == ( + VID_ESPRESSIF, + PID_ESPRESSIF_SERIAL_JTAG, + ) or port_attr("manufacturer") != "Microsoft" class SerialTransport(Transport): @@ -52,9 +78,6 @@ def __init__(self, device, baudrate=115200, wait=0, exclusive=True, timeout=None self.device_name = device self.mounted = False - import serial - import serial.tools.list_ports - # Set options, and exclusive if pyserial supports it serial_kwargs = { "baudrate": baudrate, @@ -72,12 +95,7 @@ def __init__(self, device, baudrate=115200, wait=0, exclusive=True, timeout=None elif os.name == "nt": self.serial = serial.Serial(**serial_kwargs) self.serial.port = device - portinfo = list(serial.tools.list_ports.grep(device)) # type: ignore - if portinfo and ( - getattr(portinfo[0], "vid", 0) == VID_ESPRESSIF - or getattr(portinfo[0], "manufacturer", "") != "Microsoft" - ): - # ESP8266/ESP32 boards use RTS/CTS for flashing and boot mode selection. + if has_espressif_dtr_quirk(device): # DTR False: to avoid using the reset button will hang the MCU in bootloader mode # RTS False: to prevent pulses on rts on serial.close() that would POWERON_RESET an ESPxx self.serial.dtr = False # DTR False = gpio0 High = Normal boot