diff --git a/doc/img/fw-done.svg b/doc/img/fw-done.svg new file mode 100644 index 0000000..8a80c2b --- /dev/null +++ b/doc/img/fw-done.svg @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-init.svg b/doc/img/fw-init.svg new file mode 100644 index 0000000..8c725c1 --- /dev/null +++ b/doc/img/fw-init.svg @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-1.svg b/doc/img/fw-lz4-1.svg new file mode 100644 index 0000000..9fea3fc --- /dev/null +++ b/doc/img/fw-lz4-1.svg @@ -0,0 +1,196 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4) + + + + Update (lz4) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-2.svg b/doc/img/fw-lz4-2.svg new file mode 100644 index 0000000..51a2346 --- /dev/null +++ b/doc/img/fw-lz4-2.svg @@ -0,0 +1,227 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4) + + + + Update (lz4) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-3.svg b/doc/img/fw-lz4-3.svg new file mode 100644 index 0000000..5841567 --- /dev/null +++ b/doc/img/fw-lz4-3.svg @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4) + + + + Update (lz4) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-1.svg b/doc/img/fw-lz4-dict-1.svg new file mode 100644 index 0000000..5e9d491 --- /dev/null +++ b/doc/img/fw-lz4-dict-1.svg @@ -0,0 +1,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict) + + + + Update (lz4-dict) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-2.svg b/doc/img/fw-lz4-dict-2.svg new file mode 100644 index 0000000..77d9d61 --- /dev/null +++ b/doc/img/fw-lz4-dict-2.svg @@ -0,0 +1,308 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict) + + + + Update (lz4-dict) + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-3.svg b/doc/img/fw-lz4-dict-3.svg new file mode 100644 index 0000000..c7c51c8 --- /dev/null +++ b/doc/img/fw-lz4-dict-3.svg @@ -0,0 +1,226 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict) + + + + Update (lz4-dict) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-4.svg b/doc/img/fw-lz4-dict-4.svg new file mode 100644 index 0000000..0e459c1 --- /dev/null +++ b/doc/img/fw-lz4-dict-4.svg @@ -0,0 +1,258 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict) + + + + Update (lz4-dict) + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-5.svg b/doc/img/fw-lz4-dict-5.svg new file mode 100644 index 0000000..2342e04 --- /dev/null +++ b/doc/img/fw-lz4-dict-5.svg @@ -0,0 +1,225 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict) + + + + Update (lz4-dict) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-inplace-1.svg b/doc/img/fw-lz4-dict-inplace-1.svg new file mode 100644 index 0000000..ff4baed --- /dev/null +++ b/doc/img/fw-lz4-dict-inplace-1.svg @@ -0,0 +1,198 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict-inplace) + + + + Update (lz4-dict-inplace) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-inplace-2.svg b/doc/img/fw-lz4-dict-inplace-2.svg new file mode 100644 index 0000000..8321646 --- /dev/null +++ b/doc/img/fw-lz4-dict-inplace-2.svg @@ -0,0 +1,400 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Scratch + + + + Scratch + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict-inplace) + + + + Update (lz4-dict-inplace) + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-lz4-dict-inplace-3.svg b/doc/img/fw-lz4-dict-inplace-3.svg new file mode 100644 index 0000000..28352f3 --- /dev/null +++ b/doc/img/fw-lz4-dict-inplace-3.svg @@ -0,0 +1,227 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Scratch + + + + Scratch + + + + + + + + + + + + + + + + + + + + + + + + Update (lz4-dict-inplace) + + + + Update (lz4-dict-inplace) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-plain-1.svg b/doc/img/fw-plain-1.svg new file mode 100644 index 0000000..88ef0f7 --- /dev/null +++ b/doc/img/fw-plain-1.svg @@ -0,0 +1,194 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + Current firmware + + + + Current firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-plain-2.svg b/doc/img/fw-plain-2.svg new file mode 100644 index 0000000..2ad6cb6 --- /dev/null +++ b/doc/img/fw-plain-2.svg @@ -0,0 +1,226 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/img/fw-plain-3.svg b/doc/img/fw-plain-3.svg new file mode 100644 index 0000000..1b654cb --- /dev/null +++ b/doc/img/fw-plain-3.svg @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BL + + + + BL + + + + + + + + + + + + + + + + + + + + + + + + New firmware + + + + New firmware + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update (plain) + + + + Update (plain) + + + + + + + + \ No newline at end of file diff --git a/doc/img/illustrations.odg b/doc/img/illustrations.odg new file mode 100644 index 0000000..13ffa16 Binary files /dev/null and b/doc/img/illustrations.odg differ diff --git a/doc/uptypes.md b/doc/uptypes.md new file mode 100644 index 0000000..e3e1436 --- /dev/null +++ b/doc/uptypes.md @@ -0,0 +1,134 @@ +# Firmware Update Types + +## Update Process + +Initially, only the bootloader and the original firmware reside in +the device Flash memory. Generally, the bootloader is located at the +very beginning, followed by the current firmware. The bootloader +component is static and does not change, nor is it updatable itself. + + + +It is the application's responsility to place an available update +somewhere in Flash memory, preferrably at the very end to maximize +the available space for updating. Updating overwrites the current +firmware, leaving only the new firmware in Flash memory. + + + +As the update process is destructive, extreme care is taken to make +this step robust. It is implemented in such a way that in case of +interruption, the update can be restarted or resumed. This ensures +that even in the case of power-loss or other transient failure, the +bootloader can continue updating on the next pass, leaving the device +with a functioning firmware. + +The bootloader supports a number of different update types: + +- Plain Updates +- LZ4 Compressed Updates +- LZ4 Delta Updates +- LZ4 In-Place Delta Updates + +### Plain Updates (`plain`) + +Plain updates are the most simple way to perform an update. The full +new firmware image is contained as-is in the update blob. + + + +The bootloader then simply copies the firmware from the update blob +to its intended location. + + + +After the process is complete, the new firmware is in place and the +update can be invalidated. + + + +### LZ4 Compressed Updates (`lz4`) + +LZ4 compressed updates differ from plain updates only by using [LZ4 +compression](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) +to reduce the size of the update. LZ4 was chosen because of the +simplicity and robustness of the decompressor. + + + +The bootloader then decompresses the firmware from the update blob +to its intended location. + + + +After the process is complete, the new firmware is in place and the +update can be invalidated. + + + +### LZ4 Delta Updates (`lz4-dict`) + +LZ4 delta updates make use of _dictionary mode_, whereby data can be +preloaded into the backreferences space. For delta updates, the +previous firmware image is used as the dictionary. This can dramatically +reduce the size of an update, but since decompression requires access +to the dictionary, the update is now specific to the source firmware +it was created for. Further, the entire dictionary must stay available +during the decompression process, which means the original firmware +cannot be decompressed directly to the intended location. + + + +LZ4 delta updates are therefore encoded as plain udpates before the +compression step. The bootloader then decompresses this plain update +blob into free Flash memory between the compressed update and the +current firmware, while using the current firmware as dictionary. + + + +Once the decompression is complete, the plain update is available +in Flash and the compressed update can be invalidated. + + + +The process then continues by using the plain update as above. + + + +After the process is complete, the new firmware is in place and the +update can be invalidated. + + + +As can be seen, while LZ4 delta updates are quite small and can thus +be transmitted very efficiently over-the-air, the two-step decompression +method uses much more Flash memory than any of the other methods. + +### LZ4 In-Place Delta Updates (`lz4-dict-inplace`) + +:construction: *Coming soon...* + +By carefully coding the update and making use of a temporary scratch +buffer, the delta update can be extracted directly to its final location +in Flash memory. This dramatically reduces the amount of free space +necessary on the device. The scratch buffer's size can be variable, +thereby allowing the trading of compression efficiency against Flash +memory requirements. + + + +The new firmware is uncompressed one block at a time into a temporary +scratch buffer, before it is copied to its final location. The update +blob contains itegrity protection data for each block to allow transactional +execution of this update method. + + + +While not the entire previous firmware is available as a dictionary +for every block with this method, it still yields compression efficiency +comparable to regular delta updates in practice. + +After the process is complete, the new firmware is in place and the +update can be invalidated. + + diff --git a/src/arm/stm32lx/bootloader.c b/src/arm/stm32lx/bootloader.c index 48a0f8a..64616f7 100644 --- a/src/arm/stm32lx/bootloader.c +++ b/src/arm/stm32lx/bootloader.c @@ -356,4 +356,5 @@ __attribute__((section(".boot.boottab"))) const boot_boottab boottab = { .version = 0x100, .update = set_update, .panic = fw_panic, + .crc32 = boot_crc32, }; diff --git a/src/arm/stm32lx/bootloader_impl.h b/src/arm/stm32lx/bootloader_impl.h index 419a4f4..0b23e6e 100644 --- a/src/arm/stm32lx/bootloader_impl.h +++ b/src/arm/stm32lx/bootloader_impl.h @@ -17,6 +17,7 @@ #define _bootloader_impl_h_ #include "bootloader.h" +#include "boottab.h" // Convert pointer to Thumb function (set least-significant bit) #define THUMB_FUNC(p) ((void*) (((intptr_t) (p)) | 1)) diff --git a/src/arm/stm32lx/boottab.h b/src/arm/stm32lx/boottab.h new file mode 100644 index 0000000..12bfcc0 --- /dev/null +++ b/src/arm/stm32lx/boottab.h @@ -0,0 +1,31 @@ +/* _____ _ __ _ + * /__ \_ __ __ _ ___| | __ /\ \ \___| |_ + * / /\/ '__/ _` |/ __| |/ // \/ / _ \ __| + * / / | | | (_| | (__| '/ /\ / __/ |_ + * \_\ |_| \__,_|\___|_|\_\_\ \_\\___|\__| + * + * Copyright (c) 2016-2018 Trackio International AG + * All rights reserved. + * + * This file is subject to the terms and conditions + * defined in file 'LICENSE', which is part of this + * source code package. + * + */ + +#ifndef _boottab_h_ +#define _boottab_h_ + +#include "bootloader.h" + +// Bootloader information table on STM32 + +typedef struct { + uint32_t version; // version of boot loader (values below 256 are reserved for legacy bootloaders) + __attribute__((noreturn)) + void (*panic) (uint32_t reason, uint32_t addr); // bootloader panic function + uint32_t (*update) (void* ptr, hash32* hash); // function to set firmware update pointer + uint32_t (*crc32) (void* buf, uint32_t nwords); // calculate CRC32 +} boot_boottab; + +#endif diff --git a/src/common/bootloader.h b/src/common/bootloader.h index 5dd8af2..58e2534 100644 --- a/src/common/bootloader.h +++ b/src/common/bootloader.h @@ -69,17 +69,6 @@ typedef union { } hash32; -// Bootloader information table -// TODO: move to platform-specific section ? -typedef struct { - uint32_t version; // version of boot loader (values below 256 are reserved for legacy bootloaders) - __attribute__((noreturn)) - void (*panic) (uint32_t reason, uint32_t addr); // bootloader panic function - uint32_t (*update) (void* ptr, hash32* hash); // function to set firmware update pointer - // TODO: extensions -} boot_boottab; - - // Firmware header typedef struct { uint32_t crc; // firmware CRC diff --git a/tools/fwtool/fwtool.py b/tools/fwtool/fwtool.py old mode 100644 new mode 100755 index 3f59d45..70302d4 --- a/tools/fwtool/fwtool.py +++ b/tools/fwtool/fwtool.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # _____ _ __ _ # /__ \_ __ __ _ ___| | __ /\ \ \___| |_ # / /\/ '__/ _` |/ __| |/ // \/ / _ \ __| @@ -16,6 +17,7 @@ import argparse import struct import sys +import os from intelhex import IntelHex from binascii import crc32 @@ -173,34 +175,36 @@ def main(self, args:NS) -> int: class MkUpdateCommand(Command): - def __init__(self, subs:SPA) -> None: + def __init__(self, subs:SPA, prefix:str='') -> None: p = super().create_parser(subs, 'mkupdate', 'Create a firmware update file') p.add_argument('FWFILE', type=str, help='the firmware file to process') p.add_argument('UPFILE', type=str, help='the update file to create') + self.prefix = prefix def main(self, args:NS) -> int: - print('Processing %s' % args.FWFILE) + print('%sProcessing %s' % (self.prefix, args.FWFILE)) fw = Firmware(args.FWFILE) up = Update.create(Update.TYPE_PLAIN, fw) - print('Creating %s' % args.UPFILE) + print('%sCreating %s' % (self.prefix, args.UPFILE)) up.tofile(args.UPFILE) return 0 class PatchFwCommand(Command): - def __init__(self, subs:SPA) -> None: + def __init__(self, subs:SPA, prefix:str='') -> None: p = super().create_parser(subs, 'patch', 'Patch a firmware file with CRC and length.') p.add_argument('FWFILE', type=str, help='the firmware to process') p.add_argument('--check-only', action='store_true', help='only check CRC and length values, do not patch') + self.prefix = prefix def main(self, args:NS) -> int: - print('Processing %s' % args.FWFILE) + print('%sProcessing %s' % (self.prefix, args.FWFILE)) fw = Firmware(args.FWFILE) if not args.check_only: fw.patch() fw.tofile(args.FWFILE) - print('CRC: 0x%08x (%s)' % (fw.hcrc, 'ok' if fw.hcrc == fw.crc else 'invalid')) - print('Size: 0x%08x (%s)' % (fw.hsize, 'ok' if fw.hsize == fw.size else 'invalid')) + print('%sCRC: 0x%08x (%s)' % (self.prefix, fw.hcrc, 'ok' if fw.hcrc == fw.crc else 'invalid')) + print('%sSize: 0x%08x: %d bytes (%s)' % (self.prefix, fw.hsize, fw.hsize, 'ok' if fw.hsize == fw.size else 'invalid')) return 0 if fw.hcrc == fw.crc and fw.hsize == fw.size else 1 @@ -210,8 +214,10 @@ def main(self, args:NS) -> int: subs = parser.add_subparsers(dest='COMMAND') subs.required = True # type: ignore # http://bugs.python.org/issue9253 - PatchFwCommand(subs) - MkUpdateCommand(subs) + prefix = os.environ.get('STDOUT_PREFIX', '') + + PatchFwCommand(subs, prefix) + MkUpdateCommand(subs, prefix) args = parser.parse_args() sys.exit(args.func(args))