From 5c2dc785420cbbce155243d73d263496d4a1d5b5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 14:29:28 +0000 Subject: [PATCH 1/8] gh: bump code version --- .env | 4 ++-- .github/workflows/tests.yml | 2 +- VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 98ae68e6..02e14d58 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -SYSROOT_VERSION=0.2.2 -SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.2 +SYSROOT_VERSION=0.2.3 +SYSROOT_CLI_IMAGE=faasm/cpp-sysroot:0.2.3 COMPOSE_PROJECT_NAME=cpp-dev diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e03aa98d..423efe55 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm/cpp-sysroot:0.2.2 + image: faasm/cpp-sysroot:0.2.3 steps: # --- Update code --- - name: "Checkout code" diff --git a/VERSION b/VERSION index ee1372d3..71790396 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.2 +0.2.3 From 413cda4e4bc4166e3d3ed561ea5d59d83fe640dd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 14:47:01 +0000 Subject: [PATCH 2/8] llvm: remove submodule and just track our patch --- .gitmodules | 4 -- faasmtools/env.py | 2 +- ...TEN__-definitions-from-emscripten-ta.patch | 44 +++++++++++++++++++ tasks/llvm.py | 29 +++++++++++- tasks/util/env.py | 4 +- third-party/llvm-project | 1 - 6 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 patches/llvm-project/0001-Remove-__EMSCRIPTEN__-definitions-from-emscripten-ta.patch delete mode 160000 third-party/llvm-project diff --git a/.gitmodules b/.gitmodules index 1e247d0d..bc195df9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,10 +2,6 @@ path = third-party/wasi-libc url = https://github.com/faasm/wasi-libc branch = faasm -[submodule "third-party/llvm-project"] - path = third-party/llvm-project - url = https://github.com/faasm/llvm-project.git - branch = faasm [submodule "third-party/libffi"] path = third-party/libffi url = https://github.com/faasm/libffi.git diff --git a/faasmtools/env.py b/faasmtools/env.py index f41bca23..956e36be 100644 --- a/faasmtools/env.py +++ b/faasmtools/env.py @@ -14,7 +14,7 @@ # LLVM variables LLVM_VERSION = "13.0.1" -LLVM_DIR = join(THIRD_PARTY_DIR, "llvm-project") +LLVM_DIR = join("/opt", "llvm-project") LLVM_MAKEFILE = join(PROJ_ROOT, "LLVM.makefile") WASI_LIBC_DIR = join(THIRD_PARTY_DIR, "wasi-libc") diff --git a/patches/llvm-project/0001-Remove-__EMSCRIPTEN__-definitions-from-emscripten-ta.patch b/patches/llvm-project/0001-Remove-__EMSCRIPTEN__-definitions-from-emscripten-ta.patch new file mode 100644 index 00000000..af49c34e --- /dev/null +++ b/patches/llvm-project/0001-Remove-__EMSCRIPTEN__-definitions-from-emscripten-ta.patch @@ -0,0 +1,44 @@ +From 4d755c1f417a09cbe5156af4c04782a10cb047aa Mon Sep 17 00:00:00 2001 +From: Simon Shillaker +Date: Wed, 27 Apr 2022 16:55:11 +0000 +Subject: [PATCH] Remove __EMSCRIPTEN__ definitions from emscripten target + +--- + clang/lib/Basic/Targets/OSTargets.h | 14 ++++++++++---- + 1 file changed, 10 insertions(+), 4 deletions(-) + +diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h +index 12df95c19f35..5aaf0e7fc84c 100644 +--- a/clang/lib/Basic/Targets/OSTargets.h ++++ b/clang/lib/Basic/Targets/OSTargets.h +@@ -291,7 +291,7 @@ protected: + Builder.defineMacro("__HAIKU__"); + Builder.defineMacro("__ELF__"); + DefineStd(Builder, "unix", Opts); +- if (this->HasFloat128) ++ if (this->HasFloat128) + Builder.defineMacro("__FLOAT128__"); + } + +@@ -946,9 +946,15 @@ class LLVM_LIBRARY_VISIBILITY EmscriptenTargetInfo + void getOSDefines(const LangOptions &Opts, const llvm::Triple &Triple, + MacroBuilder &Builder) const final { + WebAssemblyOSTargetInfo::getOSDefines(Opts, Triple, Builder); +- Builder.defineMacro("__EMSCRIPTEN__"); +- if (Opts.POSIXThreads) +- Builder.defineMacro("__EMSCRIPTEN_PTHREADS__"); ++ // For Faasm, we need to use the Emscripten target to build shared ++ // libraries, but we don't actually want any of the emscripten features. ++ // See htps://bugs.llvm.org/show_bug.cgi?id=42714 ++ // ++ Builder.defineMacro("__wasi__"); ++ ++ //Builder.defineMacro("__EMSCRIPTEN__"); ++ //if (Opts.POSIXThreads) ++ // Builder.defineMacro("__EMSCRIPTEN_PTHREADS__"); + } + + public: +-- +2.34.1 + diff --git a/tasks/llvm.py b/tasks/llvm.py index d53767ea..8110834a 100644 --- a/tasks/llvm.py +++ b/tasks/llvm.py @@ -13,9 +13,10 @@ WASI_LIBC_DIR, ) from invoke import task -from os import environ -from os.path import join +from os import environ, listdir +from os.path import exists, join from subprocess import run +from tasks.util.env import PATCHES_DIR LLVM_BUILD_DIR = join(LLVM_DIR, "build") LLVM_BUILD_ENV_VARS = { @@ -29,6 +30,27 @@ } +def do_llvm_clone(): + """ + Clone the llvm-project repo and apply faasm-specific patches + """ + print("WARNING: Cloning llvm-project, this can take a while...") + git_cmd = [ + "git clone", + "-b llvmorg-{}".format(LLVM_VERSION), + "--depth 1", + "https://github.com/llvm/llvm-project", + LLVM_DIR, + ] + git_cmd = " ".join(git_cmd) + run(git_cmd, shel=True, check=True) + + llvm_patches_dir = join(PATCHES_DIR, "llvm-project") + for patch in listdir(llvm_patches_dir): + git_cmd = "git apply {}".format(join(llvm_patches_dir, patch)) + run(git_cmd, shell=True, check=True, cwd=PROJ_ROOT) + + def do_llvm_build(target, clean_target=None): build_env = environ.copy() # These env. variables are required for LLVM's build configuration (and @@ -61,6 +83,9 @@ def build(ctx, clean=False): """ Build LLVM and all targets (including libc) """ + if not exists(LLVM_DIR): + do_llvm_clone() + do_llvm_build("", "clean-all" if clean else None) diff --git a/tasks/util/env.py b/tasks/util/env.py index af04e4be..137a846e 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -1,3 +1,5 @@ -from os.path import dirname, realpath +from os.path import dirname, join, realpath PROJ_ROOT = dirname(dirname(dirname(realpath(__file__)))) + +PATCHES_DIR = join(PROJ_ROOT, "patches") diff --git a/third-party/llvm-project b/third-party/llvm-project deleted file mode 160000 index 4d755c1f..00000000 --- a/third-party/llvm-project +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4d755c1f417a09cbe5156af4c04782a10cb047aa From bedecb3fc0d0f2ed8afaea968c7c61adcbcf01bd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 15:11:00 +0000 Subject: [PATCH 3/8] fix typo --- docker/llvm.dockerfile | 2 +- tasks/llvm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/llvm.dockerfile b/docker/llvm.dockerfile index 9476c864..35a8a3cc 100644 --- a/docker/llvm.dockerfile +++ b/docker/llvm.dockerfile @@ -38,7 +38,7 @@ RUN mkdir -p /code \ https://github.com/faasm/cpp \ /code/cpp \ && cd /code/cpp \ - && git submodule update --init -f third-party/llvm-project \ + # && git submodule update --init -f third-party/llvm-project \ && git submodule update --init -f third-party/wasi-libc \ && ./bin/create_venv.sh \ && source venv/bin/activate \ diff --git a/tasks/llvm.py b/tasks/llvm.py index 8110834a..8404fd7b 100644 --- a/tasks/llvm.py +++ b/tasks/llvm.py @@ -43,7 +43,7 @@ def do_llvm_clone(): LLVM_DIR, ] git_cmd = " ".join(git_cmd) - run(git_cmd, shel=True, check=True) + run(git_cmd, shell=True, check=True) llvm_patches_dir = join(PATCHES_DIR, "llvm-project") for patch in listdir(llvm_patches_dir): From 69c1524ec5c06bcf0b50f180d7612bc07500600d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 15:33:31 +0000 Subject: [PATCH 4/8] llvm: apply patch in the right directory --- tasks/llvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/llvm.py b/tasks/llvm.py index 8404fd7b..1f35097d 100644 --- a/tasks/llvm.py +++ b/tasks/llvm.py @@ -48,7 +48,7 @@ def do_llvm_clone(): llvm_patches_dir = join(PATCHES_DIR, "llvm-project") for patch in listdir(llvm_patches_dir): git_cmd = "git apply {}".format(join(llvm_patches_dir, patch)) - run(git_cmd, shell=True, check=True, cwd=PROJ_ROOT) + run(git_cmd, shell=True, check=True, cwd=LLVM_DIR) def do_llvm_build(target, clean_target=None): From f22727198b6948a3493a5f63bc84badcdfa234a8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 16:19:35 +0000 Subject: [PATCH 5/8] llvm: call clean-libc before very-clean --- LLVM.makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LLVM.makefile b/LLVM.makefile index 2f642fa3..42a8597d 100644 --- a/LLVM.makefile +++ b/LLVM.makefile @@ -30,7 +30,7 @@ clean-libc: rm -rf $(BUILD_DIR)/libc.BUILT $(WASI_LIBC_DIR)/build .PHONY: very-clean-libc -very-clean-libc: +very-clean-libc: clean-libc # WARNING: this is going to remove _everything_ in FAASM_SYSROOT cd $(WASI_LIBC_DIR) && SYSROOT=$(FAASM_SYSROOT) make clean From 40cec0f366afbb257854092daf52e8e18e409912 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 16:27:49 +0000 Subject: [PATCH 6/8] docker: get llvm sources from parent image --- docker/cpp-sysroot.dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cpp-sysroot.dockerfile b/docker/cpp-sysroot.dockerfile index ae3855ea..cf8f8115 100644 --- a/docker/cpp-sysroot.dockerfile +++ b/docker/cpp-sysroot.dockerfile @@ -1,13 +1,14 @@ # llvm image is not re-built often, so the tag may be behind -FROM faasm/llvm:0.2.2 as llvm +FROM faasm/llvm:0.2.3 as llvm # faabric-base image is not re-built often, so tag may be behind FROM faasm/faabric-base:0.4.1 SHELL ["/bin/bash", "-c"] ENV CPP_DOCKER="on" -# Copy the toolchain in from the LLVM container +# Copy the toolchain and LLVM sources from the LLVM container COPY --from=llvm /usr/local/faasm /usr/local/faasm +COPY --from=llvm /opt/llvm-project /opt/llvm-project # Update APT dependencies RUN apt update && apt install -y autotools-dev @@ -18,7 +19,6 @@ RUN mkdir -p /code \ https://github.com/faasm/cpp \ /code/cpp \ && cd /code/cpp \ - && git submodule update --init -f third-party/llvm-project \ && git submodule update --init -f third-party/faabric \ && git submodule update --init -f third-party/faasm-clapack \ && git submodule update --init -f third-party/libffi \ From 0082a650bf0bcc567d154dffe0ea3088ba482ee2 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 16:44:24 +0000 Subject: [PATCH 7/8] docker: cleanup --- docker/llvm.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/llvm.dockerfile b/docker/llvm.dockerfile index 35a8a3cc..3d69e196 100644 --- a/docker/llvm.dockerfile +++ b/docker/llvm.dockerfile @@ -38,7 +38,6 @@ RUN mkdir -p /code \ https://github.com/faasm/cpp \ /code/cpp \ && cd /code/cpp \ - # && git submodule update --init -f third-party/llvm-project \ && git submodule update --init -f third-party/wasi-libc \ && ./bin/create_venv.sh \ && source venv/bin/activate \ From 5680c32ad471299a37877152fdb7c51fc72567f4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 4 Mar 2023 17:26:04 +0000 Subject: [PATCH 8/8] gha: use access token instead of password --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7cee564..cb6f7087 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + password: ${{ secrets.DOCKER_TOKEN }} - name: "Build and push cpp-sysroot container" id: docker_build uses: docker/build-push-action@v3