diff --git a/.github/workflows/docker_smoke_tests.yml b/.github/workflows/docker_smoke_tests.yml new file mode 100644 index 00000000000..1aed85cd93e --- /dev/null +++ b/.github/workflows/docker_smoke_tests.yml @@ -0,0 +1,112 @@ +name: docker-smoke-tests + +on: + pull_request: + paths: + - "sdk/python/feast/infra/feature_servers/multicloud/**" + - "sdk/python/feast/feature_server.py" + - "infra/scripts/feature_server_docker_smoke.py" + - "Makefile" + - ".github/workflows/publish_images.yml" + - ".github/workflows/docker_smoke_tests.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + feature-server-docker-smoke: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + - name: Build feature-server image + env: + ARCH: ${{ matrix.arch }} + run: | + make build-feature-server-docker REGISTRY=feastdev VERSION=smoke-${ARCH} DOCKER_PLATFORMS=linux/${ARCH} + - name: Run container + env: + ARCH: ${{ matrix.arch }} + run: | + docker run -d --rm \ + --name feature-server-smoke-${ARCH} \ + --platform linux/${ARCH} \ + -p 6566:6566 \ + -v "${GITHUB_WORKSPACE}/infra/scripts/feature_server_docker_smoke.py:/smoke.py:ro" \ + feastdev/feature-server:smoke-${ARCH} \ + python /smoke.py + - name: Wait for /health + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:6566/health >/dev/null; then + exit 0 + fi + sleep 2 + done + echo "feature-server /health did not become ready" + docker logs feature-server-smoke-${{ matrix.arch }} || true + exit 1 + - name: Cleanup + if: always() + env: + ARCH: ${{ matrix.arch }} + run: | + docker stop feature-server-smoke-${ARCH} || true + feature-server-dev-docker-smoke: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + - name: Build feature-server-dev image + env: + ARCH: ${{ matrix.arch }} + run: | + make build-feature-server-dev-docker REGISTRY=feastdev VERSION=smoke-${ARCH} DOCKER_PLATFORMS=linux/${ARCH} + - name: Run container + env: + ARCH: ${{ matrix.arch }} + run: | + docker run -d --rm \ + --name feature-server-dev-smoke-${ARCH} \ + --platform linux/${ARCH} \ + -p 6566:6566 \ + -v "${GITHUB_WORKSPACE}/infra/scripts/feature_server_docker_smoke.py:/smoke.py:ro" \ + feastdev/feature-server:smoke-${ARCH} \ + python /smoke.py + - name: Wait for /health + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:6566/health >/dev/null; then + exit 0 + fi + sleep 2 + done + echo "feature-server /health did not become ready" + docker logs feature-server-dev-smoke-${{ matrix.arch }} || true + exit 1 + - name: Cleanup + if: always() + env: + ARCH: ${{ matrix.arch }} + run: | + docker stop feature-server-dev-smoke-${ARCH} || true diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 324c4017b5e..83355979546 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -89,7 +89,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - component: [ feature-server-dev, feature-transformation-server, feast-operator ] + include: + - component: feature-server-dev + target: feature-server-dev + build_args: DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 + push_mode: imagetools + - component: feature-transformation-server + target: feature-transformation-server + build_args: "" + push_mode: all_tags + - component: feast-operator + target: feast-operator + build_args: "" + push_mode: all_tags env: REGISTRY: quay.io/feastdev-ci steps: @@ -117,14 +129,11 @@ jobs: username: ${{ secrets.QUAYIO_CI_USERNAME }} password: ${{ secrets.QUAYIO_CI_TOKEN }} - name: Build image - run: | - make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} + run: make build-${{ matrix.target }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} ${{ matrix.build_args }} - name: Push image run: | - if [[ "${{ matrix.component }}" == "feature-server-dev" ]]; then - docker tag ${REGISTRY}/feature-server:${GITHUB_SHA} ${REGISTRY}/feature-server:develop - docker push ${REGISTRY}/feature-server --all-tags + if [[ "${{ matrix.push_mode }}" == "imagetools" ]]; then + docker buildx imagetools create -t ${REGISTRY}/feature-server:develop ${REGISTRY}/feature-server:${GITHUB_SHA} else - docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop - docker push ${REGISTRY}/${{ matrix.component }} --all-tags + docker tag ${REGISTRY}/${{ matrix.target }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.target }}:develop && docker push ${REGISTRY}/${{ matrix.target }} --all-tags fi diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 3f464367a34..63055981a90 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -69,17 +69,29 @@ jobs: env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} run: | - make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + if [ "${{ matrix.component }}" = "feature-server" ]; then + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 + else + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + fi - name: Push versioned images env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} HIGHEST_SEMVER_TAG: ${{ steps.get-version.outputs.highest_semver_tag }} run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + if [ "${{ matrix.component }}" = "feature-server" ]; then + echo "feature-server image pushed via buildx during build step" + else + make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + fi echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] then - docker tag ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} ${REGISTRY}/${{ matrix.component }}:latest - docker push ${REGISTRY}/${{ matrix.component }}:latest + if [ "${{ matrix.component }}" = "feature-server" ]; then + docker buildx imagetools create -t ${REGISTRY}/feature-server:latest ${REGISTRY}/feature-server:${VERSION_WITHOUT_PREFIX} + else + docker tag ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} ${REGISTRY}/${{ matrix.component }}:latest + docker push ${REGISTRY}/${{ matrix.component }}:latest + fi fi diff --git a/Makefile b/Makefile index 93050b4b102..3e0fbce6f6d 100644 --- a/Makefile +++ b/Makefile @@ -659,10 +659,10 @@ push-feature-server-docker: ## Push Feature Server Docker image docker push $(REGISTRY)/feature-server:$(VERSION) build-feature-server-docker: ## Build Feature Server Docker image - docker buildx build \ + docker buildx build $(if $(DOCKER_PLATFORMS),--platform $(DOCKER_PLATFORMS),) \ -t $(REGISTRY)/feature-server:$(VERSION) \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile \ - --load sdk/python/feast/infra/feature_servers/multicloud + $(if $(filter true,$(DOCKER_PUSH)),--push,--load) sdk/python/feast/infra/feature_servers/multicloud push-feature-transformation-server-docker: ## Push Feature Transformation Server Docker image docker push $(REGISTRY)/feature-transformation-server:$(VERSION) @@ -715,9 +715,9 @@ build-feature-server-dev: ## Build Feature Server Dev Docker image -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . build-feature-server-dev-docker: ## Build Feature Server Dev Docker image - docker buildx build \ + docker buildx build $(if $(DOCKER_PLATFORMS),--platform $(DOCKER_PLATFORMS),) \ -t $(REGISTRY)/feature-server:$(VERSION) \ - -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev $(if $(filter true,$(DOCKER_PUSH)),--push,--load) . build-feature-server-dev-docker_on_mac: ## Build Feature Server Dev Docker image on Mac docker buildx build --platform linux/amd64 \ diff --git a/infra/scripts/feature_server_docker_smoke.py b/infra/scripts/feature_server_docker_smoke.py new file mode 100644 index 00000000000..5eac394bccd --- /dev/null +++ b/infra/scripts/feature_server_docker_smoke.py @@ -0,0 +1,38 @@ +from types import SimpleNamespace + +import uvicorn + +from feast.feature_server import get_app + + +class _FakeRegistry: + def proto(self): + return object() + + +class _FakeStore: + def __init__(self): + self.config = SimpleNamespace() + self.registry = _FakeRegistry() + self._provider = SimpleNamespace( + async_supported=SimpleNamespace( + online=SimpleNamespace(read=False, write=False) + ) + ) + + def _get_provider(self): + return self._provider + + async def initialize(self): + return None + + def refresh_registry(self): + return None + + async def close(self): + return None + + +if __name__ == "__main__": + app = get_app(_FakeStore()) + uvicorn.run(app, host="0.0.0.0", port=6566, log_level="error")