Skip to content

Commit 82fb277

Browse files
pyalexpyalex
authored
chore: Build platform specific python package (feast-dev#2459)
* Build platform specific python package Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * update gopy Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * enable dynamic lookup in gopy Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * update gopy Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * update gopy Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * allow all flags in cgo Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * no windows for now Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * moved gopy repo to feast-dev Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * add no-go package + py3.10 version Signed-off-by: pyalex <moskalenko.alexey@gmail.com> * fix for older distutils Signed-off-by: pyalex <moskalenko.alexey@gmail.com>
1 parent b943382 commit 82fb277

6 files changed

Lines changed: 74 additions & 14 deletions

File tree

‎.github/workflows/master_only.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
strategy:
6161
fail-fast: false
6262
matrix:
63-
python-version: [ 3.7, 3.8, 3.9 ]
63+
python-version: [ 3.7, 3.8, 3.9, 3.10 ]
6464
go-version: [ 1.17.0 ]
6565
os: [ ubuntu-latest ]
6666
env:

‎.github/workflows/publish.yml‎

Lines changed: 56 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,37 +126,85 @@ jobs:
126126
run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
127127

128128
publish-python-sdk:
129-
runs-on: ubuntu-latest
129+
runs-on: ${{ matrix.os }}
130+
strategy:
131+
fail-fast: false
132+
matrix:
133+
python-version: [ 3.7, 3.8, 3.9, 3.10 ]
134+
os: [ ubuntu-latest, macOS-latest ]
135+
compile-go: [ True ]
136+
include:
137+
- python-version: 3.7
138+
os: ubuntu-latest
139+
compile-go: False
130140
env:
131141
TWINE_USERNAME: __token__
132142
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
133-
container: python:3.7
143+
COMPILE_GO: ${{ matrix.compile-go }}
134144
steps:
135145
- uses: actions/checkout@v2
146+
- name: Setup Python
147+
id: setup-python
148+
uses: actions/setup-python@v2
149+
with:
150+
python-version: ${{ matrix.python-version }}
151+
architecture: x64
152+
- name: Setup Go
153+
id: setup-go
154+
uses: actions/setup-go@v2
155+
with:
156+
go-version: 1.17.7
157+
- name: Upgrade pip version
158+
run: |
159+
pip install --upgrade "pip>=21.3.1"
136160
- name: Install pip-tools
137161
run: pip install pip-tools
138162
- name: Install dependencies
139-
run: make install-python-ci-dependencies PYTHON=3.7
163+
run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }}
140164
- name: Publish Python Package
141165
run: |
142166
cd sdk/python
143167
python3 -m pip install --user --upgrade setuptools wheel twine
144168
python3 setup.py sdist bdist_wheel
145-
python3 -m twine upload --verbose dist/*
169+
python3 -m twine upload --verbose dist/*.whl
146170
147171
publish-python-sdk-no-telemetry:
148-
runs-on: ubuntu-latest
172+
runs-on: ${{ matrix.os }}
173+
strategy:
174+
fail-fast: false
175+
matrix:
176+
python-version: [ 3.7, 3.8, 3.9, 3.10 ]
177+
os: [ ubuntu-latest, macOS-latest ]
178+
compile-go: [ True ]
179+
include:
180+
- python-version: 3.7
181+
os: ubuntu-latest
182+
compile-go: False
149183
needs: get-version
150184
env:
151185
TWINE_USERNAME: __token__
152186
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
153-
container: python:3.7
187+
COMPILE_GO: ${{ matrix.compile-go }}
154188
steps:
155189
- uses: actions/checkout@v2
190+
- name: Setup Python
191+
id: setup-python
192+
uses: actions/setup-python@v2
193+
with:
194+
python-version: ${{ matrix.python-version }}
195+
architecture: x64
196+
- name: Setup Go
197+
id: setup-go
198+
uses: actions/setup-go@v2
199+
with:
200+
go-version: 1.17.7
201+
- name: Upgrade pip version
202+
run: |
203+
pip install --upgrade "pip>=21.3.1"
156204
- name: Install pip-tools
157205
run: pip install pip-tools
158206
- name: Install dependencies
159-
run: make install-python-ci-dependencies PYTHON=3.7
207+
run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }}
160208
- name: Publish Python Package
161209
env:
162210
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }}
@@ -166,7 +214,7 @@ jobs:
166214
sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py
167215
python3 -m pip install --user --upgrade setuptools wheel twine
168216
python3 setup.py sdist bdist_wheel
169-
python3 -m twine upload --verbose dist/*
217+
python3 -m twine upload --verbose dist/*.whl
170218
171219
publish-java-sdk:
172220
container: maven:3.6-jdk-11

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,10 +141,10 @@ install-go-proto-dependencies:
141141
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
142142

143143
install-go-ci-dependencies:
144-
# ToDo: currently gopy installation doesn't work w/o explicit go get in the next two lines
144+
# ToDo: currently gopy installation doesn't work w/o explicit go get in the next line
145145
# ToDo: there should be a better way to install gopy
146-
go get golang.org/x/tools/cmd/goimports
147146
go get github.com/go-python/gopy
147+
go install golang.org/x/tools/cmd/goimports
148148
go install github.com/go-python/gopy
149149

150150
install-protoc-dependencies:

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,4 @@ require (
4949
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
5050
)
5151

52-
replace github.com/go-python/gopy v0.4.0 => github.com/pyalex/gopy v0.4.1-0.20220315220227-146bcd99e9f5
52+
replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220329011409-d705e6cd1d9b

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
8585
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
8686
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
8787
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
88+
github.com/feast-dev/gopy v0.4.1-0.20220329011409-d705e6cd1d9b h1:C/oK6gi12Q7fiiVCI3e62tqWCSXqsTz9OpyK249XI84=
89+
github.com/feast-dev/gopy v0.4.1-0.20220329011409-d705e6cd1d9b/go.mod h1:ZO6vpitQ61NVoQP/2yOubPS6ET5pP3CAWCiMYn5eqCc=
8890
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
8991
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
9092
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -318,8 +320,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
318320
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
319321
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
320322
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
321-
github.com/pyalex/gopy v0.4.1-0.20220315220227-146bcd99e9f5 h1:+N63y8ub8/tUt4Z+U0WkoLpqX08biafsKfr1ZVe3Uus=
322-
github.com/pyalex/gopy v0.4.1-0.20220315220227-146bcd99e9f5/go.mod h1:ZO6vpitQ61NVoQP/2yOubPS6ET5pP3CAWCiMYn5eqCc=
323323
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
324324
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
325325
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

‎sdk/python/setup.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@
2828
from setuptools.command.build_py import build_py
2929
from setuptools.command.develop import develop
3030
from setuptools.command.install import install
31+
from setuptools.dist import Distribution
3132
except ImportError:
3233
from distutils.command.build_py import build_py
3334
from distutils.core import setup
35+
from distutils.dist import Distribution
3436

3537
NAME = "feast"
3638
DESCRIPTION = "Python SDK for Feast"
@@ -355,9 +357,11 @@ def _compile_embedded_lib(self):
355357
"feast/embedded_go/lib",
356358
"-vm",
357359
"python3",
360+
"-no-make",
358361
"github.com/feast-dev/feast/go/embedded"
359362
], env={
360363
"PATH": self.path_val,
364+
"CGO_LDFLAGS_ALLOW": ".*",
361365
**self.go_env,
362366
})
363367

@@ -389,6 +393,13 @@ def run(self):
389393
develop.run(self)
390394

391395

396+
class BinaryDistribution(Distribution):
397+
"""Distribution which forces a binary package with platform name
398+
when go compilation is enabled"""
399+
def has_ext_modules(self):
400+
return os.getenv("COMPILE_GO", "false").lower() == "true"
401+
402+
392403
setup(
393404
name=NAME,
394405
author=AUTHOR,
@@ -445,4 +456,5 @@ def run(self):
445456
"build_py": BuildCommand,
446457
"develop": DevelopCommand,
447458
},
459+
distclass=BinaryDistribution, # generate wheel with platform-specific name
448460
)

0 commit comments

Comments
 (0)