Skip to content

Commit e567355

Browse files
pyalexpyalex
committed
protos build dest
Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
1 parent 04ba4b5 commit e567355

2 files changed

Lines changed: 77 additions & 59 deletions

File tree

‎.github/workflows/publish.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ jobs:
184184
env:
185185
COMPILE_GO: "True"
186186
run: |
187-
pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.1'
187+
pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0'
188188
pip install dist/*tar.gz
189189
- name: Install OS X dependencies
190190
if: matrix.os == 'macos-10.15'
@@ -267,7 +267,6 @@ jobs:
267267
make build-ui
268268
- name: Build
269269
run: |
270-
python3 setup.py build_python_protos
271270
python3 setup.py sdist bdist_wheel
272271
273272
- uses: actions/upload-artifact@v2

‎setup.py‎

Lines changed: 76 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,7 @@
114114
"happybase>=1.2.0,<3",
115115
]
116116

117-
GE_REQUIRED = [
118-
"great_expectations>=0.14.0,<0.15.0"
119-
]
117+
GE_REQUIRED = ["great_expectations>=0.14.0,<0.15.0"]
120118

121119
GO_REQUIRED = [
122120
"cffi==1.15.*,<2",
@@ -166,15 +164,15 @@
166164
"types-setuptools",
167165
"types-tabulate",
168166
]
169-
+ GCP_REQUIRED
170-
+ REDIS_REQUIRED
171-
+ AWS_REQUIRED
172-
+ SNOWFLAKE_REQUIRED
173-
+ SPARK_REQUIRED
174-
+ POSTGRES_REQUIRED
175-
+ TRINO_REQUIRED
176-
+ GE_REQUIRED
177-
+ HBASE_REQUIRED
167+
+ GCP_REQUIRED
168+
+ REDIS_REQUIRED
169+
+ AWS_REQUIRED
170+
+ SNOWFLAKE_REQUIRED
171+
+ SPARK_REQUIRED
172+
+ POSTGRES_REQUIRED
173+
+ TRINO_REQUIRED
174+
+ GE_REQUIRED
175+
+ HBASE_REQUIRED
178176
)
179177

180178
DEV_REQUIRED = ["mypy-protobuf==3.1", "grpcio-testing==1.*"] + CI_REQUIRED
@@ -206,7 +204,9 @@
206204

207205
class BuildPythonProtosCommand(Command):
208206
description = "Builds the proto files into Python files."
209-
user_options = []
207+
user_options = [
208+
("inplace", "i", "Write generated proto files to source directory."),
209+
]
210210

211211
def initialize_options(self):
212212
self.python_protoc = [
@@ -215,13 +215,21 @@ def initialize_options(self):
215215
"grpc_tools.protoc",
216216
] # find_executable("protoc")
217217
self.proto_folder = os.path.join(repo_root, "protos")
218-
self.python_folder = os.path.join(
219-
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
220-
)
221218
self.sub_folders = PROTO_SUBDIRS
219+
self.build_lib = None
220+
self.inplace = 0
222221

223222
def finalize_options(self):
224-
pass
223+
self.set_undefined_options("build", ("build_lib", "build_lib"))
224+
225+
@property
226+
def python_folder(self):
227+
if self.inplace:
228+
return os.path.join(
229+
os.path.dirname(__file__) or os.getcwd(), "sdk/python/feast/protos"
230+
)
231+
232+
return os.path.join(self.build_lib, "feast/protos")
225233

226234
def _generate_python_protos(self, path: str):
227235
proto_files = glob.glob(os.path.join(self.proto_folder, path))
@@ -247,13 +255,12 @@ def run(self):
247255
# We need the __init__ files for each of the generated subdirs
248256
# so that they are regular packages, and don't need the `--namespace-packages` flags
249257
# when being typechecked using mypy.
250-
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", 'w'):
258+
with open(f"{self.python_folder}/feast/{sub_folder}/__init__.py", "w"):
251259
pass
252260

253-
254-
with open(f"{self.python_folder}/__init__.py", 'w'):
261+
with open(f"{self.python_folder}/__init__.py", "w"):
255262
pass
256-
with open(f"{self.python_folder}/feast/__init__.py", 'w'):
263+
with open(f"{self.python_folder}/feast/__init__.py", "w"):
257264
pass
258265

259266
for path in Path(self.python_folder).rglob("*.py"):
@@ -295,12 +302,10 @@ def _ensure_go_and_proto_toolchain():
295302
path_val = _generate_path_with_gopath()
296303

297304
try:
298-
subprocess.check_call(["protoc-gen-go", "--version"], env={
299-
"PATH": path_val
300-
})
301-
subprocess.check_call(["protoc-gen-go-grpc", "--version"], env={
302-
"PATH": path_val
303-
})
305+
subprocess.check_call(["protoc-gen-go", "--version"], env={"PATH": path_val})
306+
subprocess.check_call(
307+
["protoc-gen-go-grpc", "--version"], env={"PATH": path_val}
308+
)
304309
except Exception as e:
305310
raise RuntimeError("Unable to find go/grpc extensions for protoc") from e
306311

@@ -329,15 +334,18 @@ def _generate_go_protos(self, path: str):
329334
try:
330335
subprocess.check_call(
331336
self.go_protoc
332-
+ ["-I", self.proto_folder,
333-
"--go_out", self.go_folder,
334-
"--go_opt=module=github.com/feast-dev/feast/go/protos",
335-
"--go-grpc_out", self.go_folder,
336-
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos"]
337+
+ [
338+
"-I",
339+
self.proto_folder,
340+
"--go_out",
341+
self.go_folder,
342+
"--go_opt=module=github.com/feast-dev/feast/go/protos",
343+
"--go-grpc_out",
344+
self.go_folder,
345+
"--go-grpc_opt=module=github.com/feast-dev/feast/go/protos",
346+
]
337347
+ proto_files,
338-
env={
339-
"PATH": self.path_val
340-
}
348+
env={"PATH": self.path_val},
341349
)
342350
except CalledProcessError as e:
343351
print(f"Stderr: {e.stderr}")
@@ -367,6 +375,7 @@ class DevelopCommand(develop):
367375
"""Custom develop command."""
368376

369377
def run(self):
378+
self.reinitialize_command("build_python_protos", inplace=1)
370379
self.run_command("build_python_protos")
371380
if os.getenv("COMPILE_GO", "false").lower() == "true":
372381
_ensure_go_and_proto_toolchain()
@@ -382,7 +391,10 @@ def finalize_options(self) -> None:
382391
self.extensions = [e for e in self.extensions if not self._is_go_ext(e)]
383392

384393
def _is_go_ext(self, ext: Extension):
385-
return any(source.endswith('.go') or source.startswith('github') for source in ext.sources)
394+
return any(
395+
source.endswith(".go") or source.startswith("github")
396+
for source in ext.sources
397+
)
386398

387399
def build_extension(self, ext: Extension):
388400
if not self._is_go_ext(ext):
@@ -400,33 +412,34 @@ def build_extension(self, ext: Extension):
400412
)
401413

402414
destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))
403-
subprocess.check_call([
404-
"gopy",
405-
"build",
406-
"-output",
407-
destination,
408-
"-vm",
409-
sys.executable,
410-
"-no-make",
411-
*ext.sources
412-
], env={
413-
"PATH": bin_path,
414-
"CGO_LDFLAGS_ALLOW": ".*",
415-
**go_env,
416-
})
415+
subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"])
416+
subprocess.check_call(["go", "install", "github.com/go-python/gopy"])
417+
subprocess.check_call(
418+
[
419+
"gopy",
420+
"build",
421+
"-output",
422+
destination,
423+
"-vm",
424+
sys.executable,
425+
"-no-make",
426+
*ext.sources,
427+
],
428+
env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,},
429+
)
417430

418431
def copy_extensions_to_source(self):
419-
build_py = self.get_finalized_command('build_py')
432+
build_py = self.get_finalized_command("build_py")
420433
for ext in self.extensions:
421434
fullname = self.get_ext_fullname(ext.name)
422-
modpath = fullname.split('.')
423-
package = '.'.join(modpath[:-1])
435+
modpath = fullname.split(".")
436+
package = ".".join(modpath[:-1])
424437
package_dir = build_py.get_package_dir(package)
425438

426439
src_dir = dest_dir = package_dir
427440

428441
if src_dir.startswith(PYTHON_CODE_PREFIX):
429-
src_dir = package_dir[len(PYTHON_CODE_PREFIX):]
442+
src_dir = package_dir[len(PYTHON_CODE_PREFIX) :]
430443
src_dir = src_dir.lstrip("/")
431444

432445
src_dir = os.path.join(self.build_lib, src_dir)
@@ -443,7 +456,9 @@ def copy_extensions_to_source(self):
443456
long_description_content_type="text/markdown",
444457
python_requires=REQUIRES_PYTHON,
445458
url=URL,
446-
packages=find_packages(where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")),
459+
packages=find_packages(
460+
where=PYTHON_CODE_PREFIX, exclude=("java", "infra", "sdk/python/tests", "ui")
461+
),
447462
package_dir={"": PYTHON_CODE_PREFIX},
448463
install_requires=REQUIRED,
449464
# https://stackoverflow.com/questions/28509965/setuptools-development-requirements
@@ -488,6 +503,10 @@ def copy_extensions_to_source(self):
488503
"develop": DevelopCommand,
489504
"build_ext": build_ext,
490505
},
491-
ext_modules=[Extension('feast.embedded_go.lib._embedded',
492-
["github.com/feast-dev/feast/go/embedded"])],
506+
ext_modules=[
507+
Extension(
508+
"feast.embedded_go.lib._embedded",
509+
["github.com/feast-dev/feast/go/embedded"],
510+
)
511+
],
493512
)

0 commit comments

Comments
 (0)