From a6eafdecf95380acf56ed32162cef5e8cd8d005b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Jul 2022 11:50:20 -0700 Subject: [PATCH 01/39] Temp fix Signed-off-by: Kevin Zhang --- Makefile | 2 +- go.mod | 8 +++--- go.sum | 9 ++++--- go/embedded/online_features.go | 1 + .../online_store/test_universal_online.py | 27 ++++++++++++++----- setup.py | 6 +++-- 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 176e2cb3545..433510ae6ea 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ install-go-proto-dependencies: install-go-ci-dependencies: # TODO: currently gopy installation doesn't work w/o explicit go get in the next line # TODO: there should be a better way to install gopy - go get github.com/go-python/gopy@v0.4.0 + go get github.com/go-python/gopy@v0.4.4 go install golang.org/x/tools/cmd/goimports # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` # but the following command is needed to install it for some reason. diff --git a/go.mod b/go.mod index 90ddb93e21c..ef64636e73e 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,10 @@ module github.com/feast-dev/feast go 1.17 -replace github.com/go-python/gopy v0.4.0 => github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 - require ( github.com/apache/arrow/go/v8 v8.0.0 github.com/ghodss/yaml v1.0.0 - github.com/go-python/gopy v0.4.0 + github.com/go-python/gopy v0.4.4 github.com/go-redis/redis/v8 v8.11.4 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 @@ -28,6 +26,8 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/gonuts/commander v0.1.0 // indirect + github.com/gonuts/flag v0.1.0 // indirect github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.1 // indirect @@ -38,7 +38,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 933ecf6b29c..d0b7f8fcd80 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0 h1:Go714ObVP1O+a6qK7haXVL28QNm6WMD8bwnN9EA8PlM= -github.com/feast-dev/gopy v0.4.1-0.20220429180328-4257ac71a4d0/go.mod h1:ZO6vpitQ61NVoQP/2yOubPS6ET5pP3CAWCiMYn5eqCc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -112,6 +110,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-python/gopy v0.4.4 h1:3LTsrfVcmg2VEM6wU+eh4d9EZn5H2iogObXjiQHrF8Q= +github.com/go-python/gopy v0.4.4/go.mod h1:tlA/KcD7rM8B+NQJR4SASwiinfKY0aiMFanHszR8BZA= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -147,7 +147,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= +github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -439,8 +441,9 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index f6b21169e10..e4791e67bf6 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -180,6 +180,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures( outputFields := make([]arrow.Field, 0) outputColumns := make([]arrow.Array, 0) + //pool := memory.NewCgoArrowAllocator() pool := memory.NewGoAllocator() for _, featureVector := range resp { outputFields = append(outputFields, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 228069c13ca..364b393fbc7 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -448,6 +448,8 @@ def test_online_retrieval_with_event_timestamps( def test_online_retrieval( environment, universal_data_sources, feature_server_endpoint, full_feature_names ): + import faulthandler; faulthandler.enable() + fs = environment.feature_store entities, datasets, data_sources = universal_data_sources feature_views = construct_universal_feature_views(data_sources) @@ -549,13 +551,24 @@ def test_online_retrieval( unprefixed_feature_refs.remove("conv_rate_plus_100") unprefixed_feature_refs.remove("conv_rate_plus_val_to_add") - online_features_dict = get_online_features_dict( - environment=environment, - endpoint=feature_server_endpoint, - features=feature_refs, - entity_rows=entity_rows, - full_feature_names=full_feature_names, - ) + # online_features_dict = get_online_features_dict( + # environment=environment, + # endpoint=feature_server_endpoint, + # features=feature_refs, + # entity_rows=entity_rows, + # full_feature_names=full_feature_names, + # ) + import psutil + + for i in range(1000): + print(psutil.virtual_memory()) + online_features_dict = get_online_features_dict( + environment=environment, + endpoint=feature_server_endpoint, + features=feature_refs, + entity_rows=entity_rows, + full_feature_names=full_feature_names, + ) # Test that the on demand feature views compute properly even if the dependent conv_rate # feature isn't requested. diff --git a/setup.py b/setup.py index 217b2e10118..26810e9a5a4 100644 --- a/setup.py +++ b/setup.py @@ -430,7 +430,7 @@ def build_extension(self, ext: Extension): destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) subprocess.check_call(["go", "install", "golang.org/x/tools/cmd/goimports"], env={"PATH": bin_path, **go_env}) - subprocess.check_call(["go", "get", "github.com/go-python/gopy@v0.4.0"], + subprocess.check_call(["go", "get", "github.com/go-python/gopy@v0.4.4"], env={"PATH": bin_path, **go_env}) subprocess.check_call(["go", "install", "github.com/go-python/gopy"], env={"PATH": bin_path, **go_env}) @@ -442,7 +442,9 @@ def build_extension(self, ext: Extension): destination, "-vm", sys.executable, - "-no-make", + "--build-tags", + 'cgo ccalloc', + # "-no-make", *ext.sources, ], env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,}, From 2558144720ab3d30d3db4cd084ea76ad86b91df3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Jul 2022 11:50:44 -0700 Subject: [PATCH 02/39] Temp fix Signed-off-by: Kevin Zhang --- go/embedded/online_features.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index e4791e67bf6..619cb1f6c60 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -180,8 +180,8 @@ func (s *OnlineFeatureService) GetOnlineFeatures( outputFields := make([]arrow.Field, 0) outputColumns := make([]arrow.Array, 0) - //pool := memory.NewCgoArrowAllocator() - pool := memory.NewGoAllocator() + pool := memory.NewCgoArrowAllocator() + // pool := memory.NewGoAllocator() for _, featureVector := range resp { outputFields = append(outputFields, arrow.Field{ From f5b98563a615eab64f99406298014d8149d98973 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Jul 2022 11:56:00 -0700 Subject: [PATCH 03/39] Fix Signed-off-by: Kevin Zhang --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26810e9a5a4..b167e403b84 100644 --- a/setup.py +++ b/setup.py @@ -444,7 +444,7 @@ def build_extension(self, ext: Extension): sys.executable, "--build-tags", 'cgo ccalloc', - # "-no-make", + "-no-make", *ext.sources, ], env={"PATH": bin_path, "CGO_LDFLAGS_ALLOW": ".*", **go_env,}, From f3901d3617dff4c1ca513db1db16e8a13d5be913 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Jul 2022 13:02:13 -0700 Subject: [PATCH 04/39] Fix Signed-off-by: Kevin Zhang --- go/embedded/online_features.go | 1 - .../online_store/test_universal_online.py | 27 +++++-------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 619cb1f6c60..e5860507e45 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -181,7 +181,6 @@ func (s *OnlineFeatureService) GetOnlineFeatures( outputFields := make([]arrow.Field, 0) outputColumns := make([]arrow.Array, 0) pool := memory.NewCgoArrowAllocator() - // pool := memory.NewGoAllocator() for _, featureVector := range resp { outputFields = append(outputFields, arrow.Field{ diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 364b393fbc7..228069c13ca 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -448,8 +448,6 @@ def test_online_retrieval_with_event_timestamps( def test_online_retrieval( environment, universal_data_sources, feature_server_endpoint, full_feature_names ): - import faulthandler; faulthandler.enable() - fs = environment.feature_store entities, datasets, data_sources = universal_data_sources feature_views = construct_universal_feature_views(data_sources) @@ -551,24 +549,13 @@ def test_online_retrieval( unprefixed_feature_refs.remove("conv_rate_plus_100") unprefixed_feature_refs.remove("conv_rate_plus_val_to_add") - # online_features_dict = get_online_features_dict( - # environment=environment, - # endpoint=feature_server_endpoint, - # features=feature_refs, - # entity_rows=entity_rows, - # full_feature_names=full_feature_names, - # ) - import psutil - - for i in range(1000): - print(psutil.virtual_memory()) - online_features_dict = get_online_features_dict( - environment=environment, - endpoint=feature_server_endpoint, - features=feature_refs, - entity_rows=entity_rows, - full_feature_names=full_feature_names, - ) + online_features_dict = get_online_features_dict( + environment=environment, + endpoint=feature_server_endpoint, + features=feature_refs, + entity_rows=entity_rows, + full_feature_names=full_feature_names, + ) # Test that the on demand feature views compute properly even if the dependent conv_rate # feature isn't requested. From 99289cd375d30dea4d472486631feee7f1181202 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Jul 2022 16:38:14 -0700 Subject: [PATCH 05/39] add dynamic linking flags Signed-off-by: Kevin Zhang --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b167e403b84..b99f85dfb29 100644 --- a/setup.py +++ b/setup.py @@ -444,6 +444,7 @@ def build_extension(self, ext: Extension): sys.executable, "--build-tags", 'cgo ccalloc', + "--dynamic-link=True", "-no-make", *ext.sources, ], From b56242d6385eea70939d158e5d516bd913cc3d07 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 10:43:57 -0700 Subject: [PATCH 06/39] Update gitignore Signed-off-by: Kevin Zhang --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6a86eb2682b..2cc826d71bc 100644 --- a/.gitignore +++ b/.gitignore @@ -58,8 +58,7 @@ __pycache__/ *$py.class *.prof -# C extensions -*.so + # Distribution / packaging .Python From 70c5efb72c34667e86a339944dc7be6710214423 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 15:22:32 -0700 Subject: [PATCH 07/39] Fix Signed-off-by: Kevin Zhang --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2cc826d71bc..6a86eb2682b 100644 --- a/.gitignore +++ b/.gitignore @@ -58,7 +58,8 @@ __pycache__/ *$py.class *.prof - +# C extensions +*.so # Distribution / packaging .Python From 66b1c916ab0d00a8804fa4195472570768ca0f5d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 15:27:03 -0700 Subject: [PATCH 08/39] Fix Signed-off-by: Kevin Zhang --- docs/reference/feature-servers/go-feature-retrieval.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/reference/feature-servers/go-feature-retrieval.md b/docs/reference/feature-servers/go-feature-retrieval.md index 685e7201cb6..d31d6e8831d 100644 --- a/docs/reference/feature-servers/go-feature-retrieval.md +++ b/docs/reference/feature-servers/go-feature-retrieval.md @@ -17,7 +17,9 @@ However, some additional dependencies are required for Go <-> Python interoperab pip install feast[go] ``` -For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. +For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. In order to build the go binaries, you will need to install the `apache-arrow` c libraries. +For macs, run `brew install apache-arrow`. +For linux users, run `sudo apt-get install apache-arrow`. ## Usage @@ -63,7 +65,7 @@ feature_server: emit_timeout_micro_secs: 10000 queue_capacity: 10000 ``` -All these parameters are optional. +All these parameters are optional. ## Future/Current Work From 4532c2239af5579c084de2e03a7b124dbaf30f26 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 15:38:17 -0700 Subject: [PATCH 09/39] Fix workflows Signed-off-by: Kevin Zhang --- .github/workflows/master_only.yml | 6 ++++++ .github/workflows/pr_integration_tests.yml | 6 ++++++ .github/workflows/unit_tests.yml | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 0cb49bb525c..d8d314369d6 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -127,6 +127,12 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install apache-arrow + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index e1c7ed2de2b..0344296da60 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -153,6 +153,12 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install apache-arrow + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a9bf3deba32..0be5fafb96d 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -54,6 +54,12 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install apache-arrow + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python From f0c9d60590e83c3b1cfb0f8b3bfa9c46af4dd5f3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 15:49:58 -0700 Subject: [PATCH 10/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 12 ++++++++++++ .github/workflows/master_only.yml | 8 +++++++- .github/workflows/pr_integration_tests.yml | 8 +++++++- .github/workflows/unit_tests.yml | 8 +++++++- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ea93cc85a44..c69ba4687d8 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -62,6 +62,18 @@ jobs: registry-url: 'https://registry.npmjs.org' - name: Build UI run: make build-ui + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Build wheels uses: pypa/cibuildwheel@v2.7.0 env: diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index d8d314369d6..c9ebcdaf040 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -129,7 +129,13 @@ jobs: run: pip install pip-tools - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install apache-arrow + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos if: matrix.os == 'macOS-latest' run: brew install apache-arrow diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 0344296da60..db9e48fc2d8 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -155,7 +155,13 @@ jobs: run: pip install pip-tools - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install apache-arrow + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos if: matrix.os == 'macOS-latest' run: brew install apache-arrow diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 0be5fafb96d..89c2d44d6e4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -56,7 +56,13 @@ jobs: run: pip install pip-tools - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install apache-arrow + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos if: matrix.os == 'macOS-latest' run: brew install apache-arrow From 172fc82bea498568998a1fda5ade587b1e4f6e76 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 15:58:30 -0700 Subject: [PATCH 11/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 18 ++++++++++++++++++ .github/workflows/unit_tests.yml | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a0a6d7dd382..a96d3c5a9e4 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -39,6 +39,15 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install dependencies run: | make compile-protos-go @@ -63,6 +72,15 @@ jobs: - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<22.1" + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install dependencies run: make install-go-proto-dependencies - name: Lint go diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 89c2d44d6e4..16e143b5f12 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -104,6 +104,15 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev - name: Install dependencies run: make install-go-proto-dependencies - name: Compile protos From ad46b0b1478c0ce1664d18497fef9aaab14a109d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 16:00:41 -0700 Subject: [PATCH 12/39] Fix Signed-off-by: Kevin Zhang --- .../feature-servers/go-feature-retrieval.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/reference/feature-servers/go-feature-retrieval.md b/docs/reference/feature-servers/go-feature-retrieval.md index d31d6e8831d..88a568b6793 100644 --- a/docs/reference/feature-servers/go-feature-retrieval.md +++ b/docs/reference/feature-servers/go-feature-retrieval.md @@ -16,10 +16,19 @@ However, some additional dependencies are required for Go <-> Python interoperab ``` pip install feast[go] ``` +You will also have to install the apache-arrow c++ libraries. +For macos, run `brew install apache-arrow`. +For linux users, you have to install `libarrow-dev`. +``` +sudo apt update +sudo apt install -y -V ca-certificates lsb-release wget +wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb +sudo apt update +sudo apt install -y -V libarrow-dev # For C++ +``` -For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. In order to build the go binaries, you will need to install the `apache-arrow` c libraries. -For macs, run `brew install apache-arrow`. -For linux users, run `sudo apt-get install apache-arrow`. +For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. In order to build the go binaries, you will need to install the `apache-arrow` c++ libraries. ## Usage From 16fd2881f92178bda672baecb4de9157735c7e6c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 16:03:21 -0700 Subject: [PATCH 13/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 2 -- .github/workflows/unit_tests.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a96d3c5a9e4..b2bf640def9 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -40,7 +40,6 @@ jobs: - name: Install pip-tools run: pip install pip-tools - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install -y -V ca-certificates lsb-release wget @@ -73,7 +72,6 @@ jobs: run: | pip install --upgrade "pip>=21.3.1,<22.1" - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install -y -V ca-certificates lsb-release wget diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 16e143b5f12..497ff9af033 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -105,7 +105,6 @@ jobs: with: go-version: 1.18.0 - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' run: | sudo apt update sudo apt install -y -V ca-certificates lsb-release wget From 4560fee530512af177c7d3cc388360d71f059aa5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 16:14:58 -0700 Subject: [PATCH 14/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 4 +++- .github/workflows/unit_tests.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index b2bf640def9..f6343364130 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -80,6 +80,8 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make install-go-proto-dependencies + run: | + make install-go-proto-dependencies + make install-go-ci-dependencies - name: Lint go run: make lint-go \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 497ff9af033..7def9d6662f 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -113,7 +113,9 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make install-go-proto-dependencies + run: | + make install-go-proto-dependencies + make install-go-ci-dependencies - name: Compile protos run: make compile-protos-go - name: Test From ff96f75296b50beceb726c5f92a398f5049c233e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 16:20:58 -0700 Subject: [PATCH 15/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 1 + .github/workflows/unit_tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index f6343364130..618e983f7b8 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -83,5 +83,6 @@ jobs: run: | make install-go-proto-dependencies make install-go-ci-dependencies + make compile-go-lib - name: Lint go run: make lint-go \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7def9d6662f..aca0ce13f93 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -116,6 +116,7 @@ jobs: run: | make install-go-proto-dependencies make install-go-ci-dependencies + make compile-go-lib - name: Compile protos run: make compile-protos-go - name: Test From 1baa36158970690de6a0bddcc1e5dca51109e25d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 16:24:35 -0700 Subject: [PATCH 16/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 5 +---- .github/workflows/unit_tests.yml | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 618e983f7b8..418824572e1 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -80,9 +80,6 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: | - make install-go-proto-dependencies - make install-go-ci-dependencies - make compile-go-lib + run: make compile-go-lib - name: Lint go run: make lint-go \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index aca0ce13f93..1d097a4d5e9 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -113,10 +113,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: | - make install-go-proto-dependencies - make install-go-ci-dependencies - make compile-go-lib + run: make compile-go-lib - name: Compile protos run: make compile-protos-go - name: Test From f8e9b2f05889bf8158435d89468fe3f9135788b7 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 17:01:14 -0700 Subject: [PATCH 17/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1d097a4d5e9..c39a066dc82 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -113,7 +113,9 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install dependencies - run: make compile-go-lib + run: | + make install-protoc-dependencies: + make compile-go-lib - name: Compile protos run: make compile-protos-go - name: Test From e2c1610a544bb6f2e0685e3791ede061f96008f0 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 17:04:44 -0700 Subject: [PATCH 18/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c39a066dc82..5a3d6adf87c 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -114,7 +114,7 @@ jobs: sudo apt install -y -V libarrow-dev - name: Install dependencies run: | - make install-protoc-dependencies: + make install-protoc-dependencies make compile-go-lib - name: Compile protos run: make compile-protos-go From 393c9dc4c0f49c61978ac635a17735cc0f7d7b3e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 17:15:12 -0700 Subject: [PATCH 19/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5a3d6adf87c..b908d376226 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -114,6 +114,7 @@ jobs: sudo apt install -y -V libarrow-dev - name: Install dependencies run: | + make install-python-ci-dependencies make install-protoc-dependencies make compile-go-lib - name: Compile protos From 205ef739c16c6314f4761805be35288156c181fa Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 20:48:32 -0700 Subject: [PATCH 20/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b908d376226..e84664a0bae 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -104,6 +104,8 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 + - name: Install pip-tools + run: pip install pip-tools - name: Install apache-arrow on ubuntu run: | sudo apt update From b6b16082c657a86d874693ee310c8fd75b8d7879 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 20:51:30 -0700 Subject: [PATCH 21/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e84664a0bae..bf89587b9d7 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -89,6 +89,8 @@ jobs: unit-test-go: runs-on: ubuntu-latest + env: + PYTHON: ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 - name: Setup Python From 52600af1d36073ae6f88e7fa3fa4f94952c46948 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 20:54:54 -0700 Subject: [PATCH 22/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index bf89587b9d7..cc0cd9b80cf 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -90,7 +90,7 @@ jobs: unit-test-go: runs-on: ubuntu-latest env: - PYTHON: ${{ matrix.python-version }} + PYTHON: "3.8" steps: - uses: actions/checkout@v2 - name: Setup Python From 12814996074fabecd0ca47492d75080c50cdf2f4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Jul 2022 21:07:34 -0700 Subject: [PATCH 23/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index cc0cd9b80cf..b4f4dbb5d92 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -121,7 +121,5 @@ jobs: make install-python-ci-dependencies make install-protoc-dependencies make compile-go-lib - - name: Compile protos - run: make compile-protos-go - name: Test run: make test-go From 6bb1da5dab1066d18e0a8b71c7a2376c6c92764a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 10:12:26 -0700 Subject: [PATCH 24/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 5 ----- Makefile | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index b4f4dbb5d92..ff402a1a518 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -116,10 +116,5 @@ jobs: sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt update sudo apt install -y -V libarrow-dev - - name: Install dependencies - run: | - make install-python-ci-dependencies - make install-protoc-dependencies - make compile-go-lib - name: Test run: make test-go diff --git a/Makefile b/Makefile index 433510ae6ea..4d9d3b0d17c 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,7 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store -test-go: compile-protos-go - pip install -e ".[ci]" +test-go: install-python-ci-dependencies compile-go-lib go test ./... format-go: From f779bbd547bec82772f590a82f82bf6c860ef578 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 10:43:53 -0700 Subject: [PATCH 25/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4d9d3b0d17c..86c9abce45d 100644 --- a/Makefile +++ b/Makefile @@ -189,8 +189,8 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store -test-go: install-python-ci-dependencies compile-go-lib - go test ./... +test-go: compile-go-lib + go test -tags cgo,ccalloc ./... format-go: gofmt -s -w go/ diff --git a/setup.py b/setup.py index b99f85dfb29..deaf47f955f 100644 --- a/setup.py +++ b/setup.py @@ -443,7 +443,7 @@ def build_extension(self, ext: Extension): "-vm", sys.executable, "--build-tags", - 'cgo ccalloc', + 'cgo,ccalloc', "--dynamic-link=True", "-no-make", *ext.sources, From 0fc2ed585b1a2d9dbb169db249ecf05dac42657b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 10:46:19 -0700 Subject: [PATCH 26/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 86c9abce45d..de81465c84f 100644 --- a/Makefile +++ b/Makefile @@ -189,14 +189,14 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store -test-go: compile-go-lib +test-go: compile-protos-go compile-go-lib go test -tags cgo,ccalloc ./... format-go: gofmt -s -w go/ -lint-go: compile-protos-go - go vet ./go/internal/feast ./go/embedded +lint-go: compile-protos-go compile-go-lib + go vet -tags cgo,ccalloc ./go/internal/feast ./go/embedded # Docker From a8b0a6a76180be27f7bb2856b07944e5e2fd556e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 10:49:55 -0700 Subject: [PATCH 27/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index de81465c84f..dae3118f981 100644 --- a/Makefile +++ b/Makefile @@ -180,7 +180,7 @@ install-go-ci-dependencies: python -m pip install pybindgen==0.22.0 install-protoc-dependencies: - pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 + pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 protobuf==3.20.1 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos From 7dd18c2187d28121e5138f72e7969742b371df65 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 10:55:42 -0700 Subject: [PATCH 28/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dae3118f981..6d159f0c6c9 100644 --- a/Makefile +++ b/Makefile @@ -177,10 +177,10 @@ install-go-ci-dependencies: # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` # but the following command is needed to install it for some reason. go install github.com/go-python/gopy - python -m pip install pybindgen==0.22.0 + python -m pip install pybindgen==0.22.0 protobuf==3.20.1 install-protoc-dependencies: - pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 protobuf==3.20.1 + pip install grpcio-tools==1.47.0 mypy-protobuf==3.1.0 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos From 3840495c4bf07cad7a431e43f9439969b2eccd05 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 11:10:44 -0700 Subject: [PATCH 29/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/linter.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 418824572e1..ba475e2585b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -79,7 +79,5 @@ jobs: sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb sudo apt update sudo apt install -y -V libarrow-dev - - name: Install dependencies - run: make compile-go-lib - name: Lint go run: make lint-go \ No newline at end of file From 687fe794d511ca3dfb022756cd3fdcfa4b955208 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 11:28:05 -0700 Subject: [PATCH 30/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 2 +- docs/reference/feature-servers/go-feature-retrieval.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6d159f0c6c9..6f563ab3f28 100644 --- a/Makefile +++ b/Makefile @@ -186,7 +186,7 @@ compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - COMPILE_GO=True python setup.py build_ext --inplace + CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace # Needs feast package to setup the feature store test-go: compile-protos-go compile-go-lib diff --git a/docs/reference/feature-servers/go-feature-retrieval.md b/docs/reference/feature-servers/go-feature-retrieval.md index 88a568b6793..92a9ca2ebe3 100644 --- a/docs/reference/feature-servers/go-feature-retrieval.md +++ b/docs/reference/feature-servers/go-feature-retrieval.md @@ -16,7 +16,8 @@ However, some additional dependencies are required for Go <-> Python interoperab ``` pip install feast[go] ``` -You will also have to install the apache-arrow c++ libraries. +You will also have to install the apache-arrow c++ libraries, since we use the cgo memory allocator to prevent memory from being incorrectly garbage collected, detailed in these [docs](https://pkg.go.dev/github.com/apache/arrow/go/arrow@v0.0.0-20211112161151-bc219186db40/cdata#ExportArrowRecordBatch). + For macos, run `brew install apache-arrow`. For linux users, you have to install `libarrow-dev`. ``` From b67679bbae4dd5bba1b5e3c9a3d8d9a6c8705aa4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 11:42:54 -0700 Subject: [PATCH 31/39] fix Signed-off-by: Kevin Zhang --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6f563ab3f28..4fb44d82bea 100644 --- a/Makefile +++ b/Makefile @@ -190,7 +190,7 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies # Needs feast package to setup the feature store test-go: compile-protos-go compile-go-lib - go test -tags cgo,ccalloc ./... + CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... format-go: gofmt -s -w go/ From 240a0c1d1d814f80017dc08b602fb35107558546 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 12:19:44 -0700 Subject: [PATCH 32/39] Fix Signed-off-by: Kevin Zhang --- go/internal/feast/onlinestore/sqliteonlinestore_test.go | 4 ++-- go/internal/test/go_integration_test_utils.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go/internal/feast/onlinestore/sqliteonlinestore_test.go b/go/internal/feast/onlinestore/sqliteonlinestore_test.go index 5af1c1f4ce4..a55c2a6df95 100644 --- a/go/internal/feast/onlinestore/sqliteonlinestore_test.go +++ b/go/internal/feast/onlinestore/sqliteonlinestore_test.go @@ -17,9 +17,9 @@ import ( func TestSqliteAndFeatureRepoSetup(t *testing.T) { dir := t.TempDir() feature_repo_path := filepath.Join(dir, "feature_repo") + err := test.SetupCleanFeatureRepo(dir) assert.Nil(t, err) - config, err := registry.NewRepoConfigFromFile(feature_repo_path) assert.Nil(t, err) assert.Equal(t, "feature_repo", config.Project) @@ -51,7 +51,7 @@ func TestSqliteOnlineRead(t *testing.T) { JoinKeys: []string{"driver_id"}, EntityValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}}, } - entity_key3 := types.EntityKey{ +entity_key3 := types.EntityKey{ JoinKeys: []string{"driver_id"}, EntityValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1003}}}, } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 275edc7b98e..39e13f196b1 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "time" "github.com/apache/arrow/go/v8/arrow/memory" "google.golang.org/protobuf/types/known/durationpb" @@ -89,17 +90,20 @@ func GetLatestFeatures(Rows []*Row, entities map[int64]bool) map[int64]*Row { func SetupCleanFeatureRepo(basePath string) error { cmd := exec.Command("feast", "init", "feature_repo") + path, err := filepath.Abs(basePath) cmd.Env = os.Environ() if err != nil { return err } + cmd.Dir = path err = cmd.Run() if err != nil { return err } + time.Sleep(30) applyCommand := exec.Command("feast", "apply") applyCommand.Env = os.Environ() featureRepoPath, err := filepath.Abs(filepath.Join(path, "feature_repo")) From 00d4808f5a0112c1f93a0465e5e2763e5b8c4051 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 12:28:40 -0700 Subject: [PATCH 33/39] Fix Signed-off-by: Kevin Zhang --- go/internal/test/go_integration_test_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 39e13f196b1..a47984e6f8e 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log" - "time" "github.com/apache/arrow/go/v8/arrow/memory" "google.golang.org/protobuf/types/known/durationpb" From ef309f873bed793fb033fd75fd7efd09c2d7321d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:04:56 -0700 Subject: [PATCH 34/39] Fix Signed-off-by: Kevin Zhang --- .github/workflows/unit_tests.yml | 4 ---- Makefile | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ff402a1a518..b3fadb21215 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -89,8 +89,6 @@ jobs: unit-test-go: runs-on: ubuntu-latest - env: - PYTHON: "3.8" steps: - uses: actions/checkout@v2 - name: Setup Python @@ -106,8 +104,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - - name: Install pip-tools - run: pip install pip-tools - name: Install apache-arrow on ubuntu run: | sudo apt update diff --git a/Makefile b/Makefile index 4fb44d82bea..a8ec8b73971 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies # Needs feast package to setup the feature store test-go: compile-protos-go compile-go-lib + pip install -e ".[ci]" \ CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... format-go: From 9ee05c94752ab15a40c9d1143b2dc66fcbfb3edf Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:08:26 -0700 Subject: [PATCH 35/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a8ec8b73971..10f20a2f93a 100644 --- a/Makefile +++ b/Makefile @@ -185,12 +185,14 @@ install-protoc-dependencies: compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos -compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies +compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies install- CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace +install-local-feast: + pip install -e ".[ci]" + # Needs feast package to setup the feature store -test-go: compile-protos-go compile-go-lib - pip install -e ".[ci]" \ +test-go: compile-protos-go compile-go-lib install-local-feast CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... format-go: From 3d27b6d4630349a94e936e48e505f3da785a173e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:12:42 -0700 Subject: [PATCH 36/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 10f20a2f93a..683e61ab510 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ install-protoc-dependencies: compile-protos-go: install-go-proto-dependencies install-protoc-dependencies python setup.py build_go_protos -compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies install- +compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace install-local-feast: From f644a0fb846ecec2bcbd22c0d98194b0bfec338c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:45:50 -0700 Subject: [PATCH 37/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 683e61ab510..0987bfab744 100644 --- a/Makefile +++ b/Makefile @@ -188,11 +188,11 @@ compile-protos-go: install-go-proto-dependencies install-protoc-dependencies compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace -install-local-feast: +install-feast-ci-locally: pip install -e ".[ci]" # Needs feast package to setup the feature store -test-go: compile-protos-go compile-go-lib install-local-feast +test-go: compile-protos-go compile-go-lib install-feast-ci-locally CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... format-go: From b446acb145c05dc6d01d4c977d2be8eb931cabd6 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:49:01 -0700 Subject: [PATCH 38/39] fix Signed-off-by: Kevin Zhang --- go/internal/test/go_integration_test_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index a47984e6f8e..d4ac23a5210 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -102,7 +102,6 @@ func SetupCleanFeatureRepo(basePath string) error { if err != nil { return err } - time.Sleep(30) applyCommand := exec.Command("feast", "apply") applyCommand.Env = os.Environ() featureRepoPath, err := filepath.Abs(filepath.Join(path, "feature_repo")) From c17f6fc75f56f8b95955c1fd97633b20a3f2a52b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Jul 2022 13:54:57 -0700 Subject: [PATCH 39/39] Fix Signed-off-by: Kevin Zhang --- Makefile | 1 + go/internal/feast/onlinestore/sqliteonlinestore_test.go | 4 ++-- go/internal/test/go_integration_test_utils.go | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0987bfab744..4b541963af9 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,7 @@ install-feast-ci-locally: pip install -e ".[ci]" # Needs feast package to setup the feature store +# CGO flag is due to this issue: https://github.com/golang/go/wiki/InvalidFlag test-go: compile-protos-go compile-go-lib install-feast-ci-locally CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... diff --git a/go/internal/feast/onlinestore/sqliteonlinestore_test.go b/go/internal/feast/onlinestore/sqliteonlinestore_test.go index a55c2a6df95..e5e6e85e56c 100644 --- a/go/internal/feast/onlinestore/sqliteonlinestore_test.go +++ b/go/internal/feast/onlinestore/sqliteonlinestore_test.go @@ -37,9 +37,9 @@ func TestSqliteOnlineRead(t *testing.T) { dir := t.TempDir() feature_repo_path := filepath.Join(dir, "feature_repo") test.SetupCleanFeatureRepo(dir) - config, err := registry.NewRepoConfigFromFile(feature_repo_path) assert.Nil(t, err) + store, err := NewSqliteOnlineStore("feature_repo", config, config.OnlineStore) defer store.Destruct() assert.Nil(t, err) @@ -51,7 +51,7 @@ func TestSqliteOnlineRead(t *testing.T) { JoinKeys: []string{"driver_id"}, EntityValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}}, } -entity_key3 := types.EntityKey{ + entity_key3 := types.EntityKey{ JoinKeys: []string{"driver_id"}, EntityValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1003}}}, } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index d4ac23a5210..275edc7b98e 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -89,14 +89,12 @@ func GetLatestFeatures(Rows []*Row, entities map[int64]bool) map[int64]*Row { func SetupCleanFeatureRepo(basePath string) error { cmd := exec.Command("feast", "init", "feature_repo") - path, err := filepath.Abs(basePath) cmd.Env = os.Environ() if err != nil { return err } - cmd.Dir = path err = cmd.Run() if err != nil {