From 7534bfb941e535be80300f955168eb3d12b64608 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 20 Sep 2021 09:38:14 -0700 Subject: [PATCH 01/39] Update feast dep to 0.12 Signed-off-by: Achal Shah --- deps/feast | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/feast b/deps/feast index 8010d2f..db43faf 160000 --- a/deps/feast +++ b/deps/feast @@ -1 +1 @@ -Subproject commit 8010d2f35d3f876db54a31b8012b13009cd5eba2 +Subproject commit db43faf7bd1385eb46f8e489766f6609abf753b9 From cd0f2b9b22c29d70f249266c62d6f020185b1c2e Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 21 Sep 2021 23:01:29 -0700 Subject: [PATCH 02/39] Port feast 0.10+ data model to feast-serving Signed-off-by: Achal Shah --- .../feast/serving/config/FeastProperties.java | 11 ++- .../config/ServingServiceConfigV2.java | 39 +++++++-- .../serving/config/SpecServiceConfig.java | 3 + .../serving/registry/LocalRegistryRepo.java | 77 ++++++++++++++++++ .../serving/registry/RegistryRepository.java | 32 ++++++++ .../service/OnlineServingServiceV2.java | 16 ++-- .../specs/CoreFeatureSpecRetriever.java | 48 +++++++++++ .../serving/specs/FeatureSpecRetriever.java | 33 ++++++++ .../specs/RegistryFeatureSpecRetriever.java | 68 ++++++++++++++++ serving/src/main/resources/application.yml | 4 +- .../service/OnlineServingServiceTest.java | 3 +- .../redis/common/RedisHashDecoder.java | 4 +- .../redis/retriever/EntityKeySerializer.java | 24 ++++++ .../retriever/EntityKeySerializerV2.java | 81 +++++++++++++++++++ .../redis/retriever/OnlineRetriever.java | 27 +++++-- 15 files changed, 445 insertions(+), 25 deletions(-) create mode 100644 serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java create mode 100644 serving/src/main/java/feast/serving/registry/RegistryRepository.java create mode 100644 serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java create mode 100644 serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java create mode 100644 serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java create mode 100644 storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java create mode 100644 storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 1b62d84..9a60923 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -72,6 +72,16 @@ public FeastProperties() {} /* Feast Core port to connect to. */ @Positive private int coreGrpcPort; + private String registry; + + public String getRegistry() { + return registry; + } + + public void setRegistry(final String registry) { + this.registry = registry; + } + private CoreAuthenticationProperties coreAuthentication; public CoreAuthenticationProperties getCoreAuthentication() { @@ -82,7 +92,6 @@ public void setCoreAuthentication(CoreAuthenticationProperties coreAuthenticatio this.coreAuthentication = coreAuthentication; } - /* Feast Core port to connect to. */ @Positive private int coreCacheRefreshInterval; private SecurityProperties security; diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 9d50a6a..2733254 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -20,9 +20,13 @@ import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import feast.serving.registry.LocalRegistryRepo; import feast.serving.service.OnlineServingServiceV2; import feast.serving.service.ServingServiceV2; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; +import feast.serving.specs.FeatureSpecRetriever; +import feast.serving.specs.RegistryFeatureSpecRetriever; import feast.storage.api.retriever.OnlineRetrieverV2; import feast.storage.connectors.bigtable.retriever.BigTableOnlineRetriever; import feast.storage.connectors.bigtable.retriever.BigTableStoreConfig; @@ -32,6 +36,7 @@ import io.opentracing.Tracer; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -69,22 +74,27 @@ public ServingServiceV2 servingServiceV2( ServingServiceV2 servingService = null; FeastProperties.Store store = feastProperties.getActiveStore(); + OnlineRetrieverV2 retrieverV2; + log.info("Online Store Type: {}", store.getType()); switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = RedisClusterClient.create(store.getRedisClusterConfig()); - OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient); - servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer); + retrieverV2 = new OnlineRetriever(redisClusterClient, (e -> e.toByteArray())); break; case REDIS: RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); - OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient); - servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer); + final EntityKeySerializer serializer; + if (feastProperties.getRegistry() != null) { + serializer = (e -> e.toByteArray()); + } else { + serializer = new EntityKeySerializerV2(); + } + retrieverV2 = new OnlineRetriever(redisClient, serializer); break; case BIGTABLE: BigtableDataClient bigtableClient = context.getBean(BigtableDataClient.class); - OnlineRetrieverV2 bigtableRetriever = new BigTableOnlineRetriever(bigtableClient); - servingService = new OnlineServingServiceV2(bigtableRetriever, specService, tracer); + retrieverV2 = new BigTableOnlineRetriever(bigtableClient); break; case CASSANDRA: CassandraStoreConfig config = feastProperties.getActiveStore().getCassandraConfig(); @@ -109,11 +119,24 @@ public ServingServiceV2 servingServiceV2( .withLocalDatacenter(dataCenter) .withKeyspace(keySpace) .build(); - OnlineRetrieverV2 cassandraRetriever = new CassandraOnlineRetriever(session); - servingService = new OnlineServingServiceV2(cassandraRetriever, specService, tracer); + retrieverV2 = new CassandraOnlineRetriever(session); break; + default: + throw new RuntimeException( + String.format("Unable to identify online store type: %s", store.getType())); } + final FeatureSpecRetriever featureSpecRetriever; + if (feastProperties.getRegistry() != null) { + final LocalRegistryRepo repo = + new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); + featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); + } else { + featureSpecRetriever = new CoreFeatureSpecRetriever(specService); + } + + servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever); + return servingService; } } diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java index 369d543..d640bf9 100644 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -45,6 +46,7 @@ public SpecServiceConfig(FeastProperties feastProperties) { this.feastCachedSpecServiceRefreshInterval = feastProperties.getCoreCacheRefreshInterval(); } + @ConditionalOnProperty(name = "feast.registry", matchIfMissing = true) @Bean public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( CachedSpecService cachedSpecStorage) { @@ -59,6 +61,7 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( return scheduledExecutorService; } + @ConditionalOnProperty(name = "feast.registry", matchIfMissing = true) @Bean public CachedSpecService specService(ObjectProvider callCredentials) throws InvalidProtocolBufferException, JsonProcessingException { diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java new file mode 100644 index 0000000..27d1576 --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class LocalRegistryRepo implements RegistryRepository { + + private final Path localRegistryPath; + + public LocalRegistryRepo(Path localRegistryPath) { + this.localRegistryPath = localRegistryPath; + } + + @Override + public RegistryProto.Registry getRegistry() { + try { + final byte[] registryContents = Files.readAllBytes(this.localRegistryPath); + + return RegistryProto.Registry.parseFrom(registryContents); + + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final FeatureViewProto.FeatureViewSpec spec = + this.getFeatureViewSpec(projectName, featureReference); + for (final FeatureProto.FeatureSpecV2 featureSpec : spec.getFeaturesList()) { + if (featureSpec.getName().equals(featureReference.getName())) { + return featureSpec; + } + } + + throw new SpecRetrievalException( + String.format( + "Unable to find feature with name: %s in feature view: %s", + featureReference.getName(), featureReference.getFeatureTable())); + } +} diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java new file mode 100644 index 0000000..776a7bf --- /dev/null +++ b/serving/src/main/java/feast/serving/registry/RegistryRepository.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; + +public interface RegistryRepository { + RegistryProto.Registry getRegistry(); + + FeatureViewProto.FeatureViewSpec getFeatureViewSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index 1d35edd..a9a1b8f 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -28,7 +28,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.types.ValueProto; import feast.serving.exception.SpecRetrievalException; -import feast.serving.specs.CachedSpecService; +import feast.serving.specs.FeatureSpecRetriever; import feast.serving.util.Metrics; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.OnlineRetrieverV2; @@ -45,15 +45,15 @@ public class OnlineServingServiceV2 implements ServingServiceV2 { private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class); - private final CachedSpecService specService; private final Tracer tracer; private final OnlineRetrieverV2 retriever; + private final FeatureSpecRetriever featureSpecRetriever; public OnlineServingServiceV2( - OnlineRetrieverV2 retriever, CachedSpecService specService, Tracer tracer) { + OnlineRetrieverV2 retriever, Tracer tracer, FeatureSpecRetriever featureSpecRetriever) { this.retriever = retriever; - this.specService = specService; this.tracer = tracer; + this.featureSpecRetriever = featureSpecRetriever; } /** {@inheritDoc} */ @@ -94,10 +94,10 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re .collect( Collectors.toMap( Function.identity(), - ref -> specService.getFeatureTableSpec(finalProjectName, ref).getMaxAge())); + ref -> this.featureSpecRetriever.getMaxAge(finalProjectName, ref))); List entityNames = featureReferences.stream() - .map(ref -> specService.getFeatureTableSpec(finalProjectName, ref).getEntitiesList()) + .map(ref -> this.featureSpecRetriever.getEntitiesList(finalProjectName, ref)) .findFirst() .get(); @@ -109,7 +109,9 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re Function.identity(), ref -> { try { - return specService.getFeatureSpec(finalProjectName, ref).getValueType(); + return this.featureSpecRetriever + .getFeatureSpec(finalProjectName, ref) + .getValueType(); } catch (SpecRetrievalException e) { return ValueProto.ValueType.Enum.INVALID; } diff --git a/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java new file mode 100644 index 0000000..fc24a10 --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/CoreFeatureSpecRetriever.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public class CoreFeatureSpecRetriever implements FeatureSpecRetriever { + private final CachedSpecService specService; + + public CoreFeatureSpecRetriever(CachedSpecService specService) { + this.specService = specService; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getMaxAge(); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureTableSpec(projectName, featureReference).getEntitiesList(); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.specService.getFeatureSpec(projectName, featureReference); + } +} diff --git a/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java new file mode 100644 index 0000000..91bc7fe --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/FeatureSpecRetriever.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.serving.ServingAPIProto; +import java.util.List; + +public interface FeatureSpecRetriever { + + Duration getMaxAge(String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); + + FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference); +} diff --git a/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java new file mode 100644 index 0000000..24026b1 --- /dev/null +++ b/serving/src/main/java/feast/serving/specs/RegistryFeatureSpecRetriever.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.specs; + +import com.google.protobuf.Duration; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureViewProto; +import feast.proto.core.RegistryProto; +import feast.proto.serving.ServingAPIProto; +import feast.serving.exception.SpecRetrievalException; +import feast.serving.registry.RegistryRepository; +import java.util.List; + +public class RegistryFeatureSpecRetriever implements FeatureSpecRetriever { + private final RegistryRepository registryRepository; + + public RegistryFeatureSpecRetriever(RegistryRepository registryRepository) { + this.registryRepository = registryRepository; + } + + @Override + public Duration getMaxAge( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getTtl(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public List getEntitiesList( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + final RegistryProto.Registry registry = this.registryRepository.getRegistry(); + for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) { + if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) { + return featureView.getSpec().getEntitiesList(); + } + } + throw new SpecRetrievalException( + String.format( + "Unable to find feature view with name: %s", featureReference.getFeatureTable())); + } + + @Override + public FeatureProto.FeatureSpecV2 getFeatureSpec( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + return this.registryRepository.getFeatureSpec(projectName, featureReference); + } +} diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index f8187e9..0db38cb 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -3,7 +3,9 @@ feast: # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) core-host: ${FEAST_CORE_HOST:localhost} core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - + + registry: "/Users/achal/tecton/feast/prompt_dory/data/registry.db" + core-authentication: enabled: false # should be set to true if authentication is enabled on core. provider: google # can be set to `oauth` or `google` diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 83dbdf0..17f6d04 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -61,7 +61,8 @@ public class OnlineServingServiceTest { @Before public void setUp() { initMocks(this); - onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); + onlineServingServiceV2 = + new OnlineServingServiceV2(retrieverV2, specService, tracer, featureSpecRetriever); mockedFeatureRows = new ArrayList<>(); mockedFeatureRows.add( diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index f78e22d..ce7d200 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -39,7 +39,7 @@ public class RedisHashDecoder { */ public static List retrieveFeature( List> redisHashValues, - Map byteToFeatureReferenceMap, + Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { List allFeatures = new ArrayList<>(); @@ -57,7 +57,7 @@ public static List retrieveFeature( featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); } else { ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK.toString()); + byteToFeatureReferenceMap.get(redisValueK); ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); featureMap.put(featureReference, featureValue); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java new file mode 100644 index 0000000..6220dd2 --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import feast.proto.storage.RedisProto; + +@FunctionalInterface +public interface EntityKeySerializer { + byte[] serialize(final RedisProto.RedisKeyV2 entityKey); +} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java new file mode 100644 index 0000000..7717253 --- /dev/null +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.storage.connectors.redis.retriever; + +import com.google.common.primitives.UnsignedBytes; +import com.google.protobuf.ProtocolStringList; +import feast.proto.storage.RedisProto; +import feast.proto.types.ValueProto; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; + +public class EntityKeySerializerV2 implements EntityKeySerializer { + @Override + public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { + final ProtocolStringList joinKeys = entityKey.getEntityNamesList(); + final List values = entityKey.getEntityValuesList(); + + assert joinKeys.size() == values.size(); + + final List buffer = new ArrayList<>(); + + final List> tuples = new ArrayList<>(joinKeys.size()); + for (int i = 0; i < joinKeys.size(); i++) { + tuples.add(Pair.of(joinKeys.get(i), values.get(i))); + } + tuples.sort(Comparator.comparing(Pair::getLeft)); + for (Pair pair : tuples) { + for (final byte b : pair.getLeft().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + } + + for (Pair pair : tuples) { + final ValueProto.Value val = pair.getRight(); + switch (val.getValCase()) { + case STRING_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.STRING.getNumber())); + buffer.add( + UnsignedBytes.checkedCast( + val.getStringVal().getBytes(StandardCharsets.UTF_8).length)); + for (final byte b : val.getStringVal().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + break; + case BYTES_VAL: + case INT32_VAL: + case INT64_VAL: + break; + default: + break; + } + } + for (final byte b : entityKey.getProject().getBytes(StandardCharsets.UTF_8)) { + buffer.add(b); + } + + final byte[] bytes = new byte[buffer.size()]; + for (int i = 0; i < buffer.size(); i++) { + bytes[i] = buffer.get(i); + } + + return bytes; + } +} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java index 073b6b4..9b11d0e 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java @@ -30,14 +30,18 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import org.slf4j.Logger; public class OnlineRetriever implements OnlineRetrieverV2 { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineRetriever.class); private static final String timestampPrefix = "_ts"; - private RedisClientAdapter redisClientAdapter; + private final RedisClientAdapter redisClientAdapter; + private final EntityKeySerializer keySerializer; - public OnlineRetriever(RedisClientAdapter redisClientAdapter) { + public OnlineRetriever(RedisClientAdapter redisClientAdapter, EntityKeySerializer keySerializer) { this.redisClientAdapter = redisClientAdapter; + this.keySerializer = keySerializer; } @Override @@ -58,11 +62,11 @@ private List> getFeaturesFromRedis( List featureReferences) { List> features = new ArrayList<>(); // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); + Map byteToFeatureReferenceMap = new HashMap<>(); // Serialize using proto List binaryRedisKeys = - redisKeys.stream().map(redisKey -> redisKey.toByteArray()).collect(Collectors.toList()); + redisKeys.stream().map(this.keySerializer::serialize).collect(Collectors.toList()); List featureReferenceWithTsByteList = new ArrayList<>(); featureReferences.stream() @@ -73,7 +77,7 @@ private List> getFeaturesFromRedis( byte[] featureReferenceBytes = RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); + byteToFeatureReferenceMap.put(featureReferenceBytes, featureReference); // eg. <_ts:featuretable_name> byte[] featureTableTsBytes = @@ -93,13 +97,26 @@ private List> getFeaturesFromRedis( // Write all commands to the transport layer redisClientAdapter.flushCommands(); + for (final Map.Entry entry : + byteToFeatureReferenceMap.entrySet()) { + log.info("Map Entry Key: {} Value: {}", entry.getKey(), entry.getValue()); + } + futures.forEach( future -> { try { List> redisValuesList = future.get(); + log.info("Redis values list: {}", redisValuesList); + for (KeyValue keyValue : redisValuesList) { + log.info("Key: {}, Value: {}", keyValue.getKey(), keyValue.getValue()); + } + List curRedisKeyFeatures = RedisHashDecoder.retrieveFeature( redisValuesList, byteToFeatureReferenceMap, timestampPrefix); + for (final Feature feature : curRedisKeyFeatures) { + log.info("Found features: {}", feature); + } features.add(curRedisKeyFeatures); } catch (InterruptedException | ExecutionException | InvalidProtocolBufferException e) { throw Status.UNKNOWN From 3bef6ec1594fda2fc110dbec4102826101bea9c1 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 21 Sep 2021 23:11:28 -0700 Subject: [PATCH 03/39] Fix tests Signed-off-by: Achal Shah --- .../java/feast/serving/service/OnlineServingServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 17f6d04..0f260b9 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -34,6 +34,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; +import feast.serving.specs.CoreFeatureSpecRetriever; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.ProtoFeature; import feast.storage.connectors.redis.retriever.OnlineRetriever; @@ -62,7 +63,7 @@ public class OnlineServingServiceTest { public void setUp() { initMocks(this); onlineServingServiceV2 = - new OnlineServingServiceV2(retrieverV2, specService, tracer, featureSpecRetriever); + new OnlineServingServiceV2(retrieverV2, tracer, new CoreFeatureSpecRetriever(specService)); mockedFeatureRows = new ArrayList<>(); mockedFeatureRows.add( From c68fb3d1218a623b95be6a4539071302449c3d50 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Sep 2021 09:37:39 -0700 Subject: [PATCH 04/39] Fix integ tests Signed-off-by: Achal Shah --- serving/src/main/resources/application.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 0db38cb..3e4e07b 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -4,8 +4,6 @@ feast: core-host: ${FEAST_CORE_HOST:localhost} core-grpc-port: ${FEAST_CORE_GRPC_PORT:6565} - registry: "/Users/achal/tecton/feast/prompt_dory/data/registry.db" - core-authentication: enabled: false # should be set to true if authentication is enabled on core. provider: google # can be set to `oauth` or `google` From 42efa34a78e7cb4e51b6c93f2e9811ebf5539b9c Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Sep 2021 09:58:06 -0700 Subject: [PATCH 05/39] Fix integ tests Signed-off-by: Achal Shah --- .../serving/config/ServingServiceConfigV2.java | 7 +++---- .../redis/common/RedisHashDecoder.java | 4 ++-- .../redis/retriever/OnlineRetriever.java | 18 ++---------------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 2733254..7625378 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -20,6 +20,7 @@ import com.datastax.oss.driver.api.core.CqlSessionBuilder; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; +import com.google.protobuf.AbstractMessageLite; import feast.serving.registry.LocalRegistryRepo; import feast.serving.service.OnlineServingServiceV2; import feast.serving.service.ServingServiceV2; @@ -49,7 +50,6 @@ @Configuration public class ServingServiceConfigV2 { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); @Autowired private ApplicationContext context; @@ -75,18 +75,17 @@ public ServingServiceV2 servingServiceV2( FeastProperties.Store store = feastProperties.getActiveStore(); OnlineRetrieverV2 retrieverV2; - log.info("Online Store Type: {}", store.getType()); switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = RedisClusterClient.create(store.getRedisClusterConfig()); - retrieverV2 = new OnlineRetriever(redisClusterClient, (e -> e.toByteArray())); + retrieverV2 = new OnlineRetriever(redisClusterClient, (AbstractMessageLite::toByteArray)); break; case REDIS: RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); final EntityKeySerializer serializer; if (feastProperties.getRegistry() != null) { - serializer = (e -> e.toByteArray()); + serializer = (AbstractMessageLite::toByteArray); } else { serializer = new EntityKeySerializerV2(); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index ce7d200..f78e22d 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -39,7 +39,7 @@ public class RedisHashDecoder { */ public static List retrieveFeature( List> redisHashValues, - Map byteToFeatureReferenceMap, + Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { List allFeatures = new ArrayList<>(); @@ -57,7 +57,7 @@ public static List retrieveFeature( featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); } else { ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK); + byteToFeatureReferenceMap.get(redisValueK.toString()); ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); featureMap.put(featureReference, featureValue); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java index 9b11d0e..e68b679 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java @@ -33,8 +33,6 @@ import org.slf4j.Logger; public class OnlineRetriever implements OnlineRetrieverV2 { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineRetriever.class); - private static final String timestampPrefix = "_ts"; private final RedisClientAdapter redisClientAdapter; private final EntityKeySerializer keySerializer; @@ -62,7 +60,7 @@ private List> getFeaturesFromRedis( List featureReferences) { List> features = new ArrayList<>(); // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); + Map byteToFeatureReferenceMap = new HashMap<>(); // Serialize using proto List binaryRedisKeys = @@ -77,7 +75,7 @@ private List> getFeaturesFromRedis( byte[] featureReferenceBytes = RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes, featureReference); + byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); // eg. <_ts:featuretable_name> byte[] featureTableTsBytes = @@ -97,26 +95,14 @@ private List> getFeaturesFromRedis( // Write all commands to the transport layer redisClientAdapter.flushCommands(); - for (final Map.Entry entry : - byteToFeatureReferenceMap.entrySet()) { - log.info("Map Entry Key: {} Value: {}", entry.getKey(), entry.getValue()); - } - futures.forEach( future -> { try { List> redisValuesList = future.get(); - log.info("Redis values list: {}", redisValuesList); - for (KeyValue keyValue : redisValuesList) { - log.info("Key: {}, Value: {}", keyValue.getKey(), keyValue.getValue()); - } List curRedisKeyFeatures = RedisHashDecoder.retrieveFeature( redisValuesList, byteToFeatureReferenceMap, timestampPrefix); - for (final Feature feature : curRedisKeyFeatures) { - log.info("Found features: {}", feature); - } features.add(curRedisKeyFeatures); } catch (InterruptedException | ExecutionException | InvalidProtocolBufferException e) { throw Status.UNKNOWN From 265b98493389db0442586239f8d2f94eba9ec8d4 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Sep 2021 10:07:40 -0700 Subject: [PATCH 06/39] remove logging Signed-off-by: Achal Shah --- .../main/java/feast/serving/config/ServingServiceConfigV2.java | 1 - 1 file changed, 1 deletion(-) diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 7625378..97a13f3 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -41,7 +41,6 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; From 7f779ce03865d2b07077478b25f1b3ca44dd36ca Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Sep 2021 10:19:06 -0700 Subject: [PATCH 07/39] Fix ilnt Signed-off-by: Achal Shah --- .../storage/connectors/redis/retriever/OnlineRetriever.java | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java index e68b679..bd682fe 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java @@ -30,7 +30,6 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -import org.slf4j.Logger; public class OnlineRetriever implements OnlineRetrieverV2 { private static final String timestampPrefix = "_ts"; From f4d68fbd6cda56d2781ce6897bf4c6fb684df12d Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 22 Sep 2021 13:22:14 -0700 Subject: [PATCH 08/39] Fix serialization Signed-off-by: Achal Shah --- .../java/feast/serving/config/ServingServiceConfigV2.java | 4 ++-- .../src/main/java/feast/serving/config/SpecServiceConfig.java | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 97a13f3..b244527 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -84,9 +84,9 @@ public ServingServiceV2 servingServiceV2( RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); final EntityKeySerializer serializer; if (feastProperties.getRegistry() != null) { - serializer = (AbstractMessageLite::toByteArray); - } else { serializer = new EntityKeySerializerV2(); + } else { + serializer = (AbstractMessageLite::toByteArray); } retrieverV2 = new OnlineRetriever(redisClient, serializer); break; diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java index d640bf9..369d543 100644 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -46,7 +45,6 @@ public SpecServiceConfig(FeastProperties feastProperties) { this.feastCachedSpecServiceRefreshInterval = feastProperties.getCoreCacheRefreshInterval(); } - @ConditionalOnProperty(name = "feast.registry", matchIfMissing = true) @Bean public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( CachedSpecService cachedSpecStorage) { @@ -61,7 +59,6 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( return scheduledExecutorService; } - @ConditionalOnProperty(name = "feast.registry", matchIfMissing = true) @Bean public CachedSpecService specService(ObjectProvider callCredentials) throws InvalidProtocolBufferException, JsonProcessingException { From 34ec33c2174cb2dd9f35fb0b937b3a11f0cac806 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Fri, 24 Sep 2021 17:09:55 -0700 Subject: [PATCH 09/39] Implement EntityKeySerialization correctly Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 14 +++++ .../config/ServingServiceConfigV2.java | 6 ++ .../test/resources/feast_project/example.py | 56 +++++++++++++++++++ .../feast_project/feature_store.yaml | 7 +++ .../redis/common/RedisHashDecoder.java | 7 ++- .../retriever/EntityKeySerializerV2.java | 41 +++++++++++++- .../redis/retriever/OnlineRetriever.java | 8 ++- 7 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 serving/src/test/resources/feast_project/example.py create mode 100644 serving/src/test/resources/feast_project/feature_store.yaml diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 7f6c3fe..5492697 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -42,6 +42,16 @@ jobs: integration-test: runs-on: ubuntu-latest needs: unit-test-java + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v2 with: @@ -56,6 +66,10 @@ jobs: with: python-version: '3.6' architecture: 'x64' + - name: Install Feast CLI + run: pip install -e "deps/feast/sdk/python[ci]" + - name: Apply and Materialize + run: feast init - uses: actions/cache@v2 with: path: ~/.m2/repository diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index b244527..7410fc6 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -49,6 +50,7 @@ @Configuration public class ServingServiceConfigV2 { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class); @Autowired private ApplicationContext context; @@ -84,8 +86,10 @@ public ServingServiceV2 servingServiceV2( RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); final EntityKeySerializer serializer; if (feastProperties.getRegistry() != null) { + log.info("Created EntityKeySerializerV2"); serializer = new EntityKeySerializerV2(); } else { + log.info("Using byteArray method"); serializer = (AbstractMessageLite::toByteArray); } retrieverV2 = new OnlineRetriever(redisClient, serializer); @@ -126,10 +130,12 @@ public ServingServiceV2 servingServiceV2( final FeatureSpecRetriever featureSpecRetriever; if (feastProperties.getRegistry() != null) { + log.info("Created RegistryFeatureSpecRetriever"); final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); } else { + log.info("Created CoreFeatureSpecRetriever"); featureSpecRetriever = new CoreFeatureSpecRetriever(specService); } diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py new file mode 100644 index 0000000..74e5d9a --- /dev/null +++ b/serving/src/test/resources/feast_project/example.py @@ -0,0 +1,56 @@ +# This is an example feature definition file + +from google.protobuf.duration_pb2 import Duration +import pandas as pd + +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, OnDemandFeatureView + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +driver_hourly_stats = FileSource( + path="/Users/achal/tecton/feast/prompt_dory/data/driver_stats.parquet", + event_timestamp_column="event_timestamp", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=Duration(seconds=86400 * 365), + features=[ + Feature(name="conv_rate", dtype=ValueType.DOUBLE), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), + ], + online=True, + batch_source=driver_hourly_stats, + tags={}, +) + +fs = FeatureService( + name="driver_hourly_stats_feature_service", + features=[driver_hourly_stats_view] +) + + +def conv_rate_plus_100(driver_hourly_stats: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_100"] = driver_hourly_stats["conv_rate"] + 100 + return df + + +odfv = OnDemandFeatureView( + name=conv_rate_plus_100.__name__, + inputs={"driver": driver_hourly_stats_view}, + # features=[Feature("conv_rate_plus_100", ValueType.FLOAT)], + features=[], + udf=conv_rate_plus_100, +) diff --git a/serving/src/test/resources/feast_project/feature_store.yaml b/serving/src/test/resources/feast_project/feature_store.yaml new file mode 100644 index 0000000..145a049 --- /dev/null +++ b/serving/src/test/resources/feast_project/feature_store.yaml @@ -0,0 +1,7 @@ +project: prompt_dory +provider: local +online_store: redis +offline_store: {} +flags: + alpha_features: true + on_demand_transforms: true diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index f78e22d..bb09c75 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -26,9 +26,12 @@ import io.lettuce.core.KeyValue; import java.nio.charset.StandardCharsets; import java.util.*; +import org.slf4j.Logger; public class RedisHashDecoder { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisHashDecoder.class); + /** * Converts all retrieved Redis Hash values based on EntityRows into {@link Feature} * @@ -39,7 +42,7 @@ public class RedisHashDecoder { */ public static List retrieveFeature( List> redisHashValues, - Map byteToFeatureReferenceMap, + Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { List allFeatures = new ArrayList<>(); @@ -57,7 +60,7 @@ public static List retrieveFeature( featureTableTimestampMap.put(new String(redisValueK), eventTimestamp); } else { ServingAPIProto.FeatureReferenceV2 featureReference = - byteToFeatureReferenceMap.get(redisValueK.toString()); + byteToFeatureReferenceMap.get(redisValueK); ValueProto.Value featureValue = ValueProto.Value.parseFrom(redisValueV); featureMap.put(featureReference, featureValue); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java index 7717253..e51efb0 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java @@ -20,6 +20,8 @@ import com.google.protobuf.ProtocolStringList; import feast.proto.storage.RedisProto; import feast.proto.types.ValueProto; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Comparator; @@ -27,6 +29,7 @@ import org.apache.commons.lang3.tuple.Pair; public class EntityKeySerializerV2 implements EntityKeySerializer { + @Override public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { final ProtocolStringList joinKeys = entityKey.getEntityNamesList(); @@ -41,7 +44,15 @@ public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { tuples.add(Pair.of(joinKeys.get(i), values.get(i))); } tuples.sort(Comparator.comparing(Pair::getLeft)); + + ByteBuffer stringBytes = ByteBuffer.allocate(Integer.BYTES); + stringBytes.order(ByteOrder.LITTLE_ENDIAN); + stringBytes.putInt(ValueProto.ValueType.Enum.STRING.getNumber()); + for (Pair pair : tuples) { + for (final byte b : stringBytes.array()) { + buffer.add(b); + } for (final byte b : pair.getLeft().getBytes(StandardCharsets.UTF_8)) { buffer.add(b); } @@ -60,11 +71,39 @@ public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { } break; case BYTES_VAL: + buffer.add(UnsignedBytes.checkedCast(ValueProto.ValueType.Enum.BYTES.getNumber())); + for (final byte b : val.getBytesVal().toByteArray()) { + buffer.add(b); + } + break; case INT32_VAL: + ByteBuffer int32ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int32ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int32ByteBuffer.putInt(ValueProto.ValueType.Enum.INT32.getNumber()); + int32ByteBuffer.putInt(Integer.BYTES); + int32ByteBuffer.putInt(val.getInt32Val()); + for (final byte b : int32ByteBuffer.array()) { + buffer.add(b); + } + break; case INT64_VAL: + ByteBuffer int64ByteBuffer = + ByteBuffer.allocate(Integer.BYTES + Integer.BYTES + Integer.BYTES); + int64ByteBuffer.order(ByteOrder.LITTLE_ENDIAN); + int64ByteBuffer.putInt(ValueProto.ValueType.Enum.INT64.getNumber()); + int64ByteBuffer.putInt(Integer.BYTES); + /* This is super dumb - but in https://github.com/feast-dev/feast/blob/dcae1606f53028ce5413567fb8b66f92cfef0f8e/sdk/python/feast/infra/key_encoding_utils.py#L9 + we use `struct.pack("> getFeaturesFromRedis( List featureReferences) { List> features = new ArrayList<>(); // To decode bytes back to Feature Reference - Map byteToFeatureReferenceMap = new HashMap<>(); + Map byteToFeatureReferenceMap = new HashMap<>(); // Serialize using proto List binaryRedisKeys = @@ -74,7 +78,7 @@ private List> getFeaturesFromRedis( byte[] featureReferenceBytes = RedisHashDecoder.getFeatureReferenceRedisHashKeyBytes(featureReference); featureReferenceWithTsByteList.add(featureReferenceBytes); - byteToFeatureReferenceMap.put(featureReferenceBytes.toString(), featureReference); + byteToFeatureReferenceMap.put(featureReferenceBytes, featureReference); // eg. <_ts:featuretable_name> byte[] featureTableTsBytes = From 676d57e3e28e74f901eeebc73235c0eebd1a870e Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Fri, 24 Sep 2021 20:08:20 -0700 Subject: [PATCH 10/39] Update workflows Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 5492697..36f8e9a 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -67,9 +67,7 @@ jobs: python-version: '3.6' architecture: 'x64' - name: Install Feast CLI - run: pip install -e "deps/feast/sdk/python[ci]" - - name: Apply and Materialize - run: feast init + run: pip install -U pip; pip install -e "deps/feast/sdk/python" - uses: actions/cache@v2 with: path: ~/.m2/repository From e0edb68948c504b3d4e1fdfb499f2200a709079d Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Fri, 24 Sep 2021 20:13:43 -0700 Subject: [PATCH 11/39] Update python version Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 36f8e9a..5d75e73 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -64,7 +64,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.7' architecture: 'x64' - name: Install Feast CLI run: pip install -U pip; pip install -e "deps/feast/sdk/python" From 771df7bcdbeb417a0f2eb05533ee4e38ce5e578f Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Sun, 26 Sep 2021 09:59:52 -0700 Subject: [PATCH 12/39] Change redis ports Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 5d75e73..6a7943f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -46,7 +46,7 @@ jobs: redis: image: redis ports: - - 6379:6379 + - 6389:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -68,6 +68,11 @@ jobs: architecture: 'x64' - name: Install Feast CLI run: pip install -U pip; pip install -e "deps/feast/sdk/python" + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + # - name: Apply and Materialize + # run: feast --chdir ./serving/src/test/resources/feast_project/example.py apply; feast --chdir ./serving/src/test/resources/feast_project/example.py materialize - uses: actions/cache@v2 with: path: ~/.m2/repository From 42d71476f55ed02d6dbd800181d3d0bddfb20221 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 12:06:55 -0700 Subject: [PATCH 13/39] materialize into redis Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 4 ++-- serving/src/test/resources/feast_project/feature_store.yaml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 6a7943f..cf9c3dd 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -71,8 +71,8 @@ jobs: - name: Get current date id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d')" - # - name: Apply and Materialize - # run: feast --chdir ./serving/src/test/resources/feast_project/example.py apply; feast --chdir ./serving/src/test/resources/feast_project/example.py materialize + - name: Apply and Materialize + run: feast --chdir ./serving/src/test/resources/feast_project/example.py apply; feast --chdir ./serving/src/test/resources/feast_project/example.py materialize-incremental ${{steps.date.outputs.date}} - uses: actions/cache@v2 with: path: ~/.m2/repository diff --git a/serving/src/test/resources/feast_project/feature_store.yaml b/serving/src/test/resources/feast_project/feature_store.yaml index 145a049..29e8b53 100644 --- a/serving/src/test/resources/feast_project/feature_store.yaml +++ b/serving/src/test/resources/feast_project/feature_store.yaml @@ -1,6 +1,8 @@ project: prompt_dory provider: local -online_store: redis +online_store: + type: redis + connection_string: localhost:6389 offline_store: {} flags: alpha_features: true From 087f4e65aed401bfb491fb091175afe1708113a3 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 13:35:35 -0700 Subject: [PATCH 14/39] fix path Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index cf9c3dd..32d2524 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -72,7 +72,7 @@ jobs: id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d')" - name: Apply and Materialize - run: feast --chdir ./serving/src/test/resources/feast_project/example.py apply; feast --chdir ./serving/src/test/resources/feast_project/example.py materialize-incremental ${{steps.date.outputs.date}} + run: feast --chdir ./serving/src/test/resources/feast_project apply; feast --chdir ./serving/src/test/resources/feast_project materialize-incremental ${{steps.date.outputs.date}} - uses: actions/cache@v2 with: path: ~/.m2/repository From 702094eb77572f4b881e2cae844cc84471d147b4 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 13:43:12 -0700 Subject: [PATCH 15/39] Install redis vairant Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 32d2524..aff0b4d 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -67,7 +67,7 @@ jobs: python-version: '3.7' architecture: 'x64' - name: Install Feast CLI - run: pip install -U pip; pip install -e "deps/feast/sdk/python" + run: pip install -U pip; pip install -e "deps/feast/sdk/python[redis]" - name: Get current date id: date run: echo "::set-output name=date::$(date +'%Y-%m-%d')" From 83039f8e82b632e4ccb12354e6f02b6633e765c9 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 13:51:13 -0700 Subject: [PATCH 16/39] Remove odfv Signed-off-by: Achal Shah --- serving/src/test/resources/feast_project/example.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 74e5d9a..91889d5 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -3,7 +3,7 @@ from google.protobuf.duration_pb2 import Duration import pandas as pd -from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, OnDemandFeatureView +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation @@ -46,11 +46,3 @@ def conv_rate_plus_100(driver_hourly_stats: pd.DataFrame) -> pd.DataFrame: df["conv_rate_plus_100"] = driver_hourly_stats["conv_rate"] + 100 return df - -odfv = OnDemandFeatureView( - name=conv_rate_plus_100.__name__, - inputs={"driver": driver_hourly_stats_view}, - # features=[Feature("conv_rate_plus_100", ValueType.FLOAT)], - features=[], - udf=conv_rate_plus_100, -) From 3f8d26e30461e8fb1afe9f4008fb90c64ceb8a79 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 13:54:05 -0700 Subject: [PATCH 17/39] Include test file Signed-off-by: Achal Shah --- .../feast_project/data/driver_stats.parquet | Bin 0 -> 34708 bytes .../src/test/resources/feast_project/example.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 serving/src/test/resources/feast_project/data/driver_stats.parquet diff --git a/serving/src/test/resources/feast_project/data/driver_stats.parquet b/serving/src/test/resources/feast_project/data/driver_stats.parquet new file mode 100644 index 0000000000000000000000000000000000000000..df8cbba388827fe2e312921abf21e6322b3b8e27 GIT binary patch literal 34708 zcmb5Uc|29o`}b|0i8513G9^jKaQ1bXXEJ3%fG~KKF{@$;eYhQa^%Sy*)ivW{A)+T{5X>)-G%mPd+ z>F9a|mFQ?iH{^Sm>FB05Wo@KqRB2jBt zj5C1;x5G{XwlkYt2x?WBb`jsIcY^p|cV2NNe!<72-6WzNb9Ez_5>(wo5MRUTPDELl zCgR}zroBWA@I3B8BK{wqo&-(lG%o@diB;Z2q-%IX#Ir`NK14KrlE05c?+)zuCCFmX z_ajir7xgFNnB2bsjl9*fpNJCHr3XmVITLh{z$er!fPjljGLVRMrI0*_Z%h2-LD=)piaLcmWY$NuZS2U+Y(1asg~?xBG@r4r1&Tb)M4^q8+iRN@~< zC*o+;nG6zzYK3MJh~Kk3PSEPUA&ZC}i$96TmipiX5o=eU&nA)8m7_TXU-j*C2@1v( z@(5`DjQPZ$`a4ei_?)_v#Fv#xDj?Bd^R7aI085o30{&@^Vj?yjc|k-M_NG%rq$|lR zA<YXL%91=N4M4vqi zMCAI`T}i~cc;*9Uz3q(x*bGDX53fU1C35GY= zTp|d%A$ysK!WMst*fRC7j)-nSwO2^Q#CEKnpt{)UDuMZCr3Qk9c9v_z&$WL_eD$|i z8;L&|m3o~-F?@TP2&B$!yFt*a&eKdp|G^m|^19u;NyLUPCt674nB;Sd;Fl=gCMdfk zaEFNcy6=cM_prT{i0S(Z+eoDJ`(Qi4XqI6IL8#Q~P9lmoeIa72S$`K1J)W0!lZfqb zNDo0Ri)Al?RgqL55x;KwH=qS=_xp)R+fhA0qN%y)L4x?m9YX}Nyb5;-1}hlu5kFwt zzX9bRxI9e!CfCIKBy#!e@_>LYaq9@dSrLv=A{t+uBH}yk>ko;THJb5=L@GWWj|s+p zYK{>^9_Je;qQv@t1KQbmdxD5Qrg=|D#P!T?lAtb7_bGuLv&b_dE){$v;wgph=S0+Q zJvBw58Qa4z2$El$OcN+XNW3KCFwegM4Jsd+A)>I_*;gcL=?kAFaNA}5ntBx+ST_>;h+*Weccn~V5wBG!KVOhl_=eSe7fRj}+YJ)1&7O)yMWABM&P~Lzndd}|3~l5gq6AmQ3KDgm-pfnivsIIifUAdh zB@yeKUlY;p!>v_BT#C)*C($VZKLG;m^E!eAGc;i#A|?-iAfkdtmoO2BzZZ*;C@Afa zD1q==6ET98`nBRjbTeEeBGbg+Y9dzeuUJDObNcYL1Pi%V5(K%jGV6$_-twD>liNon zi5T;;W<80dLSm%|dN~|75croWN)zxZGs_Uaq3a3p9i8fBiT~?evK)!ZVs^_D=<};> zB$%t>QXpcw)=MHP-Mg`gh@d$$oW-tx3MiDajbXb=n@G1nvrU|+9AME;V0 z6-1NLFpY>Vos|@c=p3Vf;Ox9D5*Qzq*Cu$kl3s`SS(X1P2o=psy2KwFiq|7iR zAmT#QzX~FkZ_u8eS6%&Fxx>Hnc=Gj%wmZyp0(1hu)(Rv_CMjE zXLoJkRqx?)lVZb*Mc4es>i0eAzF2(y;8c5xn9-$EHxAD|IKQ*!QpwGbh3URxqsyhY zB9=bCdD3(F^qr%0j2z;|b!BbG*jCBA^wyo}Na9^@ddm1pc~_e7R^Q3qD-}J*CG=9o zP3q6~<;dFBxb)SZ8#t-7xBryM)ykn_^}}x``>s~qJFR_;W3_2R_5BLt9Qj@S4d+Ly ztSU@POs~~Eyx@4%_i6vN3y&|mb)>E~Ypfl=>O;@Os}!S6&&bLvWmOlW!^pZqLO)9> zR+pJmNXcWkE>@3q#aiun-Ys$Z?5kv~&RSiGGvE~3?ADpJ<(MJ2*tUR~;VZ|CR;0h)lSZ~%IciWwbUIO9WJ|dWviyyZ`yf1;MId`X%3rr--%&frIzlv zWp7W`2HVDTr>)-i$_#SUGMrWY9@lw}G-mAFcJO&?!m4eVE*ghl51+HWp1Dgak6Y2(mR@S z_t{o$KVz7undiIXg2NTBhs}9@4wrYgB?@Te`#W9rzR$oW;YJtBz``S8T;@i9jD=^7 zcCy4Ch6D~lMYrCvJ&Z{_YaoVgojX&?N@>e7<1_BeX@ZKbt;y^5vSf&=9hmApvzPVw z8i=N2m-JveA!(Q?VN&kFo+E8ps+}U~$&n{-f61+{+>`U9qU-G#_Vr#|h00!c%SeJZnu^@6A)98T?_Yuflu9X^8$q$06mzdqy{rXPxO;AHE91%r!cxQu|h(GtF1r z(|>l~sw&G;h~?Pe%YWXc((;VyIbVSb_Lp4SQaAVsUUY6e@S^{mpU`F3+tKu#(*D9% z+`BW^nN|9WT=lwJs*@(YU-X*qqf2`RD))RA* z%%f{Cj!W*4RA09A_Hy&{hc*o4EADPjmpd#yRHSxrdhq;VnR}&hl!045NcMiY;c-ce znjpE6O3Tx_8S+QuAJ*7kb|0!avhndH*E@0C8-o?b>%H!kGxKV<(lIb|iEC$Rw=uAA zi_>Cwb=sLY1Qc9avvfLGcvjOQ>G*Uz*;Z~aKc#&_w~Iq?lS@kspI$eYsH*>D>j}Lc zo;6hDFS?cby}XhJ>Eb%s`h6>TeQ`9j*iwPCqhqb#p@-3dI9eCOj#VBMsseb6yV2R#I%cq*b?A`l$k&TssS_Um9kv zrlOfdY-WtJH&~SF7uvis&EK@^cD#t~ta*{@{%4(qwy!Nqsi;3pqIPpu4SK!WIww{|r7B{4q^2eiP$)1E>ZPR zx^;2!?ELbMO)=|^C31^vhAz=b#wYVhnHH$WN+zTVC^$ET(yvcU7g6yYYl>Z;l)0J~ zzO+Oym7KNCOg~jvBTgzMdxLqAdKklo)Lc2c8pj)P8`APOxip9VV31BP*y7P&pm9t( zqe#_%qA83~Ci9d=(A%*a$7GI|QjyC`KNw}R%5>s6L^R`NPm~*^i)w^3$z`84$(46% zj+e`+v^b?1w#+1-TWx*Tv`{laKCfm+t@Dj==8gHaP7S`}%?TS%UfR_X{$rV0p`dP0 zXR3%+qC#Q4=U|aW1k0wPhJB+oPB#-b6*ummY!3U$qIjw)aHhXdD@n1W`N+G88xgFV zOIyOez8$}rwE6Vys9(!JezGc+wZ<}Wiqevm&a@|RiD^c%Z7J_e;os=olDws&J40M6 z{1=<@+1?XUW<|6V<#YXc3Ok!4*|$~>7OMD7w4`jUx?4hvSpLPXQayY|KTVWMRXP9Q zoO!Wk6o+cf=y|&f&bLxkFFd;Fax?rlhg$8}6_0@;DlK1d6PakJ6QbF33%T&k?hG9NHJ`d1&`OkN2xDn7xag)eThA@3&@i1{3a zE+R2hM-~k?oZ&!iD^8G^c7{KTPI&w1X)V>ap6GH{5NEFJ!#(~QsG=c*=el#?`J>&~ zpK=9`Zi&P-2ZkZ2_zkG7ipF*ImiWuy9P~TvLff;^c=+5PwbfGy?|)WAokM-VQ%i^2 zj{0HqHfL zcrlj-=0&eS$>;+W9?FZ7f^OJtWQ=o9D=2Aebr1~_!$F%efP^uye*Y3e!x^FL77fq6 zG@*VNYv3lQGnCbhjkuu2fX8!Ou(B`$f5=;cZm&CJoABb579~7-K!mE54#3A3SRq4g z11`<5;fRGGiaonUjpTAs%hGw^F&Tv?`+mVQc7NQxFB|fI@4=sTF;KPFADfgpsN=lN zz#XcB>GT#LelZ$(TWxSQu@gia1@ZIUZ&31S0k{`EF~>6kMnr_M=0Y%zv@_%DG*J}k z4MEA7{rGB)7-nAfLDh<<)Z%MCI6Z8FfA_nP@4^~YIW^%mGY9T{>xE{!!l^$g zp@v0lP_~$xy0PXDoIGudyLuXd&b|%yT{{gedSWbN48?9XsM1*qTSFx>Ni|Wj)OfS$yA$|EM7J6 zN7p7cYUGG8Hs^n)==48p-HO_PW+P10-TA#}YcvQiSmuGrvKDSl??tAX6o?UB2h$QC zXxq#_fl@6a{s=4s)p7}Vb1n!^=e(gJSO=K(g8Ga%>ch9;G-VRTpmrJ2HDMUFgt{YFE++ovG< z_Ac;Jv%{nL<#7GUPdIWQi;`E^j6ZKg;A$;52x07obFK22^tcjadZRTzu9=6i>w$PR zdNXxldn-&kv7*nU4pQ-|cqhvXI`{3jd9m+}V z#Ngeh;Q87P2+g2JznxD2X3xSlb1@M4$U>?6J_EjM2ax~6G9}k*h#LC$sXZRWu*ypj zOS*>OQFArKKoF*~?!@Silkg+g1|3E>qW}IlXpiN@Aigd5drdpga%SO3x*#q`-3Ke< z?HE>*0MCtcpz%I8?s~?9lDxL~MTrTYx>Zuz+SlRfJSVcAzXF}rrubp(C~h6^gD6=M z^gMY9@+SbF9J&tEyA{y(n-BIq3cxM0ir8kIO4V(VMCr%-@phdUIz*XJ{Ec!Hza~G1 z-fM+d+@@H~?TI7wN06Dm51g7~pmgyGBrjY6Frq{5s(Ar)ZqEb4KnJG`P&NAKeDsa5KFxoZ?@N?Qg1K zUR6@|JhJ$BnHlr6P4LK$Er@sA zk#UbVdbP8o+0_V?Bi8`=vr)L}h7Ni-xFO>$U38C|2amD|@N%C54!Jcbn132RsQ-pT z_UvdXCXDstI@opb8~l3o5e~2v(s-A2G1<%lpIy~KUwHt=}F~9=pSxWL`FwRPv;SNc2G}@to3c4ZqKH?N~ zS1;1;TRY)oCKK3SwuDQGbdV;w9X@Ir(+cPkxwi9I!8eoXIJbu|b272q1amsKM)YfK#?9v{5cg_%EnJr+GNEBqM zt-`nDeAf2bgqz-9fM*l3U?vp@Uw3Z7Z$GwR!9_9#^#`!PDT+FGC(C%o5q1Y!7 zsg_1b@`+&5BPH7N-P_RQiU`UFZp6(ypTox|6|j-38lEb~P_q7Swczyz5S#PC8u}27 zt-B5fG?*!BuPt1NAAtFmdf2#H0UzI%!BRU_xcM^;M%_bkUz!-UKQzR!ToycH!hr&e z4)}{=!i^^)(R`8>*avpb<#!(Zy#2`i_0ql;Jz`<@~&`8S%A)LoFCh!5sfeSlg3G?c~kzM{i;ZfShMkW6hE zLFk&|K@~?j+*{-cEMbSiHP{eugFM`rRS*jiFm5w{Ui(=^2M&)5qMVrhthaIO51Zo8?y|h@s71CSosIz z{Elo|%LP$%Y0*aq`8+sqdnJB!$%b>}d{P$WM>XwuaP<{LVfPHk6wpSS7;PA4ItD-J z5-BDzf9yE+gt}(?1dVg~kUYY?rttS-QKolP2b4Bgf(_nHr93S3T zk8fS>!zQf^sz1vQHJqDZ%?D-}v13A432khA6c0-q_M?5r5jT!Id=?J8PUfLZvoP7kpbpo(9 zwt{pqg-Q(uc&Ew}*J@3J=#wT$IUR&L4?HRQ!a)2o?g3Z(TVVhFWw@g*j(1NBVCIH6 zDrwsgj2{og9ckYnLM0L0_mQ=OzO~@IygZpgzAr>AnhK6kLVHC zP3w{QWDn-<3V_Ms2rL}FM@3&1MX|D#IQLKq^Y7}RxfvtYCG+A+sk$=!`mFKl~&>IKqEXn?}6iMoc; zscr#=_Bv3US&f!sfSr?}6l=c&4vgM~(`OZN>y`;Hib6DTazWO#NYJ_b3GU|Kg|Xcm zu(*{4ulCxZDZeO{bAbz^cC|8H4(v2xM$rLwWtY3~TQF0p@$|7@aZ&z_|kl8g`>m@>(1<5TyF< zjYGI<9cUb03syn*XeO2`Fr|wJ_r)Z_%tj%+-M1ID%eO^gmBZLB1+nh4!sMaNmv|O7@mg_h&e9(as;l_(maR_9pdhWdY6fj|UDabEmYJ|-ZVh$tfIbSc zXHvU3Hsjlh5@oVu6pUF5n*Y z2WA}qg4)A)a*bV&(pT@m3YA&P^kEpjE0V_3a`fOT!j5~cO2V@$AF84y2*p3~P!V+D zFrBp*%lL0lVdg=|ble+qs=S`9?(E~bLW96D+`L^*txz^mmVR4j)Bx^sKOsgN89 zS`0wlQ{}YJ=TAUXI1>(E_s666rr@9p1Gv{k0(S26uqK@rt zf1vMkI6T}bfI<=NkeMPun{Zl<)BCva;8_Py6Sl>dVjd`7O@{&>!eAg}6Ks1Yj(nV3 zv8U}YL|6-;ed${Kn{SFEsu!unD^=j8vC!W#@Rs^kd=7?EB2b= zra%ddpYej@ibfdslo9{>b-~oVr__9{KV>UK&Rg?5N`YpJY43 zntT7k<{lfc_i;uVS(`BN+mDT+W3YFc6ItZ_sIB+5;plq-{Hah5wMDa(L#Zqtim8IQ z7prhnr61;){2>>H7?|=Y0u9z$Sf@*2`d>+u%}SxJwH(A!?`C+ltq)cV|DsK-yit+I(uDhPm7mkRvgGUh7qlEL{a%kCk^VG+^9iVh-4d8xryxegQ z=zMfgYqbx~d}Bo4ympvF1LP^S#l&Abknzw#tOyPUL-Jn3>=}^ce;Ra+RFJ3gB*GPb(RINgZ3A$osu@f5UGGKRMG*%Bg;S!rPhD994yp{^8!D1~HW6}V*o<+d!_Y_$6 zO+fpfceL=T?P$40AC2A$VsTzMaKZx4dr)Vq;_nVhua08V4nPs&i~$y4}QJ^ z?$Vp^;Jgz4y0Hs_qYGeY6NTH-OyI$n3J~6H01hISz;G@Y-?Z@{cZV;Qy6?kIyEeG{ zb_q_Hcwpd?fmZznO)Qn$hc_B+@vlV>gjEVcWuP%$`%ntI(?_Tsr;V}RS_pMDxbPD* zxwgzLQ8F5O=(Ifq6`x1LuI6xD)cOoFpH?(U;U?VL~4?R7(-W@v_SWFGqAVv9>2)lf3G0%OViweisjuzjVD%RI5r zO74ZeRv4lV%NghmT8YcUDU{oeXxPLfiJNNwK(w0-UgWHYNU^8Xan}y$IuMF^L7Z^c z?IXl){RQIFrPPDNyP%WCh-%w}Fjr(2HmJx$&Y@r|`XP@r1;E|M_%PzgX}HO=1D`!q z#g4=lY8}r#P;Jn~>+xE^(6Rx?j<3bHYOEm1XNNXt$@Olh80B=@2(z{dLP!}GijQc6 zJI6J+blnZ3L^tC-z6P+o6NpXuitsYY1GiSM#doRmaKv98h8C|IDl?& z5I3^Wu;@tv+}p&DZU^YmICdi@io}4N>U&!F-)QPQBl!%m0+>FO2Puc8ky(KSFT9C? zcy&1}v^|Q0TO-l9t{RxC4`Jz-2AI1-k6wq(fx3}Mc{FA~-Y0pAvDplF_4A{jC@+>2 zj)VF+R%&Tj6;+3HP@K;LbNg8E#db|>=-Q0Q4G*EUgatJuHv{`8Ie7EkjS5WjKzm77 zTqkCP)U5;>*VBDik~R+yV@E)d%@tSwalrPt+n~Lz7c_r;fb2a@zYDu~@*YQSCdBZOU*0wpg_bin=CJ@3ctV99+KCF;Za6%Y1z`z?!TS`c5vtj6-aQO!8H<6n+ZTNk#6VEr3ZFk) zf_vo!kf2X$OjSMz1QSFpb5ex2mwHLN^I>9A#3Np@Lg$WlEH1hVv)0}oV>UJ93 zBi~RQoX=rcYzZ26B3_tq1eNNOAfI^TF=tr`n9wmt{X%ont_RlliM`q$vi_8Q=1 zoPt1yy;S)m1Nx7tgY!KyOY#VgwzMuRBuC;=rFZ&BQwzN3dIo)cg)USf_V8+%0YvHviazOA4}Qs zk<i>8Bs@o0j*yu;Y)1^+@rM{oP75qPu*%v8?r{ukM^j)aRO9s1>vCa zO-O%#k({GAM?Wo18t`G;I^IY0>T5(nHkB8!QEUdXcVAbz%UrVK^D zLEL&7xDwV0w?f#F^~7qFWju)X`y{pW)YBmpxz(5AqTr@|S{I&45wh4T>GNDyqJ>-3hfFWgF9DP0xGk+BEw%SqjzWNTn zZy$m~V#c({U*agmC66aph=IgKe;kriz>q{0O!BOPtE0cE!SR<6JEnjFne%Yv??pH- z<^mDNulB71lR!eeNPyA7aFp_v!KO91BjF=;K-TCaUWCJ>WmS0wsEuKc#@{`MFfT-U4Oe?F~jjcPTrDK%8p6 z1&j2u(E4y5uD|o6-IX&yhf^WQHK|LD&$HK%LNH##pI#nBTY^6J>3o?(<<(%(ulEt}u9gv>v|hs(?Lzs-RVe4G*sS z01Tyyv=^V0kR~OJYC8u?_&X?DQBw3Bde+b%G**+$9z{-)LAcn-|{F2ZWlOH}-p z0@!3(t^rDKbsW6_Rb5T2rAGmc>pY>i=2pQ7hYj$xA4j^%Ki zVi4|)VNq+1_G{vlx9LwdUeN1$0ibcIQmM6IN z*qWs&R&BO8-bC$qN#Pf4#>R0g^IPMk)nBk%CCBksYA47$zTj}|jT3BaO;8Md!Rf|! zOt@P+QMup+myhu=(MPR`YE3V=1Co!4&uS-Wj=kUs?LD?;u{8;nUaW{=iJtU)|O%!I=!llEkV{$C)K84n!nmOLEgSC z)xK$3pe{KTV- zl}eoqU&oiCW4(!LjcpkRLSKqau_dW@>tr4(cqu+-oTT}vEi<_3+RR z$PArXFTtK-Y^a-^T`(ghYm#DW-=3Y{G_ye|CB@uJH>YT9Mq0fu#WJ`(r*vsXMw>m= zDp5DLT=2P(W3LoL`_fz&+fUwJdbK%*J>88(ub^FcRw>ye-CeMw zpj&-*OIAv{hqPW{zvHZOL0`I;T1VmC(AlkJ>={0WdPNTkW>u<9GJNejiXJu1s@A1s z_YZL5dksgI`i@60 zcAWmR^cv|nvSL{D%NRuFwAoCv;siU(STyEzcvG|DrS;EnIL+w__h%)lb)Mk~o70ov zIFW3qU%s+%PG8paM5=vfx!{dC1Etgx>0bI3qT_Rh>is7&gF7qM{FpP+=E%-U)ITdJ z^2XTMG&?)L^Q^SS8xyP4?A%KIbMj7aOdb2P^BX(QDTci`PE_`F|W13U+ zsIyY-#v6-()SOeZ`c<0aZ!AOmb4nLGtKi3*?J*p=Wh@5Ox*~6_l1+2V1-q&ZHQrih zrRJWMHaKtU^wy@JKetk?>%3*yTiY^@ylO*(8k@qmcGaeNHTGRK_BY<{s7uYO^)k5N zJpR_csXy;haMuObA8#F6Ir8fg4Qky*<{f)Y^Xv1wYP~e(orY8M8!8Ph`Z~=!kM-v_ zHg;V+5H`PaisNKcx51@Dh4U_RrYD;pbzKU+F~4ge^<>Me!R7GrdDo@>leZVUE=T{E z-%ZC^(8^+17bo)0jm@l}U9h_@QRCen-n4>FX~Qe2PVd}>2MW5?y02u0z1u6nS=ei6 zSf5?^&O_F$u;0GBKL5r$Po=cNK`+CrMdRXs9@mKtXj{FyI~&=mvNTN85%V|DEtsqZC3KuzNh)ojSola(n{WW8Qq*1 z{}9|XQ1T(T=jO8?A3|C=OBWK2TBb!lhW473e$MY{nbr6hHk?-awbJO;Tc?lVV*{m& zjXk$Mgnf*d;yk_7ZFKu{;m621v(w9udTuY?_!xC`A?@_9S))74;~%4!22TH3?78#j z$H${|TxE2u#;pvZ3o&fwWeh^Stt^@gvApSJOftr89L@`I!h>Zj+j`r0!WWK7aGhZ@ zGHzd4v=A?Aeul%Lw_UJ#Awenq441cYhv>vYqWa(&o{-*-HOmW0+Fa$lNyeR$qMwqD z&C6Gw?Cq4+{FGvqUd~@-+$Hb)Db;bXT=05tmty#*G&ima;U42|<)TmNKIRpok9)h- znm=U(q*sW)Htx}!_>>tsSh41NZx1YgIv&GyR)W=}S6B3NRpN#Fx^g!7BLPcQ1PR%V|39 zYHe22;W*K+Wo#DJxp7*2^XA^BBSjNmE7gb2TZZ(HlrDd*(&nzQN-`ZS7yVXkY*Ax# zvVXKv^V@l=j2gQt(}y+A-)bC(YV5D~KfDzF?SdQk1;-xKNA*SDYJDs&I6v-x)Y$y( zVnD_Pm)E9`n{aoi`QGZF9jr-O-_q`Z|b$Ubm-*3Z$PH|rj?J;}yx#)X~*__4Y@W%ts7Ms7{TFAH@`P%IH^2GPsOGB5VzYjeBv;6%I z9Zy{htN9dz*itK-WnG-m;1rA2QX6k(UA&C>3yz&j?ZS8K61NS$;E7o3kl?wJY-B#Y zvUsUe*78cK!{D^w&804-%q!{M<}XE`EOo2jy^#R^u@A*8b-5!@A6CwcZx<&QF#f zHQl{-DP(Bg_2=^AR-VSXB#U?MVn4@vEgS1k4!!fz`Z+$F+1OBJ@!ogm&xx_Sjg8lb z-XDng`DBXcdQ*?ZheO3bC+94$H$NWw5Pb9J(}m3IEw3#;hCliFZ0YXx+uw&iM*sZz zoNh%^E34%~oY=1^w(U*rLU#i?M0n_=n`i081=#+xGmm}m!2`j&5A6x^{2%+&J`~Uk z#ud^_58kGiZn6*;{x2oq|56bCf8Cmmyk0a{aXSZ32j$-P*EKR6jdlEoW#T<@SN z2X*m*`%WDC`W_OGYvAnZ^~mvsY=cq)WZm3D%jmJeTIu~L$vOmLTItZRS&xdolL+-@ zS@7dp9UMMl4?G=xVCQlO>d!dBNbnGta%m#Jq6zFe_7^ICFyhywB+7H(2t1my!SaUH zD0YyQY=wJD-7WORR_M7^1v+rW0f%BZs9yaF z2YL!1e1*{xCv*UYU1k^m9(|rnyBM)Mwoxg7P-UH;N0pksBo1)zm^%= zXlgNZ)SLm!3{w<&8Ud3%hfs3^FBaTAfO&=CQ1VU_XCIuQO4~V6`KlETc^7LLznOyx zHU?^7F&c5Ul6vqk5ssB@AZ;B9oV((U2Apeg)npfC|7;507?@xRStg(74Z}y4AK*(v zB8>mK4DWs%*3<|R!1F>Z$SB1M0_yA(!s1`@l%J63x%Dg3j)GV7_< z_DjWPkS{DURvMJNw`*H7G8Ax(-zzy$`VcBa3w$=OLhQ<{TYhMd5550nK zhxfuXY4I7gm4R63R`~pr8Go+!!=LYe!k*j^JiKp_x-cS&rv)mhH;!!B!(~ObVQoPE zRnoZWxH^7|FQGK;`M`N}4UVS@Bd^#?+EiOC<<}SmGXqk{(7B2d?2ZFVM;ka6dkNI@ zoH1f66LO~+VfMzqT2o}3kZF)L?tbt98ckc_;A0+O|FZ|4>P~3wU3Zd-oIC<5hm7z? zMlimWy94?B7omA_7ba-dQV)$zK<_R_jDGM5*mv?^XM;9&M`yvsl?Q1ffpPHX-a+ht zNQYThB+y|+2fSJu1h$(l`0B6>CcK>mvHdpK%^l5^&ez=C$1y>2O;<}XG zNZGg{f72vf&*TEtoh6VP0pRs`6V_Bdf&7zE$l<1r#t$s1#7|tv^GtwtsDBP58O2fe z%{7?uDxv1I#GvVH2K9bi5m$s+*D9Z@d#!uk`_H_)hqPs({82R2k`}N6#l7-{!4Apd4drG z#j|1KBR{Z?jK+_QxlnehiegMM!>8dEXqx*48YTB5crk$>*)liobW8K7<+Rt?fXUX9@=o*g@#NP&}Zq4u#a4Kq)C7+D2?KiH92nZd!m; z@L{Z&8ltLq%Hg>D5qur3h7Rwyquh~IQ2fmvd>9|XGT37Q=TnM(w2I39Lx*0e0qB>f zi#z4HG3MGWXzyml*JSQf-p~g{**i%~g+g}VMA13)T@Z^QeQsl1wOK;zhf8 zOKR7fL+HFu9&a9Jhw=F_F#ehcC!|{7a<3fhfUw6)>qD0=BJHE zOLTBy(@u;N{SF`Niox8Mv=xLJq1z`AU-18ih^P-BaLgJ7_PXLy+&(n6md6xnMwC9Z z7FUpMa95J|pt#8_YDU?>T(41*XkhZF`&SAefXNMir!q;z?fbdJNWr< z|Le8LJDN;=KJyCJ8Er#dM;@r(&4q0MxvVTFhaO5IXL_x?w4{g^+- zF(=aa>GxrMN(ctsDTVtvDmeX$1J1APfrfpSIG$yTyKihkwHMYnXUziEyPg48sUv=h zUJHro;<#Cz9c9lbpkMk+D9z#D$fv7pOB)N5DZ{8#i5W zL{6_5=q4@FO@bkCSZ6n$CtC@{8-ZS-R;1qq%Nc)VsEZegE<&}#-*;UR>N8a1JRMI$&D zT&8}K?Vv49;iyDsg^Qkz;5H@%v@c?KqF06D&h!JR#dTO#xDAr`PC!@UFt8ow#O=r0 zVH1NL{*l*1y)+$g%QM6PS$?$Bj-g6lvtg-82(rvG;f&)C5Hyp+HLuv<3%dk-3T*=^ z`#W%8l7=>(gD{mQfbZXjp)Fae{h?Du6A|pCyk|$KRUf5b;9($cJ-DANpLW2X?_t=# z1F#@m2EW8-!-G@w$dynF=c_ZJK~NYAS$5{ZiYB~Uv~1tqx1R`H7^ z6z?q>>TTr4!0VsEPXcj|k`30+T!Nk6d|;QD2l~cufTt%LN@YaA_D}|Fqx6vTU?i&7 zKZU!~&)}S=G;Y4I1rMrFIACLqvq#2ZgkBOIm9Nu8@AUw?xdyJ%Dbt!%md7nW)zEJu z0Jg^I;`1>-T-r_MhSx_?G3y`>l9)wg zm_R)m{S6O|mGPlI4K;K&QUzLxluLjj-ZkHVC$wI}+Mjz-C1V3-y=Q|fKC&pD7J}Qg z1gODkCe&Xzg6DSyQs3NcQG;iI`fu%hXH*qQ7cEJ$1VID^1q4)rqzektb-d&(AUQ}* zatV^bgrK6BF@mUwm?P#aig^rRzfaeR5hnipCeK(-F@JT-XbB8l8WNg;>XNdDc97 zFiw%Bt`8@#s%AC}(_0=U*J}-%t)f9Kv6-A_C8m~sZeT74y3i_(BdjS`mP+JGcD~|%JB?)s*EotYVRT4bmVH@wl5OawO&`Cw zaGw`XVgbv?vc%W!v}~ahpgH9))!@o<0?6vZ zXSPiewk#i~>p;5yURz^}>6!o%0)Xn}t5rqLDKP(t}Am z*u^_1S%_Y_U0mPy?8`?HRvFBrvO_5>Y{V5NJB?3b^Hng-Qi-PT97zprfpk71jpi?X z#EvHSVv3m7z5eoRwlPhNj6?QuHRp}lvc%prze$H4CrZ=9%0x0%abVVa-RRhE6LQU6 z&z$5Jas4dK@s+O|JrWTi^A}wxLa`H9=4?R|wam#yH;KmC+0w{s@^q}ply2(mVM*)5 z>{_<_kRY@x1@C#ow94hE6W%wt>bztdmQLp)wwiDkmOWx3BO^&VyNReUiCf%1fMZV5+|IuHS%AzZu1i{Pl9+#$ z>D-Gat>e+`PQEz1X>Lgqo}6Mq!3m_CJ&DQBddN1^$kE_1Ia(d~np2$9krGasP^F0} zi*J%8g;Uy8G4M7Mi_N5?zlX4L6)iG+s6*v9)tQluIF*W?XKv}i)KqIkt)*FXSM!?P z5V0a=(_5LvaN&#GhuzOq}WH>Zk0MJLCF2Nn^Vj=CdY^fzL%<{z!HcwB2K5yzwaicBiM1Tf6vooCLU3fUa&*~o2 z+xM6mNe`x^mi}Z_tI4`9?nAaK<0&gif)pk`V;Vzk$bQc#%Fw>f&232~kGZ><^4%pY zQ?(lR>Tqf}`-M$zd&!c=M3AeB3u){m7GCs$t&i@<=ER7T%$_eS^GP0!z%;2r3p%o_ zT{2YXYeac(?sB{#uNd+5X!IEu+V^S-Gb1;e8e7I*gi5eY({$Jb<+W^zycTnu`G&a& zc5o}AM5)IM4O&0iogI-#B#Ck9RDZe)W#_%aRN(%UXXeG;NoLWAjTvzN@0iZF~v^gd1`; zuM)^_y$7AK5F-=s5~e)yDtpQtDLh4qw%Sc&1!W7_hZ$qo{YauMvxrT+bcv0BdWFpz zIGm2n%cZu7?khj#wshB_XXy`ZgHAnR zm)AdJqo!+MIg|nI_^d@6ch;~2N2O@dJUcFDm=rBuh-DEMZg9iBj&M7tDbe<|!ZkxzfUreNO(e=#njy{Fj zJYjAn`P3cnE0?_(K^vx2vAjt~S?{(?n%F~;*0-5Zhh0Ww>@T2{eF>Day(3-B>r3kf zA7(>uN3pg>Uv6oi6nfWp5!2h6$NENR+Rc}}!Db}~(#OjIY+83CI?syihQG2RNw+4Z zF=#I9wIhuw$lPHf2gFzpDM`AdW6MTOdd&o*bIAB~1sjxK!j0M zJtG#egije%C@M)`_S?{n!e{LB!>cS@CX62ZUe8_gox$vSN3s>K53|Kl{-mjI!nAoA zG;KjKim@5kv}#BUwhZy2Aietn(!{@%pe3{>dJ?E&=O{tKu1{s8MX z!;mw(eUWLD^d##kJDHNJ18t}Yq$gO)$?m969LQA@{KHm7e>^euOV{UL; z2J|QH?RnO`9G~Cp?nfJ)f~h32iFKm|+@U;&)Bs>fLzE z7|Tj|SQ?fU!{=Hm7P6r_hNO8tjt1!u;l|DzN!jyPv9k^l6mae#mKC}X3*BmWBiw@? zn8?zp>8~)gE0E59Nu*CY1yp$YF>}&&qYJmR$#BOqcBWjBo}HDUbp5k-zP=^Q;?gFj zhG`5X+wJJ4f+5{dSHtuXMUvG1#6rvuvyRUy*oeNe6oDnFuM`Z}l>xJvbJc8S@eX6C zE63O}dvj8pqeCVx6WJEe=Ulkser6GyO{wumSccyX_NJ2q!rcWY&v^MWX@Erl}X z&Edw}=2Oa=V9Gf%jd}JEp`{x+c4A;POE15S<-Rd=z9o$;yUEeLCBxXokDa(T?gnht z2}L$xwJA$)97^Y+`;(zyB*hjPkYU*n3b}NYGo2bk-EYRyn8BX35bxQhUDcvPwYZ3Ee4 zrH*zYFOxDmpDnqpbvEAnWEL@xxoYH=CQp;*45T~Oa`b7mBRh@d1yyN`;@4vNr2Bie zS2l|qqThwWUbS&z`TN+#W6qd5E=r|mR@=!=uVY6N$Ft(f2qyRPGE2`INnZ7L8SiKk zZSUesZ)2Y^|Ib^P#r!ZzaEPQgcKxuF(18|Cx1psQrZfMULuun|0qLE5&&H)}j7cyDzgFzhWiVQQ;M9n6jDM zC+I*^x8An1eym2G-S;t3&lWb&0n2>ShETVt912y?CXZYN60_AM{X@8REY9S5-AtkN z8orcmyMjsH89|*d`_cBLrEIn6E9T~rKuH>-X%oiF!{7hT{8V}5x>*ZT?s73!F{f^Z zgQ;-dZdN2~K zPae&De2^{e{EEE}97cDVcq~SYUl1u z220wbjwVyr6}9YC*GRUi_8r?9f0(sAXkaUX1SGGUNQQV%c9W+`JFyHVa8VGgoNP^2 z?yk(Uv3@|8vqPAqycQ?^DuYHCccOh0`Z90D-fV3ZF>d@e#%o-`##W25m#zbti^ob9 z#T!X&3G3K(dwDi^cnceqU`fx*RH?@LG?Q@c%z`|IuzTCIn0EP7wr=kLnx+^+)3gd{ zXIU8OV5z{H$~?AYJ(e3E(4!nl4X%@JG&!Bi#AloxXw!$=%(XI*mF`cX&`Luxcwxvy zI<>M*QLfYl(-JEcqG&SXN-p6J~%x6QIcW|CErc@pDh}C0ChB5=#h@4~0@=P&losGo?^_Yg*wD!Y+&qrmcR(@$MOLLJOFgrql&F#_1 z=9UFe;It45uei?m&8x$Q@F!!Z^ym!W_!QudTBcA7_CK(kNsG|hyp4Ye3{+5YDpUSPU!O0iQFd332b>@30w5x5O*fq zlb6`=hQUm=-Aj_%uFl3BxdmRo+zfK{W%2uX+ZePscCJNo!?q|cK#^e{$JDH zYBbIf6%nxpF5!R34Z$FCBcKQbLmLZ(Vy(kmpcwIT_!Z#MqVg~I%0t`%VIPF)u;0Tz zjDo#jUjW1aTf`PX6Gbc%)M>A(W`eUP^bb`9(qU}N~ous0(w9=-vv4*nMSClDJ0lmaQxC7?G0 zH{t&VECq^z@rWG*8d0VKJR6z?Y+dj>AOd&}e-&5{=zuzske3eb5B~&W8-YQ93jDLs zh5rgL9sU*gaeylFFM;KO9H2X(3#|(9guM{h4D1Iip>0L^a{vK+XwKl(U_R>X2TX-u z1hxf-K#u_5fqxUM0-i67p`Kod1w$)^FFZF*z)FAv&=v6m&s6kt%W@Yyc-w+90jzI-xWwhP6pT;+AQEca&G}i&}#rcXl`KqFKLlT*qy;; z;Pb#PNIj~Q{4n)o{ z_#I)Jfk%M1fL(zQU>E#D;4?rC&;~T4>=5Xa!2Q4vz*@j**qZ=1paprZs5={627fK= zonTkwX@Zvnd!P-0{}NaZTmUSg^MO7#aRe;@o;;K3-@0(%$qVX)r-(_r5~tPju~_y{CJ zj|9g7zQ`X0UJjfDqJ(9EFz7dscN*Fhuqq&coeg~!;?l6!1H<5d0y{z*fSi>;Ewtm% z4#DmMn-5zHTmh6I?+Ek;a18v9u=fgcVP6&I!|sR}4}K7AAK1&FcLkpW-T+sj&jL0g zCmsGiVID9MSOA=XCI_xVIeplXfD8P1FoSjqBSUF$-G{bcPS^#(WHpufwjuxN@m_mO5=tB<& zp8#h16=fgJ{jBBvX)F3^U?*Jg@*hL1g+C8)h4wqN0pLAA9B>V~BQOK711=)I24&vDo`d*# zAP;s&Xt9V*05pL{U>dYs#P$fc0pA{Mg#2dkD(IomZ2*u6 zZ3$2W-3$6~a3-LMm>l#?utmTf5c7q$4Lkwd4Vne88Aya*1N|Ow8MYp@Fz^bX6N=x3 zzY+E=*sEaoKwb`fCHUiD?}nWKdnvdH*b2=CNCZB>_e1PD$^`I2?#-K0JI#~<=}cC1@?UK zUF1Cit|Mn2w1Kdzp)ZE522B$7DL@Ur06u<24kJe!tOV8r#zI^Jv;jyH*#<}ea=;Q` zIC4HC)(c31-vB=eiR!Rt0M7tf;2iW1$hiyq4EQ2og**dbGVD*l8raT=7lBWMO;9El z_8Fij;_m=)*n7Z+h+hN7z~2kr08~TY1>n>5`ktMS_x#JkZ{ff~U3i7UFsQf0pB{u8 z$}g`=BP{&4*RrEUB(&>ABvhm7MWyws>&4`GH|oU|2P%z`Q1S5@)1gb$^f8j%G8)H7 z^(ay*mhL&-r&y*}_4Hy{qYaJ4a;Ei4CGwVyJ|!Ld-I!jYFrclmM3F?4$0|7}`Hod~ z@h@`h^Y@!_t5 zv5k^l(sqkVbscupTuLkF;;>TX{M(C6LrhyXnsfFLpDi6dZ8twQz)(d+(ab_OcosX6cUWv%I>z%y6;K23e!E+n>uC z*B!NxGud)6Q?Boh`}X_$#{njY)3tx=WCw4XyH&T`4e!@3@s@bBAR%l#qHv}>`v}|xW#AO zjkl=;i)d2wT{r`MZ{s*PS^H)hTB9`Nkk%t73zS_vP%jFqad zlT(x`E0Q1PFMhT6K~WV~{X;YSJj~Cl%6M}f0{s2FoB{#@BgO>;hNj->6!bj%&iSCI zar(`{akJyqtP&T`o)emKX_Hi#!j>&l!&GkHP|;7(e>KNBgHxFsp4AjEHzMz5{J{3ZhmxR}GLyr>v7^<`-@F(*-N~RL zV5VuqyrY6^V-$Yz&aI8>3d4G<)*0{ruSJftAUHO?qdN zR}7m`IdsHYNtu+jJ6>Pzx;sfMOt9=&!Ud(^t9mVhxC-$usWs*S^HZ;0aaK;#zqw?h z+KvZXyQS|*Z4HYIuYTDrW51WSrtfthp60MaE6oG=heNV8GmrJzR+U<7exWMsw3Bm( zevNg?my(+zw6A7MU2=}dX)egt%Kb2F*46%(Q;$UCb;~cU&s)C5S}FhL#D3iiN@lyv z4-}i1w@{%^^|lEy4?b1P#@ViTv%m21mInTC;~huN4{saO`MTcolP=eXd7MkUK5~t8 zY2>K)Ph4b2Kdh+1Qz-)cFBJBDb#GDDIk)FMit5{xs*A*r4~!baTN$ezC#5@2TVA&B z#iJtmW*4jb7>#VbQKFQlvRLzmn|kwD)rVUTcW_!6GWS$;67YO>|wL&pd3nlDbD((kZ}yj=gv@__i;2WR#ywUwM8S2*SI(ObG5UvH1< z%06$ukUdaRX36vr&$x!^sY)w5Yxn5vcDRePZkAqmSA#x9*>!zZN>28$D453+Uv<8A zrpeS^Ei-NH3od8)y4x%*@vre&I{RwGJB3-sF%|L3GD#ZO=Y+3nj`tdp(AsI%%?VDy z6I1%M-I2Ldpww|*bftlTzx|YBIu*vp{GBT9Rb0AVp?6}FY(~ z9!Xpjabn}V5uTA+K<+lopY30 zTX=4$QQWrHox#pK{A?DjbDy={Xnj|KYTSB*GKVSa=S94DG^zf8a{PuT@jSC-d3^WBi3!`&yj^_u;lR4}6|K&d*bL_aL9?s#7#8G{preR>bc=gCz^2akI_ z)g;jNgYo7jZ|$|?vNPS>C2gw9j%+>`xH?IySvM+4_fYsGhvCVt&bQnoh35}S51vvcZcrCkue(8nH+Ae&O^KBv7wRYY7_BPqb$j=-wH?pJL&V>zX z*V@`*_t>vVJJ>1Fer$ruqy3M34SfY?s3@R(vfqkizT4E!a^ud)nY144pPO~3%jIdj*l@Vp(U!?`yJd|~Gd!1F)upS3 z=;6{MSsd?FSr4OE`Z=G=#WS;Q#)se96u9pE(=ECxYi8~$o{`;jRJ^C#MtgnAn0l<& zqUvl2Zc*xUB=*k!;}^$?rAW#Z$rP#INGUr$FGBvz5~JlqZ!5pbwbYXu zG}-9EvVFy4w5*KZ)O(~!ovU{I?e6=|elZiC+{;xO z2bM(VCPv0ZCkqN9von)3vc4{k-WaTQFxX)4R&j&cVsU%`{pTp@r!mtnKY&*1tm@fW zto_qxY0ds23DzNa;u^+cYMs0OI1-coBc%S_Lhaw}KUECKc1TicN}7jqWUu z$FJy%{p|@a`l_Cjl{JmgI*d1k34cBiMWnN|(Eq;h{Hy={(|)`y6V~_B0r*+$IID0*sZ}+LtKto* zvASeNbXr_=mikWV;kwCbaq;<)vFU=G)U+&JTkWvnx+&0L>1IZ!CC2L-YwM;(r^Z7_ z%Ml3R;h{&F*>KI;?eq+o=Kan3wR`dH-pB|zU#pC4uRs1<&3~;ZAvsZw08a?$DlAr* zp4Psb0sanXs4;fnTW{!of8x^LTlJUFPwh!g%eLnIy(!=2A395TH2<>6@7J)uZ}Kmp zpEj8wNRQ68wD|i*e+c}i&HnMl=-;>cW90w!M*n(8{KdBap72u>(5JuJ>+kY^+UIY# zv%hclhrmyp`{Qi-dq4jX`Q>@mew5o^j>0==dS?5pM35Yl8J$@me34`nL}zBE=Lu`Y z6TY&N)6;}1udi8Ob6uPry5CyyL(IIdWnVMhks|8yd_TWILAGCSo$70pOoc!Bi7tFT z-xm=6c<}i=zI`I!na@w;J0!Mi_V#~?=Ltop?{Qb5-HvZr?Qu6gKaY=cU|}WpZm@+u zpYNECeEWWGse+tPmq1>yg?Da@YhZRvnpg6mWc!%-0{6`5AgdI&q3MZkY3@lesc{0g zaFN|{u z%oEn-6>J_S%n7lu$%+dy&k45py8Y1L@7v7@2{IQ5b7Rs1vtk^C+YHSJ4a!dmw)n-i zU7XQ==fYr1KdV?5VY~RraY0rYs6Qoiu$yq(Xs3Bze4ve4e6W3cy>6*C$r%*V_#HlJ~XWe%^=wkeg z=fn^D1-2fgr}k zFCiw?Is4nON(!;?{;~c4yS4>~rUhI0B}E723y(oTbg+HlKONhUf6D#GWBkwkC_Khq z*i!qs`+s+g3qpf%e1pE87r*Gkp|Pof!q>&ubMu#N`tx=3kH;6UeOv=Fa9y+TcFxX; zH8=a`HU^pruPNWQYf~V+PT{=FiVkuw2nlw#`g*;zw}M>|V;S3CkMQ*qYaz&uNfy3F z-ON0P@^icfyW^a}we$P+D^w7h=A98^!E3+fcn#(iA}=`scLdxA_J5@ZRlt+AW-{!jbUUg(GYaS-mBi|}=TV<51}2yHKz$aieNZ^jDi`}Tv|F@DgW z0mAz!jstpK*icmVy;0x#`|C}+z3YVue7>-6?FBd&a^jqA+OOln`({iqUs%7_w|0N+ zYIO0p{s!~;!fUgT@xzIz_sEFO9F`NGZ4#T2VbVSZ(f&5b&~6`nXbA9a;!nRK>lN{Z JJm1g!{{WF|k&FNU literal 0 HcmV?d00001 diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 91889d5..e7485f4 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -9,7 +9,7 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( - path="/Users/achal/tecton/feast/prompt_dory/data/driver_stats.parquet", + path="driver_stats.parquet", event_timestamp_column="event_timestamp", created_timestamp_column="created", ) From 76ce225270939e4223f684aa02bf7830aeb724c2 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 15:05:51 -0700 Subject: [PATCH 18/39] update source Signed-off-by: Achal Shah --- serving/src/test/resources/feast_project/example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index e7485f4..33030d9 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -9,7 +9,7 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( - path="driver_stats.parquet", + path="data/driver_stats.parquet", event_timestamp_column="event_timestamp", created_timestamp_column="created", ) From 66c8d7ec122bae9951c59e9eb47a374ca5ee665c Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 16:58:36 -0700 Subject: [PATCH 19/39] update source Signed-off-by: Achal Shah --- serving/src/test/resources/feast_project/example.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 33030d9..31baf0a 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -1,15 +1,17 @@ # This is an example feature definition file -from google.protobuf.duration_pb2 import Duration import pandas as pd +from pathlib import Path +from google.protobuf.duration_pb2 import Duration from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. +path = str(Path(__file__).parent / "/data/driver_stats.parquet") driver_hourly_stats = FileSource( - path="data/driver_stats.parquet", + path=path, event_timestamp_column="event_timestamp", created_timestamp_column="created", ) From 100e37f21cf383fcd49d19e97992a9d52793b7f5 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 17:07:45 -0700 Subject: [PATCH 20/39] update source Signed-off-by: Achal Shah --- serving/src/test/resources/feast_project/example.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 31baf0a..9600fe3 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -9,9 +9,10 @@ # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. -path = str(Path(__file__).parent / "/data/driver_stats.parquet") +file_path = str(Path(__file__).parent / "/data/driver_stats.parquet") +print(f"Using file path: {file_path}") driver_hourly_stats = FileSource( - path=path, + path=file_path, event_timestamp_column="event_timestamp", created_timestamp_column="created", ) From e1e1f36c4b837fcf713d5a5ddf6c7608d90dea9d Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 27 Sep 2021 17:16:59 -0700 Subject: [PATCH 21/39] update source Signed-off-by: Achal Shah --- serving/src/test/resources/feast_project/example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 9600fe3..69df149 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -9,7 +9,7 @@ # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. -file_path = str(Path(__file__).parent / "/data/driver_stats.parquet") +file_path = "./serving/src/test/resources/feast_project/data/driver_stats.parquet" print(f"Using file path: {file_path}") driver_hourly_stats = FileSource( path=file_path, From 7ae280ecf798e65bf8c30af6247c57f0757bc907 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Sat, 2 Oct 2021 21:41:47 -0700 Subject: [PATCH 22/39] Wrestling with spring Signed-off-by: Achal Shah --- .../feast/serving/ServingApplication.java | 17 ++ .../serving/config/ContextClosedHandler.java | 2 + .../feast/serving/config/CoreCondition.java | 31 +++ .../serving/config/RegistryCondition.java | 31 +++ .../config/ServingServiceConfigV2.java | 61 +++-- .../serving/config/SpecServiceConfig.java | 14 +- .../controller/HealthServiceController.java | 9 +- .../serving/it/ServingServiceFeast10IT.java | 221 ++++++++++++++++++ .../test/resources/feast_project/example.py | 2 - .../feast_project/feature_store.yaml | 2 +- 10 files changed, 355 insertions(+), 35 deletions(-) create mode 100644 serving/src/main/java/feast/serving/config/CoreCondition.java create mode 100644 serving/src/main/java/feast/serving/config/RegistryCondition.java create mode 100644 serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java diff --git a/serving/src/main/java/feast/serving/ServingApplication.java b/serving/src/main/java/feast/serving/ServingApplication.java index ab036d0..dbea42f 100644 --- a/serving/src/main/java/feast/serving/ServingApplication.java +++ b/serving/src/main/java/feast/serving/ServingApplication.java @@ -17,12 +17,16 @@ package feast.serving; import feast.serving.config.FeastProperties; +import java.util.Arrays; +import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; @SpringBootApplication( exclude = { @@ -35,4 +39,17 @@ public class ServingApplication { public static void main(String[] args) { SpringApplication.run(ServingApplication.class, args); } + + @Bean + public CommandLineRunner commandLineRunner(ApplicationContext ctx) { + return args -> { + System.out.println("Let's inspect the beans provided by Spring Boot:"); + + String[] beanNames = ctx.getBeanDefinitionNames(); + Arrays.sort(beanNames); + for (String beanName : beanNames) { + System.out.println(beanName); + } + }; + } } diff --git a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java index 2bc9743..cdf791c 100644 --- a/serving/src/main/java/feast/serving/config/ContextClosedHandler.java +++ b/serving/src/main/java/feast/serving/config/ContextClosedHandler.java @@ -18,11 +18,13 @@ import java.util.concurrent.ScheduledExecutorService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; import org.springframework.stereotype.Component; @Component +@ConditionalOnBean(CoreCondition.class) public class ContextClosedHandler implements ApplicationListener { @Autowired ScheduledExecutorService executor; diff --git a/serving/src/main/java/feast/serving/config/CoreCondition.java b/serving/src/main/java/feast/serving/config/CoreCondition.java new file mode 100644 index 0000000..cac6f9a --- /dev/null +++ b/serving/src/main/java/feast/serving/config/CoreCondition.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class CoreCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") == null; + } +} diff --git a/serving/src/main/java/feast/serving/config/RegistryCondition.java b/serving/src/main/java/feast/serving/config/RegistryCondition.java new file mode 100644 index 0000000..7d73cc8 --- /dev/null +++ b/serving/src/main/java/feast/serving/config/RegistryCondition.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.config; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class RegistryCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + final Environment env = context.getEnvironment(); + return env.getProperty("feast.registry") != null; + } +} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 7410fc6..2b1030f 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -45,6 +45,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; @@ -70,10 +71,11 @@ public BigtableDataClient bigtableClient(FeastProperties feastProperties) throws } @Bean + @Conditional(CoreCondition.class) public ServingServiceV2 servingServiceV2( FeastProperties feastProperties, CachedSpecService specService, Tracer tracer) { - ServingServiceV2 servingService = null; - FeastProperties.Store store = feastProperties.getActiveStore(); + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); OnlineRetrieverV2 retrieverV2; switch (store.getType()) { @@ -84,15 +86,7 @@ public ServingServiceV2 servingServiceV2( break; case REDIS: RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); - final EntityKeySerializer serializer; - if (feastProperties.getRegistry() != null) { - log.info("Created EntityKeySerializerV2"); - serializer = new EntityKeySerializerV2(); - } else { - log.info("Using byteArray method"); - serializer = (AbstractMessageLite::toByteArray); - } - retrieverV2 = new OnlineRetriever(redisClient, serializer); + retrieverV2 = new OnlineRetriever(redisClient, (AbstractMessageLite::toByteArray)); break; case BIGTABLE: BigtableDataClient bigtableClient = context.getBean(BigtableDataClient.class); @@ -129,16 +123,45 @@ public ServingServiceV2 servingServiceV2( } final FeatureSpecRetriever featureSpecRetriever; - if (feastProperties.getRegistry() != null) { - log.info("Created RegistryFeatureSpecRetriever"); - final LocalRegistryRepo repo = - new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); - featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); - } else { - log.info("Created CoreFeatureSpecRetriever"); - featureSpecRetriever = new CoreFeatureSpecRetriever(specService); + log.info("Created CoreFeatureSpecRetriever"); + featureSpecRetriever = new CoreFeatureSpecRetriever(specService); + + servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever); + + return servingService; + } + + @Bean + @Conditional(RegistryCondition.class) + public ServingServiceV2 registryBasedServingServiceV2( + FeastProperties feastProperties, Tracer tracer) { + final ServingServiceV2 servingService; + final FeastProperties.Store store = feastProperties.getActiveStore(); + + OnlineRetrieverV2 retrieverV2; + switch (store.getType()) { + case REDIS_CLUSTER: + RedisClientAdapter redisClusterClient = + RedisClusterClient.create(store.getRedisClusterConfig()); + retrieverV2 = new OnlineRetriever(redisClusterClient, new EntityKeySerializerV2()); + break; + case REDIS: + RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); + log.info("Created EntityKeySerializerV2"); + retrieverV2 = new OnlineRetriever(redisClient, new EntityKeySerializerV2()); + break; + default: + throw new RuntimeException( + String.format( + "Unable to identify online store type: %s for Regsitry Backed Serving Service", + store.getType())); } + final FeatureSpecRetriever featureSpecRetriever; + log.info("Created RegistryFeatureSpecRetriever"); + final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); + featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); + servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever); return servingService; diff --git a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java index 369d543..29d3bf0 100644 --- a/serving/src/main/java/feast/serving/config/SpecServiceConfig.java +++ b/serving/src/main/java/feast/serving/config/SpecServiceConfig.java @@ -16,8 +16,6 @@ */ package feast.serving.config; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; import io.grpc.CallCredentials; @@ -28,15 +26,16 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; @Configuration public class SpecServiceConfig { private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class); - private String feastCoreHost; - private int feastCorePort; - private int feastCachedSpecServiceRefreshInterval; + private final String feastCoreHost; + private final int feastCorePort; + private final int feastCachedSpecServiceRefreshInterval; @Autowired public SpecServiceConfig(FeastProperties feastProperties) { @@ -46,6 +45,7 @@ public SpecServiceConfig(FeastProperties feastProperties) { } @Bean + @Conditional(CoreCondition.class) public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( CachedSpecService cachedSpecStorage) { ScheduledExecutorService scheduledExecutorService = @@ -60,8 +60,8 @@ public ScheduledExecutorService cachedSpecServiceScheduledExecutorService( } @Bean - public CachedSpecService specService(ObjectProvider callCredentials) - throws InvalidProtocolBufferException, JsonProcessingException { + @Conditional(CoreCondition.class) + public CachedSpecService specService(ObjectProvider callCredentials) { CoreSpecService coreService = new CoreSpecService(feastCoreHost, feastCorePort, callCredentials); CachedSpecService cachedSpecStorage = new CachedSpecService(coreService); diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java index 4bee981..ef675d4 100644 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ b/serving/src/main/java/feast/serving/controller/HealthServiceController.java @@ -19,7 +19,6 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.serving.interceptors.GrpcMonitoringInterceptor; import feast.serving.service.ServingServiceV2; -import feast.serving.specs.CachedSpecService; import io.grpc.health.v1.HealthGrpc.HealthImplBase; import io.grpc.health.v1.HealthProto.HealthCheckRequest; import io.grpc.health.v1.HealthProto.HealthCheckResponse; @@ -32,12 +31,10 @@ @GrpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class HealthServiceController extends HealthImplBase { - private CachedSpecService specService; - private ServingServiceV2 servingService; + private final ServingServiceV2 servingService; @Autowired - public HealthServiceController(CachedSpecService specService, ServingServiceV2 servingService) { - this.specService = specService; + public HealthServiceController(final ServingServiceV2 servingService) { this.servingService = servingService; } @@ -47,7 +44,7 @@ public void check( // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g. // if it's online service check that it the service can retrieve dummy/random // feature table. - // Implement similary for batch service. + // Implement similarly for batch service. try { servingService.getFeastServingInfo(GetFeastServingInfoRequest.getDefaultInstance()); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java new file mode 100644 index 0000000..9d66eff --- /dev/null +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -0,0 +1,221 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.protobuf.Timestamp; +import feast.common.it.DataGenerator; +import feast.common.models.FeatureV2; +import feast.proto.core.EntityProto; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingServiceGrpc; +import feast.proto.storage.RedisProto; +import feast.proto.types.ValueProto; +import io.grpc.ManagedChannel; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import io.lettuce.core.codec.ByteArrayCodec; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.junit.jupiter.Testcontainers; + +@ActiveProfiles("it") +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "feast.registry:=./serving/src/test/resources/feast_project/data/registry.db", + "feast.stores[0].config.port=6389" + }) +@Testcontainers +public class ServingServiceFeast10IT extends BaseAuthIT { + + static final Map options = new HashMap<>(); + static final String timestampPrefix = "_ts"; + static CoreSimpleAPIClient coreClient; + static ServingServiceGrpc.ServingServiceBlockingStub servingStub; + static RedisCommands syncCommands; + + static final int FEAST_SERVING_PORT = 6568; + @LocalServerPort private int metricsPort; + + @DynamicPropertySource + static void initialize(DynamicPropertyRegistry registry) { + registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + } + + @BeforeAll + static void globalSetup() { + servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); + + RedisClient redisClient = + RedisClient.create(new RedisURI("localhost", 6389, Duration.ofMillis(2000))); + StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); + syncCommands = connection.sync(); + + String projectName = "default"; + // Apply Entity + String entityName = "driver_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + String description = "My driver id"; + ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; + EntityProto.EntitySpecV2 entitySpec = + EntityProto.EntitySpecV2.newBuilder() + .setName(entityName) + .setDescription(description) + .setValueType(entityType) + .build(); + TestUtils.applyEntity(coreClient, projectName, entitySpec); + + // Apply FeatureTable + String featureTableName = "rides"; + + ServingAPIProto.FeatureReferenceV2 feature1Reference = + DataGenerator.createFeatureReference("rides", "trip_cost"); + ServingAPIProto.FeatureReferenceV2 feature2Reference = + DataGenerator.createFeatureReference("rides", "trip_distance"); + ServingAPIProto.FeatureReferenceV2 feature3Reference = + DataGenerator.createFeatureReference("rides", "trip_empty"); + ServingAPIProto.FeatureReferenceV2 feature4Reference = + DataGenerator.createFeatureReference("rides", "trip_wrong_type"); + + // Event Timestamp + String eventTimestampKey = timestampPrefix + ":" + featureTableName; + Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); + + // Serialize Redis Key with Entity i.e + RedisProto.RedisKeyV2 redisKey = + RedisProto.RedisKeyV2.newBuilder() + .setProject(projectName) + .addEntityNames(entityName) + .addEntityValues(entityValue) + .build(); + + ImmutableMap featureReferenceValueMap = + ImmutableMap.of( + feature1Reference, + DataGenerator.createInt64Value(42), + feature2Reference, + DataGenerator.createDoubleValue(42.2), + feature3Reference, + DataGenerator.createEmptyValue(), + feature4Reference, + DataGenerator.createDoubleValue(42.2)); + + // Insert timestamp into Redis and isTimestampMap only once + syncCommands.hset( + redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); + featureReferenceValueMap.forEach( + (featureReference, featureValue) -> { + // Murmur hash Redis Feature Field i.e murmur() + String delimitedFeatureReference = + featureReference.getFeatureTable() + ":" + featureReference.getName(); + byte[] featureReferenceBytes = + Hashing.murmur3_32() + .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) + .asBytes(); + // Insert features into Redis + syncCommands.hset( + redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); + }); + + // set up options for call credentials + options.put("oauth_url", TOKEN_URL); + options.put(CLIENT_ID, CLIENT_ID); + options.put(CLIENT_SECRET, CLIENT_SECRET); + options.put("jwkEndpointURI", JWK_URI); + options.put("audience", AUDIENCE); + options.put("grant_type", GRANT_TYPE); + } + + @AfterAll + static void tearDown() { + ((ManagedChannel) servingStub.getChannel()).shutdown(); + } + + @Test + public void shouldRegisterAndGetOnlineFeatures() { + // getOnlineFeatures Information + String projectName = "feast_project"; + String entityName = "driver_id"; + ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); + + // Instantiate EntityRows + final Timestamp timestamp = Timestamp.getDefaultInstance(); + GetOnlineFeaturesRequestV2.EntityRow entityRow1 = + DataGenerator.createEntityRow( + entityName, DataGenerator.createInt64Value(1001), timestamp.getSeconds()); + ImmutableList entityRows = ImmutableList.of(entityRow1); + + // Instantiate FeatureReferences + ServingAPIProto.FeatureReferenceV2 feature1Reference = + DataGenerator.createFeatureReference("driver_hourly_stats", "conv_rate"); + ServingAPIProto.FeatureReferenceV2 feature2Reference = + DataGenerator.createFeatureReference("driver_hourly_stats", "acc_rate"); + ServingAPIProto.FeatureReferenceV2 feature3Reference = + DataGenerator.createFeatureReference("driver_hourly_stats", "avg_daily_trips"); + ImmutableList featureReferences = + ImmutableList.of(feature1Reference, feature2Reference, feature3Reference); + + // Build GetOnlineFeaturesRequestV2 + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + TestUtils.createOnlineFeatureRequest(projectName, featureReferences, entityRows); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); + + ImmutableMap expectedValueMap = + ImmutableMap.of( + entityName, + entityValue, + FeatureV2.getFeatureStringRef(feature1Reference), + DataGenerator.createInt64Value(42)); + + ImmutableMap expectedStatusMap = + ImmutableMap.of( + entityName, + GetOnlineFeaturesResponse.FieldStatus.PRESENT, + FeatureV2.getFeatureStringRef(feature1Reference), + GetOnlineFeaturesResponse.FieldStatus.PRESENT); + + GetOnlineFeaturesResponse.FieldValues expectedFieldValues = + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(expectedValueMap) + .putAllStatuses(expectedStatusMap) + .build(); + ImmutableList expectedFieldValuesList = + ImmutableList.of(expectedFieldValues); + + assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + } +} diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py index 69df149..4d8032f 100644 --- a/serving/src/test/resources/feast_project/example.py +++ b/serving/src/test/resources/feast_project/example.py @@ -1,7 +1,6 @@ # This is an example feature definition file import pandas as pd -from pathlib import Path from google.protobuf.duration_pb2 import Duration from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService @@ -10,7 +9,6 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. file_path = "./serving/src/test/resources/feast_project/data/driver_stats.parquet" -print(f"Using file path: {file_path}") driver_hourly_stats = FileSource( path=file_path, event_timestamp_column="event_timestamp", diff --git a/serving/src/test/resources/feast_project/feature_store.yaml b/serving/src/test/resources/feast_project/feature_store.yaml index 29e8b53..eea3d61 100644 --- a/serving/src/test/resources/feast_project/feature_store.yaml +++ b/serving/src/test/resources/feast_project/feature_store.yaml @@ -1,4 +1,4 @@ -project: prompt_dory +project: feast_project provider: local online_store: type: redis From efc13b9046f9c192f477c0480144c842f4870250 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 14:57:40 -0700 Subject: [PATCH 23/39] Tests Signed-off-by: Achal Shah --- .../feast/serving/ServingApplication.java | 17 -- .../config/ServingServiceConfigV2.java | 1 + .../serving/registry/LocalRegistryRepo.java | 7 + .../serving/it/ServingServiceFeast10IT.java | 161 ++++++------------ .../docker-compose-feast10-it.yml | 18 ++ .../docker-compose/feast10/Dockerfile | 10 ++ .../feast10/driver_stats.parquet | Bin 0 -> 34708 bytes .../docker-compose/feast10/feature_store.yaml | 9 + .../docker-compose/feast10/materialize.py | 45 +++++ .../docker-compose/feast10/requirements.txt | 1 + .../resources/feast_project/data/.gitignore | 1 + .../feast_project/feature_store.yaml | 1 - 12 files changed, 140 insertions(+), 131 deletions(-) create mode 100644 serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml create mode 100644 serving/src/test/resources/docker-compose/feast10/Dockerfile create mode 100644 serving/src/test/resources/docker-compose/feast10/driver_stats.parquet create mode 100644 serving/src/test/resources/docker-compose/feast10/feature_store.yaml create mode 100644 serving/src/test/resources/docker-compose/feast10/materialize.py create mode 100644 serving/src/test/resources/docker-compose/feast10/requirements.txt create mode 100644 serving/src/test/resources/feast_project/data/.gitignore diff --git a/serving/src/main/java/feast/serving/ServingApplication.java b/serving/src/main/java/feast/serving/ServingApplication.java index dbea42f..ab036d0 100644 --- a/serving/src/main/java/feast/serving/ServingApplication.java +++ b/serving/src/main/java/feast/serving/ServingApplication.java @@ -17,16 +17,12 @@ package feast.serving; import feast.serving.config.FeastProperties; -import java.util.Arrays; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; @SpringBootApplication( exclude = { @@ -39,17 +35,4 @@ public class ServingApplication { public static void main(String[] args) { SpringApplication.run(ServingApplication.class, args); } - - @Bean - public CommandLineRunner commandLineRunner(ApplicationContext ctx) { - return args -> { - System.out.println("Let's inspect the beans provided by Spring Boot:"); - - String[] beanNames = ctx.getBeanDefinitionNames(); - Arrays.sort(beanNames); - for (String beanName : beanNames) { - System.out.println(beanName); - } - }; - } } diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 2b1030f..2e5e0c4 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -159,6 +159,7 @@ public ServingServiceV2 registryBasedServingServiceV2( final FeatureSpecRetriever featureSpecRetriever; log.info("Created RegistryFeatureSpecRetriever"); + log.info("Working Directory = " + System.getProperty("user.dir")); final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry())); featureSpecRetriever = new RegistryFeatureSpecRetriever(repo); diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java index 27d1576..e2c8acf 100644 --- a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -23,18 +23,25 @@ import feast.serving.exception.SpecRetrievalException; import java.nio.file.Files; import java.nio.file.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LocalRegistryRepo implements RegistryRepository { + public static final Logger log = LoggerFactory.getLogger(LocalRegistryRepo.class); private final Path localRegistryPath; public LocalRegistryRepo(Path localRegistryPath) { this.localRegistryPath = localRegistryPath; + log.info("Working Directory =" + System.getProperty("user.dir")); + log.info("Local Registry Path: {}", this.localRegistryPath.toAbsolutePath()); + assert this.localRegistryPath.toFile().exists(); } @Override public RegistryProto.Registry getRegistry() { try { + final byte[] registryContents = Files.readAllBytes(this.localRegistryPath); return RegistryProto.Registry.parseFrom(registryContents); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index 9d66eff..691cf19 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -17,19 +17,15 @@ package feast.serving.it; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hashing; import com.google.protobuf.Timestamp; import feast.common.it.DataGenerator; -import feast.common.models.FeatureV2; -import feast.proto.core.EntityProto; import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; -import feast.proto.storage.RedisProto; import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; import io.lettuce.core.RedisClient; @@ -37,39 +33,48 @@ import io.lettuce.core.api.StatefulRedisConnection; import io.lettuce.core.api.sync.RedisCommands; import io.lettuce.core.codec.ByteArrayCodec; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; +import java.io.File; +import java.util.List; +import org.junit.ClassRule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.server.LocalServerPort; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.WaitAllStrategy; +import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @ActiveProfiles("it") @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { - "feast.registry:=./serving/src/test/resources/feast_project/data/registry.db", - "feast.stores[0].config.port=6389" + "feast.registry:src/test/resources/feast_project/data/registry.db", }) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { - static final Map options = new HashMap<>(); + public static final Logger log = LoggerFactory.getLogger(ServingServiceFeast10IT.class); + static final String timestampPrefix = "_ts"; - static CoreSimpleAPIClient coreClient; static ServingServiceGrpc.ServingServiceBlockingStub servingStub; static RedisCommands syncCommands; static final int FEAST_SERVING_PORT = 6568; @LocalServerPort private int metricsPort; + @ClassRule @Container + public static DockerComposeContainer environment = + new DockerComposeContainer( + new File("src/test/resources/docker-compose/docker-compose-feast10-it.yml")) + .withExposedService(REDIS, REDIS_PORT); + @DynamicPropertySource static void initialize(DynamicPropertyRegistry registry) { registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); @@ -77,86 +82,22 @@ static void initialize(DynamicPropertyRegistry registry) { @BeforeAll static void globalSetup() { - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); + environment.waitingFor("materialize", new WaitAllStrategy()); + + servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); RedisClient redisClient = - RedisClient.create(new RedisURI("localhost", 6389, Duration.ofMillis(2000))); + RedisClient.create( + new RedisURI( + environment.getServiceHost("redis_1", REDIS_PORT), + environment.getServicePort("redis_1", REDIS_PORT), + java.time.Duration.ofMillis(2000))); StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); syncCommands = connection.sync(); - String projectName = "default"; - // Apply Entity - String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); - String description = "My driver id"; - ValueProto.ValueType.Enum entityType = ValueProto.ValueType.Enum.INT64; - EntityProto.EntitySpecV2 entitySpec = - EntityProto.EntitySpecV2.newBuilder() - .setName(entityName) - .setDescription(description) - .setValueType(entityType) - .build(); - TestUtils.applyEntity(coreClient, projectName, entitySpec); - - // Apply FeatureTable - String featureTableName = "rides"; + List keys = syncCommands.keys("*".getBytes()); - ServingAPIProto.FeatureReferenceV2 feature1Reference = - DataGenerator.createFeatureReference("rides", "trip_cost"); - ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("rides", "trip_distance"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = - DataGenerator.createFeatureReference("rides", "trip_empty"); - ServingAPIProto.FeatureReferenceV2 feature4Reference = - DataGenerator.createFeatureReference("rides", "trip_wrong_type"); - - // Event Timestamp - String eventTimestampKey = timestampPrefix + ":" + featureTableName; - Timestamp eventTimestampValue = Timestamp.newBuilder().setSeconds(100).build(); - - // Serialize Redis Key with Entity i.e - RedisProto.RedisKeyV2 redisKey = - RedisProto.RedisKeyV2.newBuilder() - .setProject(projectName) - .addEntityNames(entityName) - .addEntityValues(entityValue) - .build(); - - ImmutableMap featureReferenceValueMap = - ImmutableMap.of( - feature1Reference, - DataGenerator.createInt64Value(42), - feature2Reference, - DataGenerator.createDoubleValue(42.2), - feature3Reference, - DataGenerator.createEmptyValue(), - feature4Reference, - DataGenerator.createDoubleValue(42.2)); - - // Insert timestamp into Redis and isTimestampMap only once - syncCommands.hset( - redisKey.toByteArray(), eventTimestampKey.getBytes(), eventTimestampValue.toByteArray()); - featureReferenceValueMap.forEach( - (featureReference, featureValue) -> { - // Murmur hash Redis Feature Field i.e murmur() - String delimitedFeatureReference = - featureReference.getFeatureTable() + ":" + featureReference.getName(); - byte[] featureReferenceBytes = - Hashing.murmur3_32() - .hashString(delimitedFeatureReference, StandardCharsets.UTF_8) - .asBytes(); - // Insert features into Redis - syncCommands.hset( - redisKey.toByteArray(), featureReferenceBytes, featureValue.toByteArray()); - }); - - // set up options for call credentials - options.put("oauth_url", TOKEN_URL); - options.put(CLIENT_ID, CLIENT_ID); - options.put(CLIENT_SECRET, CLIENT_SECRET); - options.put("jwkEndpointURI", JWK_URI); - options.put("audience", AUDIENCE); - options.put("grant_type", GRANT_TYPE); + log.info("Keys: {}", keys); } @AfterAll @@ -182,11 +123,9 @@ public void shouldRegisterAndGetOnlineFeatures() { ServingAPIProto.FeatureReferenceV2 feature1Reference = DataGenerator.createFeatureReference("driver_hourly_stats", "conv_rate"); ServingAPIProto.FeatureReferenceV2 feature2Reference = - DataGenerator.createFeatureReference("driver_hourly_stats", "acc_rate"); - ServingAPIProto.FeatureReferenceV2 feature3Reference = DataGenerator.createFeatureReference("driver_hourly_stats", "avg_daily_trips"); ImmutableList featureReferences = - ImmutableList.of(feature1Reference, feature2Reference, feature3Reference); + ImmutableList.of(feature1Reference, feature2Reference); // Build GetOnlineFeaturesRequestV2 GetOnlineFeaturesRequestV2 onlineFeatureRequest = @@ -194,28 +133,24 @@ public void shouldRegisterAndGetOnlineFeatures() { GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeaturesV2(onlineFeatureRequest); - ImmutableMap expectedValueMap = - ImmutableMap.of( - entityName, - entityValue, - FeatureV2.getFeatureStringRef(feature1Reference), - DataGenerator.createInt64Value(42)); - - ImmutableMap expectedStatusMap = - ImmutableMap.of( - entityName, - GetOnlineFeaturesResponse.FieldStatus.PRESENT, - FeatureV2.getFeatureStringRef(feature1Reference), - GetOnlineFeaturesResponse.FieldStatus.PRESENT); - - GetOnlineFeaturesResponse.FieldValues expectedFieldValues = - GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(expectedValueMap) - .putAllStatuses(expectedStatusMap) - .build(); - ImmutableList expectedFieldValuesList = - ImmutableList.of(expectedFieldValues); - - assertEquals(expectedFieldValuesList, featureResponse.getFieldValuesList()); + assertEquals(1, featureResponse.getFieldValuesCount()); + + final GetOnlineFeaturesResponse.FieldValues fieldValue = featureResponse.getFieldValues(0); + for (final String key : + ImmutableList.of( + "driver_hourly_stats:avg_daily_trips", "driver_hourly_stats:conv_rate", "driver_id")) { + assertTrue(fieldValue.containsFields(key)); + assertTrue(fieldValue.containsStatuses(key)); + assertEquals( + GetOnlineFeaturesResponse.FieldStatus.PRESENT, fieldValue.getStatusesOrThrow(key)); + } + + assertEquals( + 721, fieldValue.getFieldsOrThrow("driver_hourly_stats:avg_daily_trips").getInt64Val()); + assertEquals(1001, fieldValue.getFieldsOrThrow("driver_id").getInt64Val()); + assertEquals( + 0.74203354, + fieldValue.getFieldsOrThrow("driver_hourly_stats:conv_rate").getDoubleVal(), + 0.0001); } } diff --git a/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml new file mode 100644 index 0000000..33d65f4 --- /dev/null +++ b/serving/src/test/resources/docker-compose/docker-compose-feast10-it.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + db: + image: postgres:12-alpine + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + redis: + image: redis:5-alpine + ports: + - "6379:6379" + materialize: + build: feast10 + links: + - redis + diff --git a/serving/src/test/resources/docker-compose/feast10/Dockerfile b/serving/src/test/resources/docker-compose/feast10/Dockerfile new file mode 100644 index 0000000..6efc3f2 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3 + +WORKDIR /usr/src/ + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "./materialize.py" ] diff --git a/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet b/serving/src/test/resources/docker-compose/feast10/driver_stats.parquet new file mode 100644 index 0000000000000000000000000000000000000000..df8cbba388827fe2e312921abf21e6322b3b8e27 GIT binary patch literal 34708 zcmb5Uc|29o`}b|0i8513G9^jKaQ1bXXEJ3%fG~KKF{@$;eYhQa^%Sy*)ivW{A)+T{5X>)-G%mPd+ z>F9a|mFQ?iH{^Sm>FB05Wo@KqRB2jBt zj5C1;x5G{XwlkYt2x?WBb`jsIcY^p|cV2NNe!<72-6WzNb9Ez_5>(wo5MRUTPDELl zCgR}zroBWA@I3B8BK{wqo&-(lG%o@diB;Z2q-%IX#Ir`NK14KrlE05c?+)zuCCFmX z_ajir7xgFNnB2bsjl9*fpNJCHr3XmVITLh{z$er!fPjljGLVRMrI0*_Z%h2-LD=)piaLcmWY$NuZS2U+Y(1asg~?xBG@r4r1&Tb)M4^q8+iRN@~< zC*o+;nG6zzYK3MJh~Kk3PSEPUA&ZC}i$96TmipiX5o=eU&nA)8m7_TXU-j*C2@1v( z@(5`DjQPZ$`a4ei_?)_v#Fv#xDj?Bd^R7aI085o30{&@^Vj?yjc|k-M_NG%rq$|lR zA<YXL%91=N4M4vqi zMCAI`T}i~cc;*9Uz3q(x*bGDX53fU1C35GY= zTp|d%A$ysK!WMst*fRC7j)-nSwO2^Q#CEKnpt{)UDuMZCr3Qk9c9v_z&$WL_eD$|i z8;L&|m3o~-F?@TP2&B$!yFt*a&eKdp|G^m|^19u;NyLUPCt674nB;Sd;Fl=gCMdfk zaEFNcy6=cM_prT{i0S(Z+eoDJ`(Qi4XqI6IL8#Q~P9lmoeIa72S$`K1J)W0!lZfqb zNDo0Ri)Al?RgqL55x;KwH=qS=_xp)R+fhA0qN%y)L4x?m9YX}Nyb5;-1}hlu5kFwt zzX9bRxI9e!CfCIKBy#!e@_>LYaq9@dSrLv=A{t+uBH}yk>ko;THJb5=L@GWWj|s+p zYK{>^9_Je;qQv@t1KQbmdxD5Qrg=|D#P!T?lAtb7_bGuLv&b_dE){$v;wgph=S0+Q zJvBw58Qa4z2$El$OcN+XNW3KCFwegM4Jsd+A)>I_*;gcL=?kAFaNA}5ntBx+ST_>;h+*Weccn~V5wBG!KVOhl_=eSe7fRj}+YJ)1&7O)yMWABM&P~Lzndd}|3~l5gq6AmQ3KDgm-pfnivsIIifUAdh zB@yeKUlY;p!>v_BT#C)*C($VZKLG;m^E!eAGc;i#A|?-iAfkdtmoO2BzZZ*;C@Afa zD1q==6ET98`nBRjbTeEeBGbg+Y9dzeuUJDObNcYL1Pi%V5(K%jGV6$_-twD>liNon zi5T;;W<80dLSm%|dN~|75croWN)zxZGs_Uaq3a3p9i8fBiT~?evK)!ZVs^_D=<};> zB$%t>QXpcw)=MHP-Mg`gh@d$$oW-tx3MiDajbXb=n@G1nvrU|+9AME;V0 z6-1NLFpY>Vos|@c=p3Vf;Ox9D5*Qzq*Cu$kl3s`SS(X1P2o=psy2KwFiq|7iR zAmT#QzX~FkZ_u8eS6%&Fxx>Hnc=Gj%wmZyp0(1hu)(Rv_CMjE zXLoJkRqx?)lVZb*Mc4es>i0eAzF2(y;8c5xn9-$EHxAD|IKQ*!QpwGbh3URxqsyhY zB9=bCdD3(F^qr%0j2z;|b!BbG*jCBA^wyo}Na9^@ddm1pc~_e7R^Q3qD-}J*CG=9o zP3q6~<;dFBxb)SZ8#t-7xBryM)ykn_^}}x``>s~qJFR_;W3_2R_5BLt9Qj@S4d+Ly ztSU@POs~~Eyx@4%_i6vN3y&|mb)>E~Ypfl=>O;@Os}!S6&&bLvWmOlW!^pZqLO)9> zR+pJmNXcWkE>@3q#aiun-Ys$Z?5kv~&RSiGGvE~3?ADpJ<(MJ2*tUR~;VZ|CR;0h)lSZ~%IciWwbUIO9WJ|dWviyyZ`yf1;MId`X%3rr--%&frIzlv zWp7W`2HVDTr>)-i$_#SUGMrWY9@lw}G-mAFcJO&?!m4eVE*ghl51+HWp1Dgak6Y2(mR@S z_t{o$KVz7undiIXg2NTBhs}9@4wrYgB?@Te`#W9rzR$oW;YJtBz``S8T;@i9jD=^7 zcCy4Ch6D~lMYrCvJ&Z{_YaoVgojX&?N@>e7<1_BeX@ZKbt;y^5vSf&=9hmApvzPVw z8i=N2m-JveA!(Q?VN&kFo+E8ps+}U~$&n{-f61+{+>`U9qU-G#_Vr#|h00!c%SeJZnu^@6A)98T?_Yuflu9X^8$q$06mzdqy{rXPxO;AHE91%r!cxQu|h(GtF1r z(|>l~sw&G;h~?Pe%YWXc((;VyIbVSb_Lp4SQaAVsUUY6e@S^{mpU`F3+tKu#(*D9% z+`BW^nN|9WT=lwJs*@(YU-X*qqf2`RD))RA* z%%f{Cj!W*4RA09A_Hy&{hc*o4EADPjmpd#yRHSxrdhq;VnR}&hl!045NcMiY;c-ce znjpE6O3Tx_8S+QuAJ*7kb|0!avhndH*E@0C8-o?b>%H!kGxKV<(lIb|iEC$Rw=uAA zi_>Cwb=sLY1Qc9avvfLGcvjOQ>G*Uz*;Z~aKc#&_w~Iq?lS@kspI$eYsH*>D>j}Lc zo;6hDFS?cby}XhJ>Eb%s`h6>TeQ`9j*iwPCqhqb#p@-3dI9eCOj#VBMsseb6yV2R#I%cq*b?A`l$k&TssS_Um9kv zrlOfdY-WtJH&~SF7uvis&EK@^cD#t~ta*{@{%4(qwy!Nqsi;3pqIPpu4SK!WIww{|r7B{4q^2eiP$)1E>ZPR zx^;2!?ELbMO)=|^C31^vhAz=b#wYVhnHH$WN+zTVC^$ET(yvcU7g6yYYl>Z;l)0J~ zzO+Oym7KNCOg~jvBTgzMdxLqAdKklo)Lc2c8pj)P8`APOxip9VV31BP*y7P&pm9t( zqe#_%qA83~Ci9d=(A%*a$7GI|QjyC`KNw}R%5>s6L^R`NPm~*^i)w^3$z`84$(46% zj+e`+v^b?1w#+1-TWx*Tv`{laKCfm+t@Dj==8gHaP7S`}%?TS%UfR_X{$rV0p`dP0 zXR3%+qC#Q4=U|aW1k0wPhJB+oPB#-b6*ummY!3U$qIjw)aHhXdD@n1W`N+G88xgFV zOIyOez8$}rwE6Vys9(!JezGc+wZ<}Wiqevm&a@|RiD^c%Z7J_e;os=olDws&J40M6 z{1=<@+1?XUW<|6V<#YXc3Ok!4*|$~>7OMD7w4`jUx?4hvSpLPXQayY|KTVWMRXP9Q zoO!Wk6o+cf=y|&f&bLxkFFd;Fax?rlhg$8}6_0@;DlK1d6PakJ6QbF33%T&k?hG9NHJ`d1&`OkN2xDn7xag)eThA@3&@i1{3a zE+R2hM-~k?oZ&!iD^8G^c7{KTPI&w1X)V>ap6GH{5NEFJ!#(~QsG=c*=el#?`J>&~ zpK=9`Zi&P-2ZkZ2_zkG7ipF*ImiWuy9P~TvLff;^c=+5PwbfGy?|)WAokM-VQ%i^2 zj{0HqHfL zcrlj-=0&eS$>;+W9?FZ7f^OJtWQ=o9D=2Aebr1~_!$F%efP^uye*Y3e!x^FL77fq6 zG@*VNYv3lQGnCbhjkuu2fX8!Ou(B`$f5=;cZm&CJoABb579~7-K!mE54#3A3SRq4g z11`<5;fRGGiaonUjpTAs%hGw^F&Tv?`+mVQc7NQxFB|fI@4=sTF;KPFADfgpsN=lN zz#XcB>GT#LelZ$(TWxSQu@gia1@ZIUZ&31S0k{`EF~>6kMnr_M=0Y%zv@_%DG*J}k z4MEA7{rGB)7-nAfLDh<<)Z%MCI6Z8FfA_nP@4^~YIW^%mGY9T{>xE{!!l^$g zp@v0lP_~$xy0PXDoIGudyLuXd&b|%yT{{gedSWbN48?9XsM1*qTSFx>Ni|Wj)OfS$yA$|EM7J6 zN7p7cYUGG8Hs^n)==48p-HO_PW+P10-TA#}YcvQiSmuGrvKDSl??tAX6o?UB2h$QC zXxq#_fl@6a{s=4s)p7}Vb1n!^=e(gJSO=K(g8Ga%>ch9;G-VRTpmrJ2HDMUFgt{YFE++ovG< z_Ac;Jv%{nL<#7GUPdIWQi;`E^j6ZKg;A$;52x07obFK22^tcjadZRTzu9=6i>w$PR zdNXxldn-&kv7*nU4pQ-|cqhvXI`{3jd9m+}V z#Ngeh;Q87P2+g2JznxD2X3xSlb1@M4$U>?6J_EjM2ax~6G9}k*h#LC$sXZRWu*ypj zOS*>OQFArKKoF*~?!@Silkg+g1|3E>qW}IlXpiN@Aigd5drdpga%SO3x*#q`-3Ke< z?HE>*0MCtcpz%I8?s~?9lDxL~MTrTYx>Zuz+SlRfJSVcAzXF}rrubp(C~h6^gD6=M z^gMY9@+SbF9J&tEyA{y(n-BIq3cxM0ir8kIO4V(VMCr%-@phdUIz*XJ{Ec!Hza~G1 z-fM+d+@@H~?TI7wN06Dm51g7~pmgyGBrjY6Frq{5s(Ar)ZqEb4KnJG`P&NAKeDsa5KFxoZ?@N?Qg1K zUR6@|JhJ$BnHlr6P4LK$Er@sA zk#UbVdbP8o+0_V?Bi8`=vr)L}h7Ni-xFO>$U38C|2amD|@N%C54!Jcbn132RsQ-pT z_UvdXCXDstI@opb8~l3o5e~2v(s-A2G1<%lpIy~KUwHt=}F~9=pSxWL`FwRPv;SNc2G}@to3c4ZqKH?N~ zS1;1;TRY)oCKK3SwuDQGbdV;w9X@Ir(+cPkxwi9I!8eoXIJbu|b272q1amsKM)YfK#?9v{5cg_%EnJr+GNEBqM zt-`nDeAf2bgqz-9fM*l3U?vp@Uw3Z7Z$GwR!9_9#^#`!PDT+FGC(C%o5q1Y!7 zsg_1b@`+&5BPH7N-P_RQiU`UFZp6(ypTox|6|j-38lEb~P_q7Swczyz5S#PC8u}27 zt-B5fG?*!BuPt1NAAtFmdf2#H0UzI%!BRU_xcM^;M%_bkUz!-UKQzR!ToycH!hr&e z4)}{=!i^^)(R`8>*avpb<#!(Zy#2`i_0ql;Jz`<@~&`8S%A)LoFCh!5sfeSlg3G?c~kzM{i;ZfShMkW6hE zLFk&|K@~?j+*{-cEMbSiHP{eugFM`rRS*jiFm5w{Ui(=^2M&)5qMVrhthaIO51Zo8?y|h@s71CSosIz z{Elo|%LP$%Y0*aq`8+sqdnJB!$%b>}d{P$WM>XwuaP<{LVfPHk6wpSS7;PA4ItD-J z5-BDzf9yE+gt}(?1dVg~kUYY?rttS-QKolP2b4Bgf(_nHr93S3T zk8fS>!zQf^sz1vQHJqDZ%?D-}v13A432khA6c0-q_M?5r5jT!Id=?J8PUfLZvoP7kpbpo(9 zwt{pqg-Q(uc&Ew}*J@3J=#wT$IUR&L4?HRQ!a)2o?g3Z(TVVhFWw@g*j(1NBVCIH6 zDrwsgj2{og9ckYnLM0L0_mQ=OzO~@IygZpgzAr>AnhK6kLVHC zP3w{QWDn-<3V_Ms2rL}FM@3&1MX|D#IQLKq^Y7}RxfvtYCG+A+sk$=!`mFKl~&>IKqEXn?}6iMoc; zscr#=_Bv3US&f!sfSr?}6l=c&4vgM~(`OZN>y`;Hib6DTazWO#NYJ_b3GU|Kg|Xcm zu(*{4ulCxZDZeO{bAbz^cC|8H4(v2xM$rLwWtY3~TQF0p@$|7@aZ&z_|kl8g`>m@>(1<5TyF< zjYGI<9cUb03syn*XeO2`Fr|wJ_r)Z_%tj%+-M1ID%eO^gmBZLB1+nh4!sMaNmv|O7@mg_h&e9(as;l_(maR_9pdhWdY6fj|UDabEmYJ|-ZVh$tfIbSc zXHvU3Hsjlh5@oVu6pUF5n*Y z2WA}qg4)A)a*bV&(pT@m3YA&P^kEpjE0V_3a`fOT!j5~cO2V@$AF84y2*p3~P!V+D zFrBp*%lL0lVdg=|ble+qs=S`9?(E~bLW96D+`L^*txz^mmVR4j)Bx^sKOsgN89 zS`0wlQ{}YJ=TAUXI1>(E_s666rr@9p1Gv{k0(S26uqK@rt zf1vMkI6T}bfI<=NkeMPun{Zl<)BCva;8_Py6Sl>dVjd`7O@{&>!eAg}6Ks1Yj(nV3 zv8U}YL|6-;ed${Kn{SFEsu!unD^=j8vC!W#@Rs^kd=7?EB2b= zra%ddpYej@ibfdslo9{>b-~oVr__9{KV>UK&Rg?5N`YpJY43 zntT7k<{lfc_i;uVS(`BN+mDT+W3YFc6ItZ_sIB+5;plq-{Hah5wMDa(L#Zqtim8IQ z7prhnr61;){2>>H7?|=Y0u9z$Sf@*2`d>+u%}SxJwH(A!?`C+ltq)cV|DsK-yit+I(uDhPm7mkRvgGUh7qlEL{a%kCk^VG+^9iVh-4d8xryxegQ z=zMfgYqbx~d}Bo4ympvF1LP^S#l&Abknzw#tOyPUL-Jn3>=}^ce;Ra+RFJ3gB*GPb(RINgZ3A$osu@f5UGGKRMG*%Bg;S!rPhD994yp{^8!D1~HW6}V*o<+d!_Y_$6 zO+fpfceL=T?P$40AC2A$VsTzMaKZx4dr)Vq;_nVhua08V4nPs&i~$y4}QJ^ z?$Vp^;Jgz4y0Hs_qYGeY6NTH-OyI$n3J~6H01hISz;G@Y-?Z@{cZV;Qy6?kIyEeG{ zb_q_Hcwpd?fmZznO)Qn$hc_B+@vlV>gjEVcWuP%$`%ntI(?_Tsr;V}RS_pMDxbPD* zxwgzLQ8F5O=(Ifq6`x1LuI6xD)cOoFpH?(U;U?VL~4?R7(-W@v_SWFGqAVv9>2)lf3G0%OViweisjuzjVD%RI5r zO74ZeRv4lV%NghmT8YcUDU{oeXxPLfiJNNwK(w0-UgWHYNU^8Xan}y$IuMF^L7Z^c z?IXl){RQIFrPPDNyP%WCh-%w}Fjr(2HmJx$&Y@r|`XP@r1;E|M_%PzgX}HO=1D`!q z#g4=lY8}r#P;Jn~>+xE^(6Rx?j<3bHYOEm1XNNXt$@Olh80B=@2(z{dLP!}GijQc6 zJI6J+blnZ3L^tC-z6P+o6NpXuitsYY1GiSM#doRmaKv98h8C|IDl?& z5I3^Wu;@tv+}p&DZU^YmICdi@io}4N>U&!F-)QPQBl!%m0+>FO2Puc8ky(KSFT9C? zcy&1}v^|Q0TO-l9t{RxC4`Jz-2AI1-k6wq(fx3}Mc{FA~-Y0pAvDplF_4A{jC@+>2 zj)VF+R%&Tj6;+3HP@K;LbNg8E#db|>=-Q0Q4G*EUgatJuHv{`8Ie7EkjS5WjKzm77 zTqkCP)U5;>*VBDik~R+yV@E)d%@tSwalrPt+n~Lz7c_r;fb2a@zYDu~@*YQSCdBZOU*0wpg_bin=CJ@3ctV99+KCF;Za6%Y1z`z?!TS`c5vtj6-aQO!8H<6n+ZTNk#6VEr3ZFk) zf_vo!kf2X$OjSMz1QSFpb5ex2mwHLN^I>9A#3Np@Lg$WlEH1hVv)0}oV>UJ93 zBi~RQoX=rcYzZ26B3_tq1eNNOAfI^TF=tr`n9wmt{X%ont_RlliM`q$vi_8Q=1 zoPt1yy;S)m1Nx7tgY!KyOY#VgwzMuRBuC;=rFZ&BQwzN3dIo)cg)USf_V8+%0YvHviazOA4}Qs zk<i>8Bs@o0j*yu;Y)1^+@rM{oP75qPu*%v8?r{ukM^j)aRO9s1>vCa zO-O%#k({GAM?Wo18t`G;I^IY0>T5(nHkB8!QEUdXcVAbz%UrVK^D zLEL&7xDwV0w?f#F^~7qFWju)X`y{pW)YBmpxz(5AqTr@|S{I&45wh4T>GNDyqJ>-3hfFWgF9DP0xGk+BEw%SqjzWNTn zZy$m~V#c({U*agmC66aph=IgKe;kriz>q{0O!BOPtE0cE!SR<6JEnjFne%Yv??pH- z<^mDNulB71lR!eeNPyA7aFp_v!KO91BjF=;K-TCaUWCJ>WmS0wsEuKc#@{`MFfT-U4Oe?F~jjcPTrDK%8p6 z1&j2u(E4y5uD|o6-IX&yhf^WQHK|LD&$HK%LNH##pI#nBTY^6J>3o?(<<(%(ulEt}u9gv>v|hs(?Lzs-RVe4G*sS z01Tyyv=^V0kR~OJYC8u?_&X?DQBw3Bde+b%G**+$9z{-)LAcn-|{F2ZWlOH}-p z0@!3(t^rDKbsW6_Rb5T2rAGmc>pY>i=2pQ7hYj$xA4j^%Ki zVi4|)VNq+1_G{vlx9LwdUeN1$0ibcIQmM6IN z*qWs&R&BO8-bC$qN#Pf4#>R0g^IPMk)nBk%CCBksYA47$zTj}|jT3BaO;8Md!Rf|! zOt@P+QMup+myhu=(MPR`YE3V=1Co!4&uS-Wj=kUs?LD?;u{8;nUaW{=iJtU)|O%!I=!llEkV{$C)K84n!nmOLEgSC z)xK$3pe{KTV- zl}eoqU&oiCW4(!LjcpkRLSKqau_dW@>tr4(cqu+-oTT}vEi<_3+RR z$PArXFTtK-Y^a-^T`(ghYm#DW-=3Y{G_ye|CB@uJH>YT9Mq0fu#WJ`(r*vsXMw>m= zDp5DLT=2P(W3LoL`_fz&+fUwJdbK%*J>88(ub^FcRw>ye-CeMw zpj&-*OIAv{hqPW{zvHZOL0`I;T1VmC(AlkJ>={0WdPNTkW>u<9GJNejiXJu1s@A1s z_YZL5dksgI`i@60 zcAWmR^cv|nvSL{D%NRuFwAoCv;siU(STyEzcvG|DrS;EnIL+w__h%)lb)Mk~o70ov zIFW3qU%s+%PG8paM5=vfx!{dC1Etgx>0bI3qT_Rh>is7&gF7qM{FpP+=E%-U)ITdJ z^2XTMG&?)L^Q^SS8xyP4?A%KIbMj7aOdb2P^BX(QDTci`PE_`F|W13U+ zsIyY-#v6-()SOeZ`c<0aZ!AOmb4nLGtKi3*?J*p=Wh@5Ox*~6_l1+2V1-q&ZHQrih zrRJWMHaKtU^wy@JKetk?>%3*yTiY^@ylO*(8k@qmcGaeNHTGRK_BY<{s7uYO^)k5N zJpR_csXy;haMuObA8#F6Ir8fg4Qky*<{f)Y^Xv1wYP~e(orY8M8!8Ph`Z~=!kM-v_ zHg;V+5H`PaisNKcx51@Dh4U_RrYD;pbzKU+F~4ge^<>Me!R7GrdDo@>leZVUE=T{E z-%ZC^(8^+17bo)0jm@l}U9h_@QRCen-n4>FX~Qe2PVd}>2MW5?y02u0z1u6nS=ei6 zSf5?^&O_F$u;0GBKL5r$Po=cNK`+CrMdRXs9@mKtXj{FyI~&=mvNTN85%V|DEtsqZC3KuzNh)ojSola(n{WW8Qq*1 z{}9|XQ1T(T=jO8?A3|C=OBWK2TBb!lhW473e$MY{nbr6hHk?-awbJO;Tc?lVV*{m& zjXk$Mgnf*d;yk_7ZFKu{;m621v(w9udTuY?_!xC`A?@_9S))74;~%4!22TH3?78#j z$H${|TxE2u#;pvZ3o&fwWeh^Stt^@gvApSJOftr89L@`I!h>Zj+j`r0!WWK7aGhZ@ zGHzd4v=A?Aeul%Lw_UJ#Awenq441cYhv>vYqWa(&o{-*-HOmW0+Fa$lNyeR$qMwqD z&C6Gw?Cq4+{FGvqUd~@-+$Hb)Db;bXT=05tmty#*G&ima;U42|<)TmNKIRpok9)h- znm=U(q*sW)Htx}!_>>tsSh41NZx1YgIv&GyR)W=}S6B3NRpN#Fx^g!7BLPcQ1PR%V|39 zYHe22;W*K+Wo#DJxp7*2^XA^BBSjNmE7gb2TZZ(HlrDd*(&nzQN-`ZS7yVXkY*Ax# zvVXKv^V@l=j2gQt(}y+A-)bC(YV5D~KfDzF?SdQk1;-xKNA*SDYJDs&I6v-x)Y$y( zVnD_Pm)E9`n{aoi`QGZF9jr-O-_q`Z|b$Ubm-*3Z$PH|rj?J;}yx#)X~*__4Y@W%ts7Ms7{TFAH@`P%IH^2GPsOGB5VzYjeBv;6%I z9Zy{htN9dz*itK-WnG-m;1rA2QX6k(UA&C>3yz&j?ZS8K61NS$;E7o3kl?wJY-B#Y zvUsUe*78cK!{D^w&804-%q!{M<}XE`EOo2jy^#R^u@A*8b-5!@A6CwcZx<&QF#f zHQl{-DP(Bg_2=^AR-VSXB#U?MVn4@vEgS1k4!!fz`Z+$F+1OBJ@!ogm&xx_Sjg8lb z-XDng`DBXcdQ*?ZheO3bC+94$H$NWw5Pb9J(}m3IEw3#;hCliFZ0YXx+uw&iM*sZz zoNh%^E34%~oY=1^w(U*rLU#i?M0n_=n`i081=#+xGmm}m!2`j&5A6x^{2%+&J`~Uk z#ud^_58kGiZn6*;{x2oq|56bCf8Cmmyk0a{aXSZ32j$-P*EKR6jdlEoW#T<@SN z2X*m*`%WDC`W_OGYvAnZ^~mvsY=cq)WZm3D%jmJeTIu~L$vOmLTItZRS&xdolL+-@ zS@7dp9UMMl4?G=xVCQlO>d!dBNbnGta%m#Jq6zFe_7^ICFyhywB+7H(2t1my!SaUH zD0YyQY=wJD-7WORR_M7^1v+rW0f%BZs9yaF z2YL!1e1*{xCv*UYU1k^m9(|rnyBM)Mwoxg7P-UH;N0pksBo1)zm^%= zXlgNZ)SLm!3{w<&8Ud3%hfs3^FBaTAfO&=CQ1VU_XCIuQO4~V6`KlETc^7LLznOyx zHU?^7F&c5Ul6vqk5ssB@AZ;B9oV((U2Apeg)npfC|7;507?@xRStg(74Z}y4AK*(v zB8>mK4DWs%*3<|R!1F>Z$SB1M0_yA(!s1`@l%J63x%Dg3j)GV7_< z_DjWPkS{DURvMJNw`*H7G8Ax(-zzy$`VcBa3w$=OLhQ<{TYhMd5550nK zhxfuXY4I7gm4R63R`~pr8Go+!!=LYe!k*j^JiKp_x-cS&rv)mhH;!!B!(~ObVQoPE zRnoZWxH^7|FQGK;`M`N}4UVS@Bd^#?+EiOC<<}SmGXqk{(7B2d?2ZFVM;ka6dkNI@ zoH1f66LO~+VfMzqT2o}3kZF)L?tbt98ckc_;A0+O|FZ|4>P~3wU3Zd-oIC<5hm7z? zMlimWy94?B7omA_7ba-dQV)$zK<_R_jDGM5*mv?^XM;9&M`yvsl?Q1ffpPHX-a+ht zNQYThB+y|+2fSJu1h$(l`0B6>CcK>mvHdpK%^l5^&ez=C$1y>2O;<}XG zNZGg{f72vf&*TEtoh6VP0pRs`6V_Bdf&7zE$l<1r#t$s1#7|tv^GtwtsDBP58O2fe z%{7?uDxv1I#GvVH2K9bi5m$s+*D9Z@d#!uk`_H_)hqPs({82R2k`}N6#l7-{!4Apd4drG z#j|1KBR{Z?jK+_QxlnehiegMM!>8dEXqx*48YTB5crk$>*)liobW8K7<+Rt?fXUX9@=o*g@#NP&}Zq4u#a4Kq)C7+D2?KiH92nZd!m; z@L{Z&8ltLq%Hg>D5qur3h7Rwyquh~IQ2fmvd>9|XGT37Q=TnM(w2I39Lx*0e0qB>f zi#z4HG3MGWXzyml*JSQf-p~g{**i%~g+g}VMA13)T@Z^QeQsl1wOK;zhf8 zOKR7fL+HFu9&a9Jhw=F_F#ehcC!|{7a<3fhfUw6)>qD0=BJHE zOLTBy(@u;N{SF`Niox8Mv=xLJq1z`AU-18ih^P-BaLgJ7_PXLy+&(n6md6xnMwC9Z z7FUpMa95J|pt#8_YDU?>T(41*XkhZF`&SAefXNMir!q;z?fbdJNWr< z|Le8LJDN;=KJyCJ8Er#dM;@r(&4q0MxvVTFhaO5IXL_x?w4{g^+- zF(=aa>GxrMN(ctsDTVtvDmeX$1J1APfrfpSIG$yTyKihkwHMYnXUziEyPg48sUv=h zUJHro;<#Cz9c9lbpkMk+D9z#D$fv7pOB)N5DZ{8#i5W zL{6_5=q4@FO@bkCSZ6n$CtC@{8-ZS-R;1qq%Nc)VsEZegE<&}#-*;UR>N8a1JRMI$&D zT&8}K?Vv49;iyDsg^Qkz;5H@%v@c?KqF06D&h!JR#dTO#xDAr`PC!@UFt8ow#O=r0 zVH1NL{*l*1y)+$g%QM6PS$?$Bj-g6lvtg-82(rvG;f&)C5Hyp+HLuv<3%dk-3T*=^ z`#W%8l7=>(gD{mQfbZXjp)Fae{h?Du6A|pCyk|$KRUf5b;9($cJ-DANpLW2X?_t=# z1F#@m2EW8-!-G@w$dynF=c_ZJK~NYAS$5{ZiYB~Uv~1tqx1R`H7^ z6z?q>>TTr4!0VsEPXcj|k`30+T!Nk6d|;QD2l~cufTt%LN@YaA_D}|Fqx6vTU?i&7 zKZU!~&)}S=G;Y4I1rMrFIACLqvq#2ZgkBOIm9Nu8@AUw?xdyJ%Dbt!%md7nW)zEJu z0Jg^I;`1>-T-r_MhSx_?G3y`>l9)wg zm_R)m{S6O|mGPlI4K;K&QUzLxluLjj-ZkHVC$wI}+Mjz-C1V3-y=Q|fKC&pD7J}Qg z1gODkCe&Xzg6DSyQs3NcQG;iI`fu%hXH*qQ7cEJ$1VID^1q4)rqzektb-d&(AUQ}* zatV^bgrK6BF@mUwm?P#aig^rRzfaeR5hnipCeK(-F@JT-XbB8l8WNg;>XNdDc97 zFiw%Bt`8@#s%AC}(_0=U*J}-%t)f9Kv6-A_C8m~sZeT74y3i_(BdjS`mP+JGcD~|%JB?)s*EotYVRT4bmVH@wl5OawO&`Cw zaGw`XVgbv?vc%W!v}~ahpgH9))!@o<0?6vZ zXSPiewk#i~>p;5yURz^}>6!o%0)Xn}t5rqLDKP(t}Am z*u^_1S%_Y_U0mPy?8`?HRvFBrvO_5>Y{V5NJB?3b^Hng-Qi-PT97zprfpk71jpi?X z#EvHSVv3m7z5eoRwlPhNj6?QuHRp}lvc%prze$H4CrZ=9%0x0%abVVa-RRhE6LQU6 z&z$5Jas4dK@s+O|JrWTi^A}wxLa`H9=4?R|wam#yH;KmC+0w{s@^q}ply2(mVM*)5 z>{_<_kRY@x1@C#ow94hE6W%wt>bztdmQLp)wwiDkmOWx3BO^&VyNReUiCf%1fMZV5+|IuHS%AzZu1i{Pl9+#$ z>D-Gat>e+`PQEz1X>Lgqo}6Mq!3m_CJ&DQBddN1^$kE_1Ia(d~np2$9krGasP^F0} zi*J%8g;Uy8G4M7Mi_N5?zlX4L6)iG+s6*v9)tQluIF*W?XKv}i)KqIkt)*FXSM!?P z5V0a=(_5LvaN&#GhuzOq}WH>Zk0MJLCF2Nn^Vj=CdY^fzL%<{z!HcwB2K5yzwaicBiM1Tf6vooCLU3fUa&*~o2 z+xM6mNe`x^mi}Z_tI4`9?nAaK<0&gif)pk`V;Vzk$bQc#%Fw>f&232~kGZ><^4%pY zQ?(lR>Tqf}`-M$zd&!c=M3AeB3u){m7GCs$t&i@<=ER7T%$_eS^GP0!z%;2r3p%o_ zT{2YXYeac(?sB{#uNd+5X!IEu+V^S-Gb1;e8e7I*gi5eY({$Jb<+W^zycTnu`G&a& zc5o}AM5)IM4O&0iogI-#B#Ck9RDZe)W#_%aRN(%UXXeG;NoLWAjTvzN@0iZF~v^gd1`; zuM)^_y$7AK5F-=s5~e)yDtpQtDLh4qw%Sc&1!W7_hZ$qo{YauMvxrT+bcv0BdWFpz zIGm2n%cZu7?khj#wshB_XXy`ZgHAnR zm)AdJqo!+MIg|nI_^d@6ch;~2N2O@dJUcFDm=rBuh-DEMZg9iBj&M7tDbe<|!ZkxzfUreNO(e=#njy{Fj zJYjAn`P3cnE0?_(K^vx2vAjt~S?{(?n%F~;*0-5Zhh0Ww>@T2{eF>Day(3-B>r3kf zA7(>uN3pg>Uv6oi6nfWp5!2h6$NENR+Rc}}!Db}~(#OjIY+83CI?syihQG2RNw+4Z zF=#I9wIhuw$lPHf2gFzpDM`AdW6MTOdd&o*bIAB~1sjxK!j0M zJtG#egije%C@M)`_S?{n!e{LB!>cS@CX62ZUe8_gox$vSN3s>K53|Kl{-mjI!nAoA zG;KjKim@5kv}#BUwhZy2Aietn(!{@%pe3{>dJ?E&=O{tKu1{s8MX z!;mw(eUWLD^d##kJDHNJ18t}Yq$gO)$?m969LQA@{KHm7e>^euOV{UL; z2J|QH?RnO`9G~Cp?nfJ)f~h32iFKm|+@U;&)Bs>fLzE z7|Tj|SQ?fU!{=Hm7P6r_hNO8tjt1!u;l|DzN!jyPv9k^l6mae#mKC}X3*BmWBiw@? zn8?zp>8~)gE0E59Nu*CY1yp$YF>}&&qYJmR$#BOqcBWjBo}HDUbp5k-zP=^Q;?gFj zhG`5X+wJJ4f+5{dSHtuXMUvG1#6rvuvyRUy*oeNe6oDnFuM`Z}l>xJvbJc8S@eX6C zE63O}dvj8pqeCVx6WJEe=Ulkser6GyO{wumSccyX_NJ2q!rcWY&v^MWX@Erl}X z&Edw}=2Oa=V9Gf%jd}JEp`{x+c4A;POE15S<-Rd=z9o$;yUEeLCBxXokDa(T?gnht z2}L$xwJA$)97^Y+`;(zyB*hjPkYU*n3b}NYGo2bk-EYRyn8BX35bxQhUDcvPwYZ3Ee4 zrH*zYFOxDmpDnqpbvEAnWEL@xxoYH=CQp;*45T~Oa`b7mBRh@d1yyN`;@4vNr2Bie zS2l|qqThwWUbS&z`TN+#W6qd5E=r|mR@=!=uVY6N$Ft(f2qyRPGE2`INnZ7L8SiKk zZSUesZ)2Y^|Ib^P#r!ZzaEPQgcKxuF(18|Cx1psQrZfMULuun|0qLE5&&H)}j7cyDzgFzhWiVQQ;M9n6jDM zC+I*^x8An1eym2G-S;t3&lWb&0n2>ShETVt912y?CXZYN60_AM{X@8REY9S5-AtkN z8orcmyMjsH89|*d`_cBLrEIn6E9T~rKuH>-X%oiF!{7hT{8V}5x>*ZT?s73!F{f^Z zgQ;-dZdN2~K zPae&De2^{e{EEE}97cDVcq~SYUl1u z220wbjwVyr6}9YC*GRUi_8r?9f0(sAXkaUX1SGGUNQQV%c9W+`JFyHVa8VGgoNP^2 z?yk(Uv3@|8vqPAqycQ?^DuYHCccOh0`Z90D-fV3ZF>d@e#%o-`##W25m#zbti^ob9 z#T!X&3G3K(dwDi^cnceqU`fx*RH?@LG?Q@c%z`|IuzTCIn0EP7wr=kLnx+^+)3gd{ zXIU8OV5z{H$~?AYJ(e3E(4!nl4X%@JG&!Bi#AloxXw!$=%(XI*mF`cX&`Luxcwxvy zI<>M*QLfYl(-JEcqG&SXN-p6J~%x6QIcW|CErc@pDh}C0ChB5=#h@4~0@=P&losGo?^_Yg*wD!Y+&qrmcR(@$MOLLJOFgrql&F#_1 z=9UFe;It45uei?m&8x$Q@F!!Z^ym!W_!QudTBcA7_CK(kNsG|hyp4Ye3{+5YDpUSPU!O0iQFd332b>@30w5x5O*fq zlb6`=hQUm=-Aj_%uFl3BxdmRo+zfK{W%2uX+ZePscCJNo!?q|cK#^e{$JDH zYBbIf6%nxpF5!R34Z$FCBcKQbLmLZ(Vy(kmpcwIT_!Z#MqVg~I%0t`%VIPF)u;0Tz zjDo#jUjW1aTf`PX6Gbc%)M>A(W`eUP^bb`9(qU}N~ous0(w9=-vv4*nMSClDJ0lmaQxC7?G0 zH{t&VECq^z@rWG*8d0VKJR6z?Y+dj>AOd&}e-&5{=zuzske3eb5B~&W8-YQ93jDLs zh5rgL9sU*gaeylFFM;KO9H2X(3#|(9guM{h4D1Iip>0L^a{vK+XwKl(U_R>X2TX-u z1hxf-K#u_5fqxUM0-i67p`Kod1w$)^FFZF*z)FAv&=v6m&s6kt%W@Yyc-w+90jzI-xWwhP6pT;+AQEca&G}i&}#rcXl`KqFKLlT*qy;; z;Pb#PNIj~Q{4n)o{ z_#I)Jfk%M1fL(zQU>E#D;4?rC&;~T4>=5Xa!2Q4vz*@j**qZ=1paprZs5={627fK= zonTkwX@Zvnd!P-0{}NaZTmUSg^MO7#aRe;@o;;K3-@0(%$qVX)r-(_r5~tPju~_y{CJ zj|9g7zQ`X0UJjfDqJ(9EFz7dscN*Fhuqq&coeg~!;?l6!1H<5d0y{z*fSi>;Ewtm% z4#DmMn-5zHTmh6I?+Ek;a18v9u=fgcVP6&I!|sR}4}K7AAK1&FcLkpW-T+sj&jL0g zCmsGiVID9MSOA=XCI_xVIeplXfD8P1FoSjqBSUF$-G{bcPS^#(WHpufwjuxN@m_mO5=tB<& zp8#h16=fgJ{jBBvX)F3^U?*Jg@*hL1g+C8)h4wqN0pLAA9B>V~BQOK711=)I24&vDo`d*# zAP;s&Xt9V*05pL{U>dYs#P$fc0pA{Mg#2dkD(IomZ2*u6 zZ3$2W-3$6~a3-LMm>l#?utmTf5c7q$4Lkwd4Vne88Aya*1N|Ow8MYp@Fz^bX6N=x3 zzY+E=*sEaoKwb`fCHUiD?}nWKdnvdH*b2=CNCZB>_e1PD$^`I2?#-K0JI#~<=}cC1@?UK zUF1Cit|Mn2w1Kdzp)ZE522B$7DL@Ur06u<24kJe!tOV8r#zI^Jv;jyH*#<}ea=;Q` zIC4HC)(c31-vB=eiR!Rt0M7tf;2iW1$hiyq4EQ2og**dbGVD*l8raT=7lBWMO;9El z_8Fij;_m=)*n7Z+h+hN7z~2kr08~TY1>n>5`ktMS_x#JkZ{ff~U3i7UFsQf0pB{u8 z$}g`=BP{&4*RrEUB(&>ABvhm7MWyws>&4`GH|oU|2P%z`Q1S5@)1gb$^f8j%G8)H7 z^(ay*mhL&-r&y*}_4Hy{qYaJ4a;Ei4CGwVyJ|!Ld-I!jYFrclmM3F?4$0|7}`Hod~ z@h@`h^Y@!_t5 zv5k^l(sqkVbscupTuLkF;;>TX{M(C6LrhyXnsfFLpDi6dZ8twQz)(d+(ab_OcosX6cUWv%I>z%y6;K23e!E+n>uC z*B!NxGud)6Q?Boh`}X_$#{njY)3tx=WCw4XyH&T`4e!@3@s@bBAR%l#qHv}>`v}|xW#AO zjkl=;i)d2wT{r`MZ{s*PS^H)hTB9`Nkk%t73zS_vP%jFqad zlT(x`E0Q1PFMhT6K~WV~{X;YSJj~Cl%6M}f0{s2FoB{#@BgO>;hNj->6!bj%&iSCI zar(`{akJyqtP&T`o)emKX_Hi#!j>&l!&GkHP|;7(e>KNBgHxFsp4AjEHzMz5{J{3ZhmxR}GLyr>v7^<`-@F(*-N~RL zV5VuqyrY6^V-$Yz&aI8>3d4G<)*0{ruSJftAUHO?qdN zR}7m`IdsHYNtu+jJ6>Pzx;sfMOt9=&!Ud(^t9mVhxC-$usWs*S^HZ;0aaK;#zqw?h z+KvZXyQS|*Z4HYIuYTDrW51WSrtfthp60MaE6oG=heNV8GmrJzR+U<7exWMsw3Bm( zevNg?my(+zw6A7MU2=}dX)egt%Kb2F*46%(Q;$UCb;~cU&s)C5S}FhL#D3iiN@lyv z4-}i1w@{%^^|lEy4?b1P#@ViTv%m21mInTC;~huN4{saO`MTcolP=eXd7MkUK5~t8 zY2>K)Ph4b2Kdh+1Qz-)cFBJBDb#GDDIk)FMit5{xs*A*r4~!baTN$ezC#5@2TVA&B z#iJtmW*4jb7>#VbQKFQlvRLzmn|kwD)rVUTcW_!6GWS$;67YO>|wL&pd3nlDbD((kZ}yj=gv@__i;2WR#ywUwM8S2*SI(ObG5UvH1< z%06$ukUdaRX36vr&$x!^sY)w5Yxn5vcDRePZkAqmSA#x9*>!zZN>28$D453+Uv<8A zrpeS^Ei-NH3od8)y4x%*@vre&I{RwGJB3-sF%|L3GD#ZO=Y+3nj`tdp(AsI%%?VDy z6I1%M-I2Ldpww|*bftlTzx|YBIu*vp{GBT9Rb0AVp?6}FY(~ z9!Xpjabn}V5uTA+K<+lopY30 zTX=4$QQWrHox#pK{A?DjbDy={Xnj|KYTSB*GKVSa=S94DG^zf8a{PuT@jSC-d3^WBi3!`&yj^_u;lR4}6|K&d*bL_aL9?s#7#8G{preR>bc=gCz^2akI_ z)g;jNgYo7jZ|$|?vNPS>C2gw9j%+>`xH?IySvM+4_fYsGhvCVt&bQnoh35}S51vvcZcrCkue(8nH+Ae&O^KBv7wRYY7_BPqb$j=-wH?pJL&V>zX z*V@`*_t>vVJJ>1Fer$ruqy3M34SfY?s3@R(vfqkizT4E!a^ud)nY144pPO~3%jIdj*l@Vp(U!?`yJd|~Gd!1F)upS3 z=;6{MSsd?FSr4OE`Z=G=#WS;Q#)se96u9pE(=ECxYi8~$o{`;jRJ^C#MtgnAn0l<& zqUvl2Zc*xUB=*k!;}^$?rAW#Z$rP#INGUr$FGBvz5~JlqZ!5pbwbYXu zG}-9EvVFy4w5*KZ)O(~!ovU{I?e6=|elZiC+{;xO z2bM(VCPv0ZCkqN9von)3vc4{k-WaTQFxX)4R&j&cVsU%`{pTp@r!mtnKY&*1tm@fW zto_qxY0ds23DzNa;u^+cYMs0OI1-coBc%S_Lhaw}KUECKc1TicN}7jqWUu z$FJy%{p|@a`l_Cjl{JmgI*d1k34cBiMWnN|(Eq;h{Hy={(|)`y6V~_B0r*+$IID0*sZ}+LtKto* zvASeNbXr_=mikWV;kwCbaq;<)vFU=G)U+&JTkWvnx+&0L>1IZ!CC2L-YwM;(r^Z7_ z%Ml3R;h{&F*>KI;?eq+o=Kan3wR`dH-pB|zU#pC4uRs1<&3~;ZAvsZw08a?$DlAr* zp4Psb0sanXs4;fnTW{!of8x^LTlJUFPwh!g%eLnIy(!=2A395TH2<>6@7J)uZ}Kmp zpEj8wNRQ68wD|i*e+c}i&HnMl=-;>cW90w!M*n(8{KdBap72u>(5JuJ>+kY^+UIY# zv%hclhrmyp`{Qi-dq4jX`Q>@mew5o^j>0==dS?5pM35Yl8J$@me34`nL}zBE=Lu`Y z6TY&N)6;}1udi8Ob6uPry5CyyL(IIdWnVMhks|8yd_TWILAGCSo$70pOoc!Bi7tFT z-xm=6c<}i=zI`I!na@w;J0!Mi_V#~?=Ltop?{Qb5-HvZr?Qu6gKaY=cU|}WpZm@+u zpYNECeEWWGse+tPmq1>yg?Da@YhZRvnpg6mWc!%-0{6`5AgdI&q3MZkY3@lesc{0g zaFN|{u z%oEn-6>J_S%n7lu$%+dy&k45py8Y1L@7v7@2{IQ5b7Rs1vtk^C+YHSJ4a!dmw)n-i zU7XQ==fYr1KdV?5VY~RraY0rYs6Qoiu$yq(Xs3Bze4ve4e6W3cy>6*C$r%*V_#HlJ~XWe%^=wkeg z=fn^D1-2fgr}k zFCiw?Is4nON(!;?{;~c4yS4>~rUhI0B}E723y(oTbg+HlKONhUf6D#GWBkwkC_Khq z*i!qs`+s+g3qpf%e1pE87r*Gkp|Pof!q>&ubMu#N`tx=3kH;6UeOv=Fa9y+TcFxX; zH8=a`HU^pruPNWQYf~V+PT{=FiVkuw2nlw#`g*;zw}M>|V;S3CkMQ*qYaz&uNfy3F z-ON0P@^icfyW^a}we$P+D^w7h=A98^!E3+fcn#(iA}=`scLdxA_J5@ZRlt+AW-{!jbUUg(GYaS-mBi|}=TV<51}2yHKz$aieNZ^jDi`}Tv|F@DgW z0mAz!jstpK*icmVy;0x#`|C}+z3YVue7>-6?FBd&a^jqA+OOln`({iqUs%7_w|0N+ zYIO0p{s!~;!fUgT@xzIz_sEFO9F`NGZ4#T2VbVSZ(f&5b&~6`nXbA9a;!nRK>lN{Z JJm1g!{{WF|k&FNU literal 0 HcmV?d00001 diff --git a/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml new file mode 100644 index 0000000..87e3310 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/feature_store.yaml @@ -0,0 +1,9 @@ +project: feast_project +provider: local +online_store: + type: redis + connection_string: "redis:6379" +offline_store: {} +flags: + alpha_features: true + on_demand_transforms: true diff --git a/serving/src/test/resources/docker-compose/feast10/materialize.py b/serving/src/test/resources/docker-compose/feast10/materialize.py new file mode 100644 index 0000000..6338c16 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/materialize.py @@ -0,0 +1,45 @@ +# This is an example feature definition file + +from google.protobuf.duration_pb2 import Duration + +from datetime import datetime +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService, FeatureStore + +print("Running materialize.py") + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +file_path = "driver_stats.parquet" +driver_hourly_stats = FileSource( + path=file_path, + event_timestamp_column="event_timestamp", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=Duration(seconds=86400 * 365), + features=[ + Feature(name="conv_rate", dtype=ValueType.DOUBLE), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), + ], + online=True, + batch_source=driver_hourly_stats, + tags={}, +) + +fs = FeatureStore(".") +fs.apply([driver_hourly_stats_view, driver]) + +now = datetime.now() +fs.materialize_incremental(now) diff --git a/serving/src/test/resources/docker-compose/feast10/requirements.txt b/serving/src/test/resources/docker-compose/feast10/requirements.txt new file mode 100644 index 0000000..cb579a2 --- /dev/null +++ b/serving/src/test/resources/docker-compose/feast10/requirements.txt @@ -0,0 +1 @@ +feast[redis]>=0.13,<1 diff --git a/serving/src/test/resources/feast_project/data/.gitignore b/serving/src/test/resources/feast_project/data/.gitignore new file mode 100644 index 0000000..6336181 --- /dev/null +++ b/serving/src/test/resources/feast_project/data/.gitignore @@ -0,0 +1 @@ +registry.db \ No newline at end of file diff --git a/serving/src/test/resources/feast_project/feature_store.yaml b/serving/src/test/resources/feast_project/feature_store.yaml index eea3d61..e276cfe 100644 --- a/serving/src/test/resources/feast_project/feature_store.yaml +++ b/serving/src/test/resources/feast_project/feature_store.yaml @@ -2,7 +2,6 @@ project: feast_project provider: local online_store: type: redis - connection_string: localhost:6389 offline_store: {} flags: alpha_features: true From dfff7b9a1eb1815cc14e8bd4d2c6d5598099cd43 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 15:02:53 -0700 Subject: [PATCH 24/39] Remove github action Signed-off-by: Achal Shah --- .github/workflows/complete.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index aff0b4d..c33ff8f 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -66,13 +66,6 @@ jobs: with: python-version: '3.7' architecture: 'x64' - - name: Install Feast CLI - run: pip install -U pip; pip install -e "deps/feast/sdk/python[redis]" - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%Y-%m-%d')" - - name: Apply and Materialize - run: feast --chdir ./serving/src/test/resources/feast_project apply; feast --chdir ./serving/src/test/resources/feast_project materialize-incremental ${{steps.date.outputs.date}} - uses: actions/cache@v2 with: path: ~/.m2/repository From c544ea19192a9c049d69251db83fee68e8362550 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 15:30:39 -0700 Subject: [PATCH 25/39] Add registry Signed-off-by: Achal Shah --- .../test/resources/feast_project/data/.gitignore | 1 - .../test/resources/feast_project/data/registry.db | Bin 0 -> 997 bytes 2 files changed, 1 deletion(-) delete mode 100644 serving/src/test/resources/feast_project/data/.gitignore create mode 100644 serving/src/test/resources/feast_project/data/registry.db diff --git a/serving/src/test/resources/feast_project/data/.gitignore b/serving/src/test/resources/feast_project/data/.gitignore deleted file mode 100644 index 6336181..0000000 --- a/serving/src/test/resources/feast_project/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -registry.db \ No newline at end of file diff --git a/serving/src/test/resources/feast_project/data/registry.db b/serving/src/test/resources/feast_project/data/registry.db new file mode 100644 index 0000000000000000000000000000000000000000..774b4938e71d16cb5c9d598c26500e528c86a075 GIT binary patch literal 997 zcmds#y>1gh5XZeva=8paK5ZllS8$|2mV9@1Y~oA9M*)Zy(a@}x-956>ePwU&gq0#L zj06#);VF0kT3!GOBp!l>h9)8iAvTVI7_OkE+L_Vp%zu6}Fn|EoxRRAnL~>`jJ$Eij z?*5M7J(sI2$u;4b8si3BflXK?gqKM&c9vY2o2J4anQhc_H+$iTvi^3^4To&h9c*0< zyOcg0b@^b}4~M~Mz{9Y!Zhv|5;m~s4KY#HK_1`w(JA#e5u-&Xw@t72v>H?pY@>8v8 z?4E};mZcR@R10UtT?d;ocIsK~2DE7Ph;S-R9j#iZek5q2{WDm6PBr!Cif2;2oT|N2=tI<- zWqUG>6!o$wie)OyIGGZK7s=SzW Date: Tue, 5 Oct 2021 15:34:33 -0700 Subject: [PATCH 26/39] Remove redundant stuff Signed-off-by: Achal Shah --- .../serving/it/ServingServiceFeast10IT.java | 2 +- .../feast10}/registry.db | Bin .../feast_project/data/driver_stats.parquet | Bin 34708 -> 0 bytes .../test/resources/feast_project/example.py | 49 ------------------ .../feast_project/feature_store.yaml | 8 --- 5 files changed, 1 insertion(+), 58 deletions(-) rename serving/src/test/resources/{feast_project/data => docker-compose/feast10}/registry.db (100%) delete mode 100644 serving/src/test/resources/feast_project/data/driver_stats.parquet delete mode 100644 serving/src/test/resources/feast_project/example.py delete mode 100644 serving/src/test/resources/feast_project/feature_store.yaml diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index 691cf19..ad25561 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -55,7 +55,7 @@ @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { - "feast.registry:src/test/resources/feast_project/data/registry.db", + "feast.registry:src/test/resources/docker-compose/feast10/registry.db", }) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { diff --git a/serving/src/test/resources/feast_project/data/registry.db b/serving/src/test/resources/docker-compose/feast10/registry.db similarity index 100% rename from serving/src/test/resources/feast_project/data/registry.db rename to serving/src/test/resources/docker-compose/feast10/registry.db diff --git a/serving/src/test/resources/feast_project/data/driver_stats.parquet b/serving/src/test/resources/feast_project/data/driver_stats.parquet deleted file mode 100644 index df8cbba388827fe2e312921abf21e6322b3b8e27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34708 zcmb5Uc|29o`}b|0i8513G9^jKaQ1bXXEJ3%fG~KKF{@$;eYhQa^%Sy*)ivW{A)+T{5X>)-G%mPd+ z>F9a|mFQ?iH{^Sm>FB05Wo@KqRB2jBt zj5C1;x5G{XwlkYt2x?WBb`jsIcY^p|cV2NNe!<72-6WzNb9Ez_5>(wo5MRUTPDELl zCgR}zroBWA@I3B8BK{wqo&-(lG%o@diB;Z2q-%IX#Ir`NK14KrlE05c?+)zuCCFmX z_ajir7xgFNnB2bsjl9*fpNJCHr3XmVITLh{z$er!fPjljGLVRMrI0*_Z%h2-LD=)piaLcmWY$NuZS2U+Y(1asg~?xBG@r4r1&Tb)M4^q8+iRN@~< zC*o+;nG6zzYK3MJh~Kk3PSEPUA&ZC}i$96TmipiX5o=eU&nA)8m7_TXU-j*C2@1v( z@(5`DjQPZ$`a4ei_?)_v#Fv#xDj?Bd^R7aI085o30{&@^Vj?yjc|k-M_NG%rq$|lR zA<YXL%91=N4M4vqi zMCAI`T}i~cc;*9Uz3q(x*bGDX53fU1C35GY= zTp|d%A$ysK!WMst*fRC7j)-nSwO2^Q#CEKnpt{)UDuMZCr3Qk9c9v_z&$WL_eD$|i z8;L&|m3o~-F?@TP2&B$!yFt*a&eKdp|G^m|^19u;NyLUPCt674nB;Sd;Fl=gCMdfk zaEFNcy6=cM_prT{i0S(Z+eoDJ`(Qi4XqI6IL8#Q~P9lmoeIa72S$`K1J)W0!lZfqb zNDo0Ri)Al?RgqL55x;KwH=qS=_xp)R+fhA0qN%y)L4x?m9YX}Nyb5;-1}hlu5kFwt zzX9bRxI9e!CfCIKBy#!e@_>LYaq9@dSrLv=A{t+uBH}yk>ko;THJb5=L@GWWj|s+p zYK{>^9_Je;qQv@t1KQbmdxD5Qrg=|D#P!T?lAtb7_bGuLv&b_dE){$v;wgph=S0+Q zJvBw58Qa4z2$El$OcN+XNW3KCFwegM4Jsd+A)>I_*;gcL=?kAFaNA}5ntBx+ST_>;h+*Weccn~V5wBG!KVOhl_=eSe7fRj}+YJ)1&7O)yMWABM&P~Lzndd}|3~l5gq6AmQ3KDgm-pfnivsIIifUAdh zB@yeKUlY;p!>v_BT#C)*C($VZKLG;m^E!eAGc;i#A|?-iAfkdtmoO2BzZZ*;C@Afa zD1q==6ET98`nBRjbTeEeBGbg+Y9dzeuUJDObNcYL1Pi%V5(K%jGV6$_-twD>liNon zi5T;;W<80dLSm%|dN~|75croWN)zxZGs_Uaq3a3p9i8fBiT~?evK)!ZVs^_D=<};> zB$%t>QXpcw)=MHP-Mg`gh@d$$oW-tx3MiDajbXb=n@G1nvrU|+9AME;V0 z6-1NLFpY>Vos|@c=p3Vf;Ox9D5*Qzq*Cu$kl3s`SS(X1P2o=psy2KwFiq|7iR zAmT#QzX~FkZ_u8eS6%&Fxx>Hnc=Gj%wmZyp0(1hu)(Rv_CMjE zXLoJkRqx?)lVZb*Mc4es>i0eAzF2(y;8c5xn9-$EHxAD|IKQ*!QpwGbh3URxqsyhY zB9=bCdD3(F^qr%0j2z;|b!BbG*jCBA^wyo}Na9^@ddm1pc~_e7R^Q3qD-}J*CG=9o zP3q6~<;dFBxb)SZ8#t-7xBryM)ykn_^}}x``>s~qJFR_;W3_2R_5BLt9Qj@S4d+Ly ztSU@POs~~Eyx@4%_i6vN3y&|mb)>E~Ypfl=>O;@Os}!S6&&bLvWmOlW!^pZqLO)9> zR+pJmNXcWkE>@3q#aiun-Ys$Z?5kv~&RSiGGvE~3?ADpJ<(MJ2*tUR~;VZ|CR;0h)lSZ~%IciWwbUIO9WJ|dWviyyZ`yf1;MId`X%3rr--%&frIzlv zWp7W`2HVDTr>)-i$_#SUGMrWY9@lw}G-mAFcJO&?!m4eVE*ghl51+HWp1Dgak6Y2(mR@S z_t{o$KVz7undiIXg2NTBhs}9@4wrYgB?@Te`#W9rzR$oW;YJtBz``S8T;@i9jD=^7 zcCy4Ch6D~lMYrCvJ&Z{_YaoVgojX&?N@>e7<1_BeX@ZKbt;y^5vSf&=9hmApvzPVw z8i=N2m-JveA!(Q?VN&kFo+E8ps+}U~$&n{-f61+{+>`U9qU-G#_Vr#|h00!c%SeJZnu^@6A)98T?_Yuflu9X^8$q$06mzdqy{rXPxO;AHE91%r!cxQu|h(GtF1r z(|>l~sw&G;h~?Pe%YWXc((;VyIbVSb_Lp4SQaAVsUUY6e@S^{mpU`F3+tKu#(*D9% z+`BW^nN|9WT=lwJs*@(YU-X*qqf2`RD))RA* z%%f{Cj!W*4RA09A_Hy&{hc*o4EADPjmpd#yRHSxrdhq;VnR}&hl!045NcMiY;c-ce znjpE6O3Tx_8S+QuAJ*7kb|0!avhndH*E@0C8-o?b>%H!kGxKV<(lIb|iEC$Rw=uAA zi_>Cwb=sLY1Qc9avvfLGcvjOQ>G*Uz*;Z~aKc#&_w~Iq?lS@kspI$eYsH*>D>j}Lc zo;6hDFS?cby}XhJ>Eb%s`h6>TeQ`9j*iwPCqhqb#p@-3dI9eCOj#VBMsseb6yV2R#I%cq*b?A`l$k&TssS_Um9kv zrlOfdY-WtJH&~SF7uvis&EK@^cD#t~ta*{@{%4(qwy!Nqsi;3pqIPpu4SK!WIww{|r7B{4q^2eiP$)1E>ZPR zx^;2!?ELbMO)=|^C31^vhAz=b#wYVhnHH$WN+zTVC^$ET(yvcU7g6yYYl>Z;l)0J~ zzO+Oym7KNCOg~jvBTgzMdxLqAdKklo)Lc2c8pj)P8`APOxip9VV31BP*y7P&pm9t( zqe#_%qA83~Ci9d=(A%*a$7GI|QjyC`KNw}R%5>s6L^R`NPm~*^i)w^3$z`84$(46% zj+e`+v^b?1w#+1-TWx*Tv`{laKCfm+t@Dj==8gHaP7S`}%?TS%UfR_X{$rV0p`dP0 zXR3%+qC#Q4=U|aW1k0wPhJB+oPB#-b6*ummY!3U$qIjw)aHhXdD@n1W`N+G88xgFV zOIyOez8$}rwE6Vys9(!JezGc+wZ<}Wiqevm&a@|RiD^c%Z7J_e;os=olDws&J40M6 z{1=<@+1?XUW<|6V<#YXc3Ok!4*|$~>7OMD7w4`jUx?4hvSpLPXQayY|KTVWMRXP9Q zoO!Wk6o+cf=y|&f&bLxkFFd;Fax?rlhg$8}6_0@;DlK1d6PakJ6QbF33%T&k?hG9NHJ`d1&`OkN2xDn7xag)eThA@3&@i1{3a zE+R2hM-~k?oZ&!iD^8G^c7{KTPI&w1X)V>ap6GH{5NEFJ!#(~QsG=c*=el#?`J>&~ zpK=9`Zi&P-2ZkZ2_zkG7ipF*ImiWuy9P~TvLff;^c=+5PwbfGy?|)WAokM-VQ%i^2 zj{0HqHfL zcrlj-=0&eS$>;+W9?FZ7f^OJtWQ=o9D=2Aebr1~_!$F%efP^uye*Y3e!x^FL77fq6 zG@*VNYv3lQGnCbhjkuu2fX8!Ou(B`$f5=;cZm&CJoABb579~7-K!mE54#3A3SRq4g z11`<5;fRGGiaonUjpTAs%hGw^F&Tv?`+mVQc7NQxFB|fI@4=sTF;KPFADfgpsN=lN zz#XcB>GT#LelZ$(TWxSQu@gia1@ZIUZ&31S0k{`EF~>6kMnr_M=0Y%zv@_%DG*J}k z4MEA7{rGB)7-nAfLDh<<)Z%MCI6Z8FfA_nP@4^~YIW^%mGY9T{>xE{!!l^$g zp@v0lP_~$xy0PXDoIGudyLuXd&b|%yT{{gedSWbN48?9XsM1*qTSFx>Ni|Wj)OfS$yA$|EM7J6 zN7p7cYUGG8Hs^n)==48p-HO_PW+P10-TA#}YcvQiSmuGrvKDSl??tAX6o?UB2h$QC zXxq#_fl@6a{s=4s)p7}Vb1n!^=e(gJSO=K(g8Ga%>ch9;G-VRTpmrJ2HDMUFgt{YFE++ovG< z_Ac;Jv%{nL<#7GUPdIWQi;`E^j6ZKg;A$;52x07obFK22^tcjadZRTzu9=6i>w$PR zdNXxldn-&kv7*nU4pQ-|cqhvXI`{3jd9m+}V z#Ngeh;Q87P2+g2JznxD2X3xSlb1@M4$U>?6J_EjM2ax~6G9}k*h#LC$sXZRWu*ypj zOS*>OQFArKKoF*~?!@Silkg+g1|3E>qW}IlXpiN@Aigd5drdpga%SO3x*#q`-3Ke< z?HE>*0MCtcpz%I8?s~?9lDxL~MTrTYx>Zuz+SlRfJSVcAzXF}rrubp(C~h6^gD6=M z^gMY9@+SbF9J&tEyA{y(n-BIq3cxM0ir8kIO4V(VMCr%-@phdUIz*XJ{Ec!Hza~G1 z-fM+d+@@H~?TI7wN06Dm51g7~pmgyGBrjY6Frq{5s(Ar)ZqEb4KnJG`P&NAKeDsa5KFxoZ?@N?Qg1K zUR6@|JhJ$BnHlr6P4LK$Er@sA zk#UbVdbP8o+0_V?Bi8`=vr)L}h7Ni-xFO>$U38C|2amD|@N%C54!Jcbn132RsQ-pT z_UvdXCXDstI@opb8~l3o5e~2v(s-A2G1<%lpIy~KUwHt=}F~9=pSxWL`FwRPv;SNc2G}@to3c4ZqKH?N~ zS1;1;TRY)oCKK3SwuDQGbdV;w9X@Ir(+cPkxwi9I!8eoXIJbu|b272q1amsKM)YfK#?9v{5cg_%EnJr+GNEBqM zt-`nDeAf2bgqz-9fM*l3U?vp@Uw3Z7Z$GwR!9_9#^#`!PDT+FGC(C%o5q1Y!7 zsg_1b@`+&5BPH7N-P_RQiU`UFZp6(ypTox|6|j-38lEb~P_q7Swczyz5S#PC8u}27 zt-B5fG?*!BuPt1NAAtFmdf2#H0UzI%!BRU_xcM^;M%_bkUz!-UKQzR!ToycH!hr&e z4)}{=!i^^)(R`8>*avpb<#!(Zy#2`i_0ql;Jz`<@~&`8S%A)LoFCh!5sfeSlg3G?c~kzM{i;ZfShMkW6hE zLFk&|K@~?j+*{-cEMbSiHP{eugFM`rRS*jiFm5w{Ui(=^2M&)5qMVrhthaIO51Zo8?y|h@s71CSosIz z{Elo|%LP$%Y0*aq`8+sqdnJB!$%b>}d{P$WM>XwuaP<{LVfPHk6wpSS7;PA4ItD-J z5-BDzf9yE+gt}(?1dVg~kUYY?rttS-QKolP2b4Bgf(_nHr93S3T zk8fS>!zQf^sz1vQHJqDZ%?D-}v13A432khA6c0-q_M?5r5jT!Id=?J8PUfLZvoP7kpbpo(9 zwt{pqg-Q(uc&Ew}*J@3J=#wT$IUR&L4?HRQ!a)2o?g3Z(TVVhFWw@g*j(1NBVCIH6 zDrwsgj2{og9ckYnLM0L0_mQ=OzO~@IygZpgzAr>AnhK6kLVHC zP3w{QWDn-<3V_Ms2rL}FM@3&1MX|D#IQLKq^Y7}RxfvtYCG+A+sk$=!`mFKl~&>IKqEXn?}6iMoc; zscr#=_Bv3US&f!sfSr?}6l=c&4vgM~(`OZN>y`;Hib6DTazWO#NYJ_b3GU|Kg|Xcm zu(*{4ulCxZDZeO{bAbz^cC|8H4(v2xM$rLwWtY3~TQF0p@$|7@aZ&z_|kl8g`>m@>(1<5TyF< zjYGI<9cUb03syn*XeO2`Fr|wJ_r)Z_%tj%+-M1ID%eO^gmBZLB1+nh4!sMaNmv|O7@mg_h&e9(as;l_(maR_9pdhWdY6fj|UDabEmYJ|-ZVh$tfIbSc zXHvU3Hsjlh5@oVu6pUF5n*Y z2WA}qg4)A)a*bV&(pT@m3YA&P^kEpjE0V_3a`fOT!j5~cO2V@$AF84y2*p3~P!V+D zFrBp*%lL0lVdg=|ble+qs=S`9?(E~bLW96D+`L^*txz^mmVR4j)Bx^sKOsgN89 zS`0wlQ{}YJ=TAUXI1>(E_s666rr@9p1Gv{k0(S26uqK@rt zf1vMkI6T}bfI<=NkeMPun{Zl<)BCva;8_Py6Sl>dVjd`7O@{&>!eAg}6Ks1Yj(nV3 zv8U}YL|6-;ed${Kn{SFEsu!unD^=j8vC!W#@Rs^kd=7?EB2b= zra%ddpYej@ibfdslo9{>b-~oVr__9{KV>UK&Rg?5N`YpJY43 zntT7k<{lfc_i;uVS(`BN+mDT+W3YFc6ItZ_sIB+5;plq-{Hah5wMDa(L#Zqtim8IQ z7prhnr61;){2>>H7?|=Y0u9z$Sf@*2`d>+u%}SxJwH(A!?`C+ltq)cV|DsK-yit+I(uDhPm7mkRvgGUh7qlEL{a%kCk^VG+^9iVh-4d8xryxegQ z=zMfgYqbx~d}Bo4ympvF1LP^S#l&Abknzw#tOyPUL-Jn3>=}^ce;Ra+RFJ3gB*GPb(RINgZ3A$osu@f5UGGKRMG*%Bg;S!rPhD994yp{^8!D1~HW6}V*o<+d!_Y_$6 zO+fpfceL=T?P$40AC2A$VsTzMaKZx4dr)Vq;_nVhua08V4nPs&i~$y4}QJ^ z?$Vp^;Jgz4y0Hs_qYGeY6NTH-OyI$n3J~6H01hISz;G@Y-?Z@{cZV;Qy6?kIyEeG{ zb_q_Hcwpd?fmZznO)Qn$hc_B+@vlV>gjEVcWuP%$`%ntI(?_Tsr;V}RS_pMDxbPD* zxwgzLQ8F5O=(Ifq6`x1LuI6xD)cOoFpH?(U;U?VL~4?R7(-W@v_SWFGqAVv9>2)lf3G0%OViweisjuzjVD%RI5r zO74ZeRv4lV%NghmT8YcUDU{oeXxPLfiJNNwK(w0-UgWHYNU^8Xan}y$IuMF^L7Z^c z?IXl){RQIFrPPDNyP%WCh-%w}Fjr(2HmJx$&Y@r|`XP@r1;E|M_%PzgX}HO=1D`!q z#g4=lY8}r#P;Jn~>+xE^(6Rx?j<3bHYOEm1XNNXt$@Olh80B=@2(z{dLP!}GijQc6 zJI6J+blnZ3L^tC-z6P+o6NpXuitsYY1GiSM#doRmaKv98h8C|IDl?& z5I3^Wu;@tv+}p&DZU^YmICdi@io}4N>U&!F-)QPQBl!%m0+>FO2Puc8ky(KSFT9C? zcy&1}v^|Q0TO-l9t{RxC4`Jz-2AI1-k6wq(fx3}Mc{FA~-Y0pAvDplF_4A{jC@+>2 zj)VF+R%&Tj6;+3HP@K;LbNg8E#db|>=-Q0Q4G*EUgatJuHv{`8Ie7EkjS5WjKzm77 zTqkCP)U5;>*VBDik~R+yV@E)d%@tSwalrPt+n~Lz7c_r;fb2a@zYDu~@*YQSCdBZOU*0wpg_bin=CJ@3ctV99+KCF;Za6%Y1z`z?!TS`c5vtj6-aQO!8H<6n+ZTNk#6VEr3ZFk) zf_vo!kf2X$OjSMz1QSFpb5ex2mwHLN^I>9A#3Np@Lg$WlEH1hVv)0}oV>UJ93 zBi~RQoX=rcYzZ26B3_tq1eNNOAfI^TF=tr`n9wmt{X%ont_RlliM`q$vi_8Q=1 zoPt1yy;S)m1Nx7tgY!KyOY#VgwzMuRBuC;=rFZ&BQwzN3dIo)cg)USf_V8+%0YvHviazOA4}Qs zk<i>8Bs@o0j*yu;Y)1^+@rM{oP75qPu*%v8?r{ukM^j)aRO9s1>vCa zO-O%#k({GAM?Wo18t`G;I^IY0>T5(nHkB8!QEUdXcVAbz%UrVK^D zLEL&7xDwV0w?f#F^~7qFWju)X`y{pW)YBmpxz(5AqTr@|S{I&45wh4T>GNDyqJ>-3hfFWgF9DP0xGk+BEw%SqjzWNTn zZy$m~V#c({U*agmC66aph=IgKe;kriz>q{0O!BOPtE0cE!SR<6JEnjFne%Yv??pH- z<^mDNulB71lR!eeNPyA7aFp_v!KO91BjF=;K-TCaUWCJ>WmS0wsEuKc#@{`MFfT-U4Oe?F~jjcPTrDK%8p6 z1&j2u(E4y5uD|o6-IX&yhf^WQHK|LD&$HK%LNH##pI#nBTY^6J>3o?(<<(%(ulEt}u9gv>v|hs(?Lzs-RVe4G*sS z01Tyyv=^V0kR~OJYC8u?_&X?DQBw3Bde+b%G**+$9z{-)LAcn-|{F2ZWlOH}-p z0@!3(t^rDKbsW6_Rb5T2rAGmc>pY>i=2pQ7hYj$xA4j^%Ki zVi4|)VNq+1_G{vlx9LwdUeN1$0ibcIQmM6IN z*qWs&R&BO8-bC$qN#Pf4#>R0g^IPMk)nBk%CCBksYA47$zTj}|jT3BaO;8Md!Rf|! zOt@P+QMup+myhu=(MPR`YE3V=1Co!4&uS-Wj=kUs?LD?;u{8;nUaW{=iJtU)|O%!I=!llEkV{$C)K84n!nmOLEgSC z)xK$3pe{KTV- zl}eoqU&oiCW4(!LjcpkRLSKqau_dW@>tr4(cqu+-oTT}vEi<_3+RR z$PArXFTtK-Y^a-^T`(ghYm#DW-=3Y{G_ye|CB@uJH>YT9Mq0fu#WJ`(r*vsXMw>m= zDp5DLT=2P(W3LoL`_fz&+fUwJdbK%*J>88(ub^FcRw>ye-CeMw zpj&-*OIAv{hqPW{zvHZOL0`I;T1VmC(AlkJ>={0WdPNTkW>u<9GJNejiXJu1s@A1s z_YZL5dksgI`i@60 zcAWmR^cv|nvSL{D%NRuFwAoCv;siU(STyEzcvG|DrS;EnIL+w__h%)lb)Mk~o70ov zIFW3qU%s+%PG8paM5=vfx!{dC1Etgx>0bI3qT_Rh>is7&gF7qM{FpP+=E%-U)ITdJ z^2XTMG&?)L^Q^SS8xyP4?A%KIbMj7aOdb2P^BX(QDTci`PE_`F|W13U+ zsIyY-#v6-()SOeZ`c<0aZ!AOmb4nLGtKi3*?J*p=Wh@5Ox*~6_l1+2V1-q&ZHQrih zrRJWMHaKtU^wy@JKetk?>%3*yTiY^@ylO*(8k@qmcGaeNHTGRK_BY<{s7uYO^)k5N zJpR_csXy;haMuObA8#F6Ir8fg4Qky*<{f)Y^Xv1wYP~e(orY8M8!8Ph`Z~=!kM-v_ zHg;V+5H`PaisNKcx51@Dh4U_RrYD;pbzKU+F~4ge^<>Me!R7GrdDo@>leZVUE=T{E z-%ZC^(8^+17bo)0jm@l}U9h_@QRCen-n4>FX~Qe2PVd}>2MW5?y02u0z1u6nS=ei6 zSf5?^&O_F$u;0GBKL5r$Po=cNK`+CrMdRXs9@mKtXj{FyI~&=mvNTN85%V|DEtsqZC3KuzNh)ojSola(n{WW8Qq*1 z{}9|XQ1T(T=jO8?A3|C=OBWK2TBb!lhW473e$MY{nbr6hHk?-awbJO;Tc?lVV*{m& zjXk$Mgnf*d;yk_7ZFKu{;m621v(w9udTuY?_!xC`A?@_9S))74;~%4!22TH3?78#j z$H${|TxE2u#;pvZ3o&fwWeh^Stt^@gvApSJOftr89L@`I!h>Zj+j`r0!WWK7aGhZ@ zGHzd4v=A?Aeul%Lw_UJ#Awenq441cYhv>vYqWa(&o{-*-HOmW0+Fa$lNyeR$qMwqD z&C6Gw?Cq4+{FGvqUd~@-+$Hb)Db;bXT=05tmty#*G&ima;U42|<)TmNKIRpok9)h- znm=U(q*sW)Htx}!_>>tsSh41NZx1YgIv&GyR)W=}S6B3NRpN#Fx^g!7BLPcQ1PR%V|39 zYHe22;W*K+Wo#DJxp7*2^XA^BBSjNmE7gb2TZZ(HlrDd*(&nzQN-`ZS7yVXkY*Ax# zvVXKv^V@l=j2gQt(}y+A-)bC(YV5D~KfDzF?SdQk1;-xKNA*SDYJDs&I6v-x)Y$y( zVnD_Pm)E9`n{aoi`QGZF9jr-O-_q`Z|b$Ubm-*3Z$PH|rj?J;}yx#)X~*__4Y@W%ts7Ms7{TFAH@`P%IH^2GPsOGB5VzYjeBv;6%I z9Zy{htN9dz*itK-WnG-m;1rA2QX6k(UA&C>3yz&j?ZS8K61NS$;E7o3kl?wJY-B#Y zvUsUe*78cK!{D^w&804-%q!{M<}XE`EOo2jy^#R^u@A*8b-5!@A6CwcZx<&QF#f zHQl{-DP(Bg_2=^AR-VSXB#U?MVn4@vEgS1k4!!fz`Z+$F+1OBJ@!ogm&xx_Sjg8lb z-XDng`DBXcdQ*?ZheO3bC+94$H$NWw5Pb9J(}m3IEw3#;hCliFZ0YXx+uw&iM*sZz zoNh%^E34%~oY=1^w(U*rLU#i?M0n_=n`i081=#+xGmm}m!2`j&5A6x^{2%+&J`~Uk z#ud^_58kGiZn6*;{x2oq|56bCf8Cmmyk0a{aXSZ32j$-P*EKR6jdlEoW#T<@SN z2X*m*`%WDC`W_OGYvAnZ^~mvsY=cq)WZm3D%jmJeTIu~L$vOmLTItZRS&xdolL+-@ zS@7dp9UMMl4?G=xVCQlO>d!dBNbnGta%m#Jq6zFe_7^ICFyhywB+7H(2t1my!SaUH zD0YyQY=wJD-7WORR_M7^1v+rW0f%BZs9yaF z2YL!1e1*{xCv*UYU1k^m9(|rnyBM)Mwoxg7P-UH;N0pksBo1)zm^%= zXlgNZ)SLm!3{w<&8Ud3%hfs3^FBaTAfO&=CQ1VU_XCIuQO4~V6`KlETc^7LLznOyx zHU?^7F&c5Ul6vqk5ssB@AZ;B9oV((U2Apeg)npfC|7;507?@xRStg(74Z}y4AK*(v zB8>mK4DWs%*3<|R!1F>Z$SB1M0_yA(!s1`@l%J63x%Dg3j)GV7_< z_DjWPkS{DURvMJNw`*H7G8Ax(-zzy$`VcBa3w$=OLhQ<{TYhMd5550nK zhxfuXY4I7gm4R63R`~pr8Go+!!=LYe!k*j^JiKp_x-cS&rv)mhH;!!B!(~ObVQoPE zRnoZWxH^7|FQGK;`M`N}4UVS@Bd^#?+EiOC<<}SmGXqk{(7B2d?2ZFVM;ka6dkNI@ zoH1f66LO~+VfMzqT2o}3kZF)L?tbt98ckc_;A0+O|FZ|4>P~3wU3Zd-oIC<5hm7z? zMlimWy94?B7omA_7ba-dQV)$zK<_R_jDGM5*mv?^XM;9&M`yvsl?Q1ffpPHX-a+ht zNQYThB+y|+2fSJu1h$(l`0B6>CcK>mvHdpK%^l5^&ez=C$1y>2O;<}XG zNZGg{f72vf&*TEtoh6VP0pRs`6V_Bdf&7zE$l<1r#t$s1#7|tv^GtwtsDBP58O2fe z%{7?uDxv1I#GvVH2K9bi5m$s+*D9Z@d#!uk`_H_)hqPs({82R2k`}N6#l7-{!4Apd4drG z#j|1KBR{Z?jK+_QxlnehiegMM!>8dEXqx*48YTB5crk$>*)liobW8K7<+Rt?fXUX9@=o*g@#NP&}Zq4u#a4Kq)C7+D2?KiH92nZd!m; z@L{Z&8ltLq%Hg>D5qur3h7Rwyquh~IQ2fmvd>9|XGT37Q=TnM(w2I39Lx*0e0qB>f zi#z4HG3MGWXzyml*JSQf-p~g{**i%~g+g}VMA13)T@Z^QeQsl1wOK;zhf8 zOKR7fL+HFu9&a9Jhw=F_F#ehcC!|{7a<3fhfUw6)>qD0=BJHE zOLTBy(@u;N{SF`Niox8Mv=xLJq1z`AU-18ih^P-BaLgJ7_PXLy+&(n6md6xnMwC9Z z7FUpMa95J|pt#8_YDU?>T(41*XkhZF`&SAefXNMir!q;z?fbdJNWr< z|Le8LJDN;=KJyCJ8Er#dM;@r(&4q0MxvVTFhaO5IXL_x?w4{g^+- zF(=aa>GxrMN(ctsDTVtvDmeX$1J1APfrfpSIG$yTyKihkwHMYnXUziEyPg48sUv=h zUJHro;<#Cz9c9lbpkMk+D9z#D$fv7pOB)N5DZ{8#i5W zL{6_5=q4@FO@bkCSZ6n$CtC@{8-ZS-R;1qq%Nc)VsEZegE<&}#-*;UR>N8a1JRMI$&D zT&8}K?Vv49;iyDsg^Qkz;5H@%v@c?KqF06D&h!JR#dTO#xDAr`PC!@UFt8ow#O=r0 zVH1NL{*l*1y)+$g%QM6PS$?$Bj-g6lvtg-82(rvG;f&)C5Hyp+HLuv<3%dk-3T*=^ z`#W%8l7=>(gD{mQfbZXjp)Fae{h?Du6A|pCyk|$KRUf5b;9($cJ-DANpLW2X?_t=# z1F#@m2EW8-!-G@w$dynF=c_ZJK~NYAS$5{ZiYB~Uv~1tqx1R`H7^ z6z?q>>TTr4!0VsEPXcj|k`30+T!Nk6d|;QD2l~cufTt%LN@YaA_D}|Fqx6vTU?i&7 zKZU!~&)}S=G;Y4I1rMrFIACLqvq#2ZgkBOIm9Nu8@AUw?xdyJ%Dbt!%md7nW)zEJu z0Jg^I;`1>-T-r_MhSx_?G3y`>l9)wg zm_R)m{S6O|mGPlI4K;K&QUzLxluLjj-ZkHVC$wI}+Mjz-C1V3-y=Q|fKC&pD7J}Qg z1gODkCe&Xzg6DSyQs3NcQG;iI`fu%hXH*qQ7cEJ$1VID^1q4)rqzektb-d&(AUQ}* zatV^bgrK6BF@mUwm?P#aig^rRzfaeR5hnipCeK(-F@JT-XbB8l8WNg;>XNdDc97 zFiw%Bt`8@#s%AC}(_0=U*J}-%t)f9Kv6-A_C8m~sZeT74y3i_(BdjS`mP+JGcD~|%JB?)s*EotYVRT4bmVH@wl5OawO&`Cw zaGw`XVgbv?vc%W!v}~ahpgH9))!@o<0?6vZ zXSPiewk#i~>p;5yURz^}>6!o%0)Xn}t5rqLDKP(t}Am z*u^_1S%_Y_U0mPy?8`?HRvFBrvO_5>Y{V5NJB?3b^Hng-Qi-PT97zprfpk71jpi?X z#EvHSVv3m7z5eoRwlPhNj6?QuHRp}lvc%prze$H4CrZ=9%0x0%abVVa-RRhE6LQU6 z&z$5Jas4dK@s+O|JrWTi^A}wxLa`H9=4?R|wam#yH;KmC+0w{s@^q}ply2(mVM*)5 z>{_<_kRY@x1@C#ow94hE6W%wt>bztdmQLp)wwiDkmOWx3BO^&VyNReUiCf%1fMZV5+|IuHS%AzZu1i{Pl9+#$ z>D-Gat>e+`PQEz1X>Lgqo}6Mq!3m_CJ&DQBddN1^$kE_1Ia(d~np2$9krGasP^F0} zi*J%8g;Uy8G4M7Mi_N5?zlX4L6)iG+s6*v9)tQluIF*W?XKv}i)KqIkt)*FXSM!?P z5V0a=(_5LvaN&#GhuzOq}WH>Zk0MJLCF2Nn^Vj=CdY^fzL%<{z!HcwB2K5yzwaicBiM1Tf6vooCLU3fUa&*~o2 z+xM6mNe`x^mi}Z_tI4`9?nAaK<0&gif)pk`V;Vzk$bQc#%Fw>f&232~kGZ><^4%pY zQ?(lR>Tqf}`-M$zd&!c=M3AeB3u){m7GCs$t&i@<=ER7T%$_eS^GP0!z%;2r3p%o_ zT{2YXYeac(?sB{#uNd+5X!IEu+V^S-Gb1;e8e7I*gi5eY({$Jb<+W^zycTnu`G&a& zc5o}AM5)IM4O&0iogI-#B#Ck9RDZe)W#_%aRN(%UXXeG;NoLWAjTvzN@0iZF~v^gd1`; zuM)^_y$7AK5F-=s5~e)yDtpQtDLh4qw%Sc&1!W7_hZ$qo{YauMvxrT+bcv0BdWFpz zIGm2n%cZu7?khj#wshB_XXy`ZgHAnR zm)AdJqo!+MIg|nI_^d@6ch;~2N2O@dJUcFDm=rBuh-DEMZg9iBj&M7tDbe<|!ZkxzfUreNO(e=#njy{Fj zJYjAn`P3cnE0?_(K^vx2vAjt~S?{(?n%F~;*0-5Zhh0Ww>@T2{eF>Day(3-B>r3kf zA7(>uN3pg>Uv6oi6nfWp5!2h6$NENR+Rc}}!Db}~(#OjIY+83CI?syihQG2RNw+4Z zF=#I9wIhuw$lPHf2gFzpDM`AdW6MTOdd&o*bIAB~1sjxK!j0M zJtG#egije%C@M)`_S?{n!e{LB!>cS@CX62ZUe8_gox$vSN3s>K53|Kl{-mjI!nAoA zG;KjKim@5kv}#BUwhZy2Aietn(!{@%pe3{>dJ?E&=O{tKu1{s8MX z!;mw(eUWLD^d##kJDHNJ18t}Yq$gO)$?m969LQA@{KHm7e>^euOV{UL; z2J|QH?RnO`9G~Cp?nfJ)f~h32iFKm|+@U;&)Bs>fLzE z7|Tj|SQ?fU!{=Hm7P6r_hNO8tjt1!u;l|DzN!jyPv9k^l6mae#mKC}X3*BmWBiw@? zn8?zp>8~)gE0E59Nu*CY1yp$YF>}&&qYJmR$#BOqcBWjBo}HDUbp5k-zP=^Q;?gFj zhG`5X+wJJ4f+5{dSHtuXMUvG1#6rvuvyRUy*oeNe6oDnFuM`Z}l>xJvbJc8S@eX6C zE63O}dvj8pqeCVx6WJEe=Ulkser6GyO{wumSccyX_NJ2q!rcWY&v^MWX@Erl}X z&Edw}=2Oa=V9Gf%jd}JEp`{x+c4A;POE15S<-Rd=z9o$;yUEeLCBxXokDa(T?gnht z2}L$xwJA$)97^Y+`;(zyB*hjPkYU*n3b}NYGo2bk-EYRyn8BX35bxQhUDcvPwYZ3Ee4 zrH*zYFOxDmpDnqpbvEAnWEL@xxoYH=CQp;*45T~Oa`b7mBRh@d1yyN`;@4vNr2Bie zS2l|qqThwWUbS&z`TN+#W6qd5E=r|mR@=!=uVY6N$Ft(f2qyRPGE2`INnZ7L8SiKk zZSUesZ)2Y^|Ib^P#r!ZzaEPQgcKxuF(18|Cx1psQrZfMULuun|0qLE5&&H)}j7cyDzgFzhWiVQQ;M9n6jDM zC+I*^x8An1eym2G-S;t3&lWb&0n2>ShETVt912y?CXZYN60_AM{X@8REY9S5-AtkN z8orcmyMjsH89|*d`_cBLrEIn6E9T~rKuH>-X%oiF!{7hT{8V}5x>*ZT?s73!F{f^Z zgQ;-dZdN2~K zPae&De2^{e{EEE}97cDVcq~SYUl1u z220wbjwVyr6}9YC*GRUi_8r?9f0(sAXkaUX1SGGUNQQV%c9W+`JFyHVa8VGgoNP^2 z?yk(Uv3@|8vqPAqycQ?^DuYHCccOh0`Z90D-fV3ZF>d@e#%o-`##W25m#zbti^ob9 z#T!X&3G3K(dwDi^cnceqU`fx*RH?@LG?Q@c%z`|IuzTCIn0EP7wr=kLnx+^+)3gd{ zXIU8OV5z{H$~?AYJ(e3E(4!nl4X%@JG&!Bi#AloxXw!$=%(XI*mF`cX&`Luxcwxvy zI<>M*QLfYl(-JEcqG&SXN-p6J~%x6QIcW|CErc@pDh}C0ChB5=#h@4~0@=P&losGo?^_Yg*wD!Y+&qrmcR(@$MOLLJOFgrql&F#_1 z=9UFe;It45uei?m&8x$Q@F!!Z^ym!W_!QudTBcA7_CK(kNsG|hyp4Ye3{+5YDpUSPU!O0iQFd332b>@30w5x5O*fq zlb6`=hQUm=-Aj_%uFl3BxdmRo+zfK{W%2uX+ZePscCJNo!?q|cK#^e{$JDH zYBbIf6%nxpF5!R34Z$FCBcKQbLmLZ(Vy(kmpcwIT_!Z#MqVg~I%0t`%VIPF)u;0Tz zjDo#jUjW1aTf`PX6Gbc%)M>A(W`eUP^bb`9(qU}N~ous0(w9=-vv4*nMSClDJ0lmaQxC7?G0 zH{t&VECq^z@rWG*8d0VKJR6z?Y+dj>AOd&}e-&5{=zuzske3eb5B~&W8-YQ93jDLs zh5rgL9sU*gaeylFFM;KO9H2X(3#|(9guM{h4D1Iip>0L^a{vK+XwKl(U_R>X2TX-u z1hxf-K#u_5fqxUM0-i67p`Kod1w$)^FFZF*z)FAv&=v6m&s6kt%W@Yyc-w+90jzI-xWwhP6pT;+AQEca&G}i&}#rcXl`KqFKLlT*qy;; z;Pb#PNIj~Q{4n)o{ z_#I)Jfk%M1fL(zQU>E#D;4?rC&;~T4>=5Xa!2Q4vz*@j**qZ=1paprZs5={627fK= zonTkwX@Zvnd!P-0{}NaZTmUSg^MO7#aRe;@o;;K3-@0(%$qVX)r-(_r5~tPju~_y{CJ zj|9g7zQ`X0UJjfDqJ(9EFz7dscN*Fhuqq&coeg~!;?l6!1H<5d0y{z*fSi>;Ewtm% z4#DmMn-5zHTmh6I?+Ek;a18v9u=fgcVP6&I!|sR}4}K7AAK1&FcLkpW-T+sj&jL0g zCmsGiVID9MSOA=XCI_xVIeplXfD8P1FoSjqBSUF$-G{bcPS^#(WHpufwjuxN@m_mO5=tB<& zp8#h16=fgJ{jBBvX)F3^U?*Jg@*hL1g+C8)h4wqN0pLAA9B>V~BQOK711=)I24&vDo`d*# zAP;s&Xt9V*05pL{U>dYs#P$fc0pA{Mg#2dkD(IomZ2*u6 zZ3$2W-3$6~a3-LMm>l#?utmTf5c7q$4Lkwd4Vne88Aya*1N|Ow8MYp@Fz^bX6N=x3 zzY+E=*sEaoKwb`fCHUiD?}nWKdnvdH*b2=CNCZB>_e1PD$^`I2?#-K0JI#~<=}cC1@?UK zUF1Cit|Mn2w1Kdzp)ZE522B$7DL@Ur06u<24kJe!tOV8r#zI^Jv;jyH*#<}ea=;Q` zIC4HC)(c31-vB=eiR!Rt0M7tf;2iW1$hiyq4EQ2og**dbGVD*l8raT=7lBWMO;9El z_8Fij;_m=)*n7Z+h+hN7z~2kr08~TY1>n>5`ktMS_x#JkZ{ff~U3i7UFsQf0pB{u8 z$}g`=BP{&4*RrEUB(&>ABvhm7MWyws>&4`GH|oU|2P%z`Q1S5@)1gb$^f8j%G8)H7 z^(ay*mhL&-r&y*}_4Hy{qYaJ4a;Ei4CGwVyJ|!Ld-I!jYFrclmM3F?4$0|7}`Hod~ z@h@`h^Y@!_t5 zv5k^l(sqkVbscupTuLkF;;>TX{M(C6LrhyXnsfFLpDi6dZ8twQz)(d+(ab_OcosX6cUWv%I>z%y6;K23e!E+n>uC z*B!NxGud)6Q?Boh`}X_$#{njY)3tx=WCw4XyH&T`4e!@3@s@bBAR%l#qHv}>`v}|xW#AO zjkl=;i)d2wT{r`MZ{s*PS^H)hTB9`Nkk%t73zS_vP%jFqad zlT(x`E0Q1PFMhT6K~WV~{X;YSJj~Cl%6M}f0{s2FoB{#@BgO>;hNj->6!bj%&iSCI zar(`{akJyqtP&T`o)emKX_Hi#!j>&l!&GkHP|;7(e>KNBgHxFsp4AjEHzMz5{J{3ZhmxR}GLyr>v7^<`-@F(*-N~RL zV5VuqyrY6^V-$Yz&aI8>3d4G<)*0{ruSJftAUHO?qdN zR}7m`IdsHYNtu+jJ6>Pzx;sfMOt9=&!Ud(^t9mVhxC-$usWs*S^HZ;0aaK;#zqw?h z+KvZXyQS|*Z4HYIuYTDrW51WSrtfthp60MaE6oG=heNV8GmrJzR+U<7exWMsw3Bm( zevNg?my(+zw6A7MU2=}dX)egt%Kb2F*46%(Q;$UCb;~cU&s)C5S}FhL#D3iiN@lyv z4-}i1w@{%^^|lEy4?b1P#@ViTv%m21mInTC;~huN4{saO`MTcolP=eXd7MkUK5~t8 zY2>K)Ph4b2Kdh+1Qz-)cFBJBDb#GDDIk)FMit5{xs*A*r4~!baTN$ezC#5@2TVA&B z#iJtmW*4jb7>#VbQKFQlvRLzmn|kwD)rVUTcW_!6GWS$;67YO>|wL&pd3nlDbD((kZ}yj=gv@__i;2WR#ywUwM8S2*SI(ObG5UvH1< z%06$ukUdaRX36vr&$x!^sY)w5Yxn5vcDRePZkAqmSA#x9*>!zZN>28$D453+Uv<8A zrpeS^Ei-NH3od8)y4x%*@vre&I{RwGJB3-sF%|L3GD#ZO=Y+3nj`tdp(AsI%%?VDy z6I1%M-I2Ldpww|*bftlTzx|YBIu*vp{GBT9Rb0AVp?6}FY(~ z9!Xpjabn}V5uTA+K<+lopY30 zTX=4$QQWrHox#pK{A?DjbDy={Xnj|KYTSB*GKVSa=S94DG^zf8a{PuT@jSC-d3^WBi3!`&yj^_u;lR4}6|K&d*bL_aL9?s#7#8G{preR>bc=gCz^2akI_ z)g;jNgYo7jZ|$|?vNPS>C2gw9j%+>`xH?IySvM+4_fYsGhvCVt&bQnoh35}S51vvcZcrCkue(8nH+Ae&O^KBv7wRYY7_BPqb$j=-wH?pJL&V>zX z*V@`*_t>vVJJ>1Fer$ruqy3M34SfY?s3@R(vfqkizT4E!a^ud)nY144pPO~3%jIdj*l@Vp(U!?`yJd|~Gd!1F)upS3 z=;6{MSsd?FSr4OE`Z=G=#WS;Q#)se96u9pE(=ECxYi8~$o{`;jRJ^C#MtgnAn0l<& zqUvl2Zc*xUB=*k!;}^$?rAW#Z$rP#INGUr$FGBvz5~JlqZ!5pbwbYXu zG}-9EvVFy4w5*KZ)O(~!ovU{I?e6=|elZiC+{;xO z2bM(VCPv0ZCkqN9von)3vc4{k-WaTQFxX)4R&j&cVsU%`{pTp@r!mtnKY&*1tm@fW zto_qxY0ds23DzNa;u^+cYMs0OI1-coBc%S_Lhaw}KUECKc1TicN}7jqWUu z$FJy%{p|@a`l_Cjl{JmgI*d1k34cBiMWnN|(Eq;h{Hy={(|)`y6V~_B0r*+$IID0*sZ}+LtKto* zvASeNbXr_=mikWV;kwCbaq;<)vFU=G)U+&JTkWvnx+&0L>1IZ!CC2L-YwM;(r^Z7_ z%Ml3R;h{&F*>KI;?eq+o=Kan3wR`dH-pB|zU#pC4uRs1<&3~;ZAvsZw08a?$DlAr* zp4Psb0sanXs4;fnTW{!of8x^LTlJUFPwh!g%eLnIy(!=2A395TH2<>6@7J)uZ}Kmp zpEj8wNRQ68wD|i*e+c}i&HnMl=-;>cW90w!M*n(8{KdBap72u>(5JuJ>+kY^+UIY# zv%hclhrmyp`{Qi-dq4jX`Q>@mew5o^j>0==dS?5pM35Yl8J$@me34`nL}zBE=Lu`Y z6TY&N)6;}1udi8Ob6uPry5CyyL(IIdWnVMhks|8yd_TWILAGCSo$70pOoc!Bi7tFT z-xm=6c<}i=zI`I!na@w;J0!Mi_V#~?=Ltop?{Qb5-HvZr?Qu6gKaY=cU|}WpZm@+u zpYNECeEWWGse+tPmq1>yg?Da@YhZRvnpg6mWc!%-0{6`5AgdI&q3MZkY3@lesc{0g zaFN|{u z%oEn-6>J_S%n7lu$%+dy&k45py8Y1L@7v7@2{IQ5b7Rs1vtk^C+YHSJ4a!dmw)n-i zU7XQ==fYr1KdV?5VY~RraY0rYs6Qoiu$yq(Xs3Bze4ve4e6W3cy>6*C$r%*V_#HlJ~XWe%^=wkeg z=fn^D1-2fgr}k zFCiw?Is4nON(!;?{;~c4yS4>~rUhI0B}E723y(oTbg+HlKONhUf6D#GWBkwkC_Khq z*i!qs`+s+g3qpf%e1pE87r*Gkp|Pof!q>&ubMu#N`tx=3kH;6UeOv=Fa9y+TcFxX; zH8=a`HU^pruPNWQYf~V+PT{=FiVkuw2nlw#`g*;zw}M>|V;S3CkMQ*qYaz&uNfy3F z-ON0P@^icfyW^a}we$P+D^w7h=A98^!E3+fcn#(iA}=`scLdxA_J5@ZRlt+AW-{!jbUUg(GYaS-mBi|}=TV<51}2yHKz$aieNZ^jDi`}Tv|F@DgW z0mAz!jstpK*icmVy;0x#`|C}+z3YVue7>-6?FBd&a^jqA+OOln`({iqUs%7_w|0N+ zYIO0p{s!~;!fUgT@xzIz_sEFO9F`NGZ4#T2VbVSZ(f&5b&~6`nXbA9a;!nRK>lN{Z JJm1g!{{WF|k&FNU diff --git a/serving/src/test/resources/feast_project/example.py b/serving/src/test/resources/feast_project/example.py deleted file mode 100644 index 4d8032f..0000000 --- a/serving/src/test/resources/feast_project/example.py +++ /dev/null @@ -1,49 +0,0 @@ -# This is an example feature definition file - -import pandas as pd -from google.protobuf.duration_pb2 import Duration - -from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService - -# Read data from parquet files. Parquet is convenient for local development mode. For -# production, you can use your favorite DWH, such as BigQuery. See Feast documentation -# for more info. -file_path = "./serving/src/test/resources/feast_project/data/driver_stats.parquet" -driver_hourly_stats = FileSource( - path=file_path, - event_timestamp_column="event_timestamp", - created_timestamp_column="created", -) - -# Define an entity for the driver. You can think of entity as a primary key used to -# fetch features. -driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) - -# Our parquet files contain sample data that includes a driver_id column, timestamps and -# three feature column. Here we define a Feature View that will allow us to serve this -# data to our model online. -driver_hourly_stats_view = FeatureView( - name="driver_hourly_stats", - entities=["driver_id"], - ttl=Duration(seconds=86400 * 365), - features=[ - Feature(name="conv_rate", dtype=ValueType.DOUBLE), - Feature(name="acc_rate", dtype=ValueType.FLOAT), - Feature(name="avg_daily_trips", dtype=ValueType.INT64), - ], - online=True, - batch_source=driver_hourly_stats, - tags={}, -) - -fs = FeatureService( - name="driver_hourly_stats_feature_service", - features=[driver_hourly_stats_view] -) - - -def conv_rate_plus_100(driver_hourly_stats: pd.DataFrame) -> pd.DataFrame: - df = pd.DataFrame() - df["conv_rate_plus_100"] = driver_hourly_stats["conv_rate"] + 100 - return df - diff --git a/serving/src/test/resources/feast_project/feature_store.yaml b/serving/src/test/resources/feast_project/feature_store.yaml deleted file mode 100644 index e276cfe..0000000 --- a/serving/src/test/resources/feast_project/feature_store.yaml +++ /dev/null @@ -1,8 +0,0 @@ -project: feast_project -provider: local -online_store: - type: redis -offline_store: {} -flags: - alpha_features: true - on_demand_transforms: true From 9de9d019fe13d4457df1db7ac79333ee1090e3ae Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 16:16:15 -0700 Subject: [PATCH 27/39] Rename test Signed-off-by: Achal Shah --- .../src/test/java/feast/serving/it/ServingServiceFeast10IT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index ad25561..c84f368 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -106,7 +106,7 @@ static void tearDown() { } @Test - public void shouldRegisterAndGetOnlineFeatures() { + public void shouldGetOnlineFeatures() { // getOnlineFeatures Information String projectName = "feast_project"; String entityName = "driver_id"; From f3031015731293719f74314ff18615904ef32ec0 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 16:27:42 -0700 Subject: [PATCH 28/39] awaitTermination Signed-off-by: Achal Shah --- .../feast/serving/it/ServingServiceFeast10IT.java | 12 +++--------- .../test/java/feast/serving/it/ServingServiceIT.java | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index c84f368..d0facbb 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -35,6 +35,8 @@ import io.lettuce.core.codec.ByteArrayCodec; import java.io.File; import java.util.List; +import java.util.concurrent.TimeUnit; + import org.junit.ClassRule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -64,7 +66,6 @@ public class ServingServiceFeast10IT extends BaseAuthIT { static final String timestampPrefix = "_ts"; static ServingServiceGrpc.ServingServiceBlockingStub servingStub; - static RedisCommands syncCommands; static final int FEAST_SERVING_PORT = 6568; @LocalServerPort private int metricsPort; @@ -92,17 +93,11 @@ static void globalSetup() { environment.getServiceHost("redis_1", REDIS_PORT), environment.getServicePort("redis_1", REDIS_PORT), java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - syncCommands = connection.sync(); - - List keys = syncCommands.keys("*".getBytes()); - - log.info("Keys: {}", keys); } @AfterAll static void tearDown() { - ((ManagedChannel) servingStub.getChannel()).shutdown(); + ((ManagedChannel) servingStub.getChannel()).shutdown().awaitTermination(10, TimeUnit.SECONDS); } @Test @@ -110,7 +105,6 @@ public void shouldGetOnlineFeatures() { // getOnlineFeatures Information String projectName = "feast_project"; String entityName = "driver_id"; - ValueProto.Value entityValue = ValueProto.Value.newBuilder().setInt64Val(1).build(); // Instantiate EntityRows final Timestamp timestamp = Timestamp.getDefaultInstance(); diff --git a/serving/src/test/java/feast/serving/it/ServingServiceIT.java b/serving/src/test/java/feast/serving/it/ServingServiceIT.java index 8e0a82e..37cbbc3 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceIT.java @@ -212,7 +212,7 @@ public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOExcepti .build(); Response response = new OkHttpClient().newCall(request).execute(); assertTrue(response.isSuccessful()); - assertTrue(!response.body().string().isEmpty()); + assertFalse(response.body().string().isEmpty()); } @Test From 588f14969c026123976ec84fafae8a10fe446d30 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 17:03:39 -0700 Subject: [PATCH 29/39] lint Signed-off-by: Achal Shah --- .../test/java/feast/serving/it/ServingServiceFeast10IT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index d0facbb..da87afa 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -26,17 +26,11 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; import io.grpc.ManagedChannel; import io.lettuce.core.RedisClient; import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; import java.io.File; -import java.util.List; import java.util.concurrent.TimeUnit; - import org.junit.ClassRule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; From 2bbac62d0b478c0cfd06f55a0b12488ccea9464a Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 17:13:49 -0700 Subject: [PATCH 30/39] lint Signed-off-by: Achal Shah --- .../src/test/java/feast/serving/it/ServingServiceFeast10IT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index da87afa..c20caa8 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -90,7 +90,7 @@ static void globalSetup() { } @AfterAll - static void tearDown() { + static void tearDown() throws Exception { ((ManagedChannel) servingStub.getChannel()).shutdown().awaitTermination(10, TimeUnit.SECONDS); } From 79757e4a8618928d3620b36c01c6f41c5ddb2c9c Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 19:29:44 -0700 Subject: [PATCH 31/39] dynamic properties instead Signed-off-by: Achal Shah --- .../serving/it/ServingServiceFeast10IT.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index c20caa8..93a7c88 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -27,8 +27,6 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; import io.grpc.ManagedChannel; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; import java.io.File; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; @@ -43,16 +41,11 @@ import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @ActiveProfiles("it") -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "feast.registry:src/test/resources/docker-compose/feast10/registry.db", - }) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { @@ -73,20 +66,12 @@ public class ServingServiceFeast10IT extends BaseAuthIT { @DynamicPropertySource static void initialize(DynamicPropertyRegistry registry) { registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); + registry.add("feast.registry", () -> "src/test/resources/docker-compose/feast10/registry.db"); } @BeforeAll static void globalSetup() { - - environment.waitingFor("materialize", new WaitAllStrategy()); - servingStub = TestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - RedisClient redisClient = - RedisClient.create( - new RedisURI( - environment.getServiceHost("redis_1", REDIS_PORT), - environment.getServicePort("redis_1", REDIS_PORT), - java.time.Duration.ofMillis(2000))); } @AfterAll From 749c4d005dafdc2db19ca583e42b7c6128881075 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 21:08:23 -0700 Subject: [PATCH 32/39] dirtiescontext Signed-off-by: Achal Shah --- .../src/test/java/feast/serving/it/ServingServiceFeast10IT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index 93a7c88..be9da43 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; @@ -46,6 +47,7 @@ @ActiveProfiles("it") @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { From ebce9aca1fe45d6ac355253a312dcce90ff801de Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 21:52:55 -0700 Subject: [PATCH 33/39] python 3.7 Signed-off-by: Achal Shah --- .../test/java/feast/serving/it/ServingServiceFeast10IT.java | 6 ++++-- .../src/test/resources/docker-compose/feast10/Dockerfile | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index be9da43..45b2d7b 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -46,7 +46,10 @@ import org.testcontainers.junit.jupiter.Testcontainers; @ActiveProfiles("it") -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, +properties = { + "feast.registry:src/test/resources/docker-compose/feast10/registry.db", +}) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { @@ -68,7 +71,6 @@ public class ServingServiceFeast10IT extends BaseAuthIT { @DynamicPropertySource static void initialize(DynamicPropertyRegistry registry) { registry.add("grpc.server.port", () -> FEAST_SERVING_PORT); - registry.add("feast.registry", () -> "src/test/resources/docker-compose/feast10/registry.db"); } @BeforeAll diff --git a/serving/src/test/resources/docker-compose/feast10/Dockerfile b/serving/src/test/resources/docker-compose/feast10/Dockerfile index 6efc3f2..bde9f11 100644 --- a/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.7 WORKDIR /usr/src/ From 174d7572c7ec17210573f7a29f72ae7eb80e8bec Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 21:59:51 -0700 Subject: [PATCH 34/39] spotless Signed-off-by: Achal Shah --- .../java/feast/serving/it/ServingServiceFeast10IT.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index 45b2d7b..2f49176 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -46,10 +46,11 @@ import org.testcontainers.junit.jupiter.Testcontainers; @ActiveProfiles("it") -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, -properties = { - "feast.registry:src/test/resources/docker-compose/feast10/registry.db", -}) +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "feast.registry:src/test/resources/docker-compose/feast10/registry.db", + }) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { From 8e7ed44cb71e161c0bba33a86020eba0378223cc Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 22:19:02 -0700 Subject: [PATCH 35/39] Dirty Context after test method as well Signed-off-by: Achal Shah --- .../test/java/feast/serving/it/ServingServiceFeast10IT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java index 2f49176..c1e7a15 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceFeast10IT.java @@ -51,7 +51,7 @@ properties = { "feast.registry:src/test/resources/docker-compose/feast10/registry.db", }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) @Testcontainers public class ServingServiceFeast10IT extends BaseAuthIT { @@ -85,6 +85,7 @@ static void tearDown() throws Exception { } @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.AFTER_METHOD) public void shouldGetOnlineFeatures() { // getOnlineFeatures Information String projectName = "feast_project"; From cd011c380db17928c80e889c6fce89867ad5c3e1 Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 22:30:56 -0700 Subject: [PATCH 36/39] Cleanup Signed-off-by: Achal Shah --- .../java/feast/serving/registry/LocalRegistryRepo.java | 8 +++----- .../storage/connectors/redis/common/RedisHashDecoder.java | 2 -- .../connectors/redis/retriever/EntityKeySerializerV2.java | 3 +++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java index e2c8acf..91c4211 100644 --- a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -27,15 +27,13 @@ import org.slf4j.LoggerFactory; public class LocalRegistryRepo implements RegistryRepository { - public static final Logger log = LoggerFactory.getLogger(LocalRegistryRepo.class); - private final Path localRegistryPath; public LocalRegistryRepo(Path localRegistryPath) { this.localRegistryPath = localRegistryPath; - log.info("Working Directory =" + System.getProperty("user.dir")); - log.info("Local Registry Path: {}", this.localRegistryPath.toAbsolutePath()); - assert this.localRegistryPath.toFile().exists(); + if(!this.localRegistryPath.toFile().exists()) { + throw new RuntimeException(String.format("Local regstry path %s not found", this.localRegistryPath)); + } } @Override diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index bb09c75..a4e8ca3 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -30,8 +30,6 @@ public class RedisHashDecoder { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisHashDecoder.class); - /** * Converts all retrieved Redis Hash values based on EntityRows into {@link Feature} * diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java index e51efb0..922a09d 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java @@ -28,6 +28,9 @@ import java.util.List; import org.apache.commons.lang3.tuple.Pair; +// This is derived from +// https://github.com/feast-dev/feast/blob/b1ccf8dd1535f721aee8bea937ee38feff80bec5/sdk/python/feast/infra/key_encoding_utils.py#L22 +// and must be kept up to date with any changes in that logic. public class EntityKeySerializerV2 implements EntityKeySerializer { @Override From 28a2b2bbc7d214bfd2e2696258d33a254a280cab Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Tue, 5 Oct 2021 22:35:29 -0700 Subject: [PATCH 37/39] Cleanup Signed-off-by: Achal Shah --- .../java/feast/serving/registry/LocalRegistryRepo.java | 7 +++---- .../storage/connectors/redis/common/RedisHashDecoder.java | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java index 91c4211..e8ac6c3 100644 --- a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -23,16 +23,15 @@ import feast.serving.exception.SpecRetrievalException; import java.nio.file.Files; import java.nio.file.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class LocalRegistryRepo implements RegistryRepository { private final Path localRegistryPath; public LocalRegistryRepo(Path localRegistryPath) { this.localRegistryPath = localRegistryPath; - if(!this.localRegistryPath.toFile().exists()) { - throw new RuntimeException(String.format("Local regstry path %s not found", this.localRegistryPath)); + if (!this.localRegistryPath.toFile().exists()) { + throw new RuntimeException( + String.format("Local regstry path %s not found", this.localRegistryPath)); } } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index a4e8ca3..ce7d200 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -26,7 +26,6 @@ import io.lettuce.core.KeyValue; import java.nio.charset.StandardCharsets; import java.util.*; -import org.slf4j.Logger; public class RedisHashDecoder { From f8f1c0a1d473daa6c6cff5297646257e8a17bfaf Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 6 Oct 2021 08:31:09 -0700 Subject: [PATCH 38/39] cr Signed-off-by: Achal Shah --- .../feast/serving/config/CoreCondition.java | 5 +++- .../serving/config/RegistryCondition.java | 5 ++++ .../config/ServingServiceConfigV2.java | 1 + .../serving/registry/LocalRegistryRepo.java | 23 ++++++++----------- .../serving/registry/RegistryRepository.java | 4 ++++ 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/serving/src/main/java/feast/serving/config/CoreCondition.java b/serving/src/main/java/feast/serving/config/CoreCondition.java index cac6f9a..83fbff5 100644 --- a/serving/src/main/java/feast/serving/config/CoreCondition.java +++ b/serving/src/main/java/feast/serving/config/CoreCondition.java @@ -21,8 +21,11 @@ import org.springframework.core.env.Environment; import org.springframework.core.type.AnnotatedTypeMetadata; +/** + * A {@link Condition} to signal that the ServingService should get + * feature definitions and metadata from Core service. + */ public class CoreCondition implements Condition { - @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { final Environment env = context.getEnvironment(); diff --git a/serving/src/main/java/feast/serving/config/RegistryCondition.java b/serving/src/main/java/feast/serving/config/RegistryCondition.java index 7d73cc8..f2cfb3b 100644 --- a/serving/src/main/java/feast/serving/config/RegistryCondition.java +++ b/serving/src/main/java/feast/serving/config/RegistryCondition.java @@ -21,6 +21,11 @@ import org.springframework.core.env.Environment; import org.springframework.core.type.AnnotatedTypeMetadata; +/** + * A {@link Condition} to signal that the ServingService should get + * feature definitions and metadata from the Registry object. This is needed for versions of the feature store + * written by feast 0.10+. + */ public class RegistryCondition implements Condition { @Override diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 2e5e0c4..02aff4d 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -139,6 +139,7 @@ public ServingServiceV2 registryBasedServingServiceV2( final FeastProperties.Store store = feastProperties.getActiveStore(); OnlineRetrieverV2 retrieverV2; + //TODO: Support more store types, and potentially use a plugin model here. switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = diff --git a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java index e8ac6c3..ff41dd4 100644 --- a/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java +++ b/serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java @@ -25,29 +25,26 @@ import java.nio.file.Path; public class LocalRegistryRepo implements RegistryRepository { - private final Path localRegistryPath; + private final RegistryProto.Registry registry; public LocalRegistryRepo(Path localRegistryPath) { - this.localRegistryPath = localRegistryPath; - if (!this.localRegistryPath.toFile().exists()) { + if (!localRegistryPath.toFile().exists()) { throw new RuntimeException( - String.format("Local regstry path %s not found", this.localRegistryPath)); + String.format("Local registry not found at path %s", localRegistryPath)); } - } - - @Override - public RegistryProto.Registry getRegistry() { try { - - final byte[] registryContents = Files.readAllBytes(this.localRegistryPath); - - return RegistryProto.Registry.parseFrom(registryContents); - + final byte[] registryContents = Files.readAllBytes(localRegistryPath); + this.registry = RegistryProto.Registry.parseFrom(registryContents); } catch (final Exception e) { throw new RuntimeException(e); } } + @Override + public RegistryProto.Registry getRegistry() { + return this.registry; + } + @Override public FeatureViewProto.FeatureViewSpec getFeatureViewSpec( String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java index 776a7bf..9f9ed40 100644 --- a/serving/src/main/java/feast/serving/registry/RegistryRepository.java +++ b/serving/src/main/java/feast/serving/registry/RegistryRepository.java @@ -21,6 +21,10 @@ import feast.proto.core.RegistryProto; import feast.proto.serving.ServingAPIProto; +/** + * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry object. + * This approach is needed for a feature store created using feast 0.10+. + */ public interface RegistryRepository { RegistryProto.Registry getRegistry(); From ba1420ae57a4071492a78cc121c00629e9758d0e Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Wed, 6 Oct 2021 08:34:34 -0700 Subject: [PATCH 39/39] spotless Signed-off-by: Achal Shah --- .../src/main/java/feast/serving/config/CoreCondition.java | 4 ++-- .../main/java/feast/serving/config/RegistryCondition.java | 6 +++--- .../java/feast/serving/config/ServingServiceConfigV2.java | 2 +- .../java/feast/serving/registry/RegistryRepository.java | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/serving/src/main/java/feast/serving/config/CoreCondition.java b/serving/src/main/java/feast/serving/config/CoreCondition.java index 83fbff5..10dabfa 100644 --- a/serving/src/main/java/feast/serving/config/CoreCondition.java +++ b/serving/src/main/java/feast/serving/config/CoreCondition.java @@ -22,8 +22,8 @@ import org.springframework.core.type.AnnotatedTypeMetadata; /** - * A {@link Condition} to signal that the ServingService should get - * feature definitions and metadata from Core service. + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from Core service. */ public class CoreCondition implements Condition { @Override diff --git a/serving/src/main/java/feast/serving/config/RegistryCondition.java b/serving/src/main/java/feast/serving/config/RegistryCondition.java index f2cfb3b..621d124 100644 --- a/serving/src/main/java/feast/serving/config/RegistryCondition.java +++ b/serving/src/main/java/feast/serving/config/RegistryCondition.java @@ -22,9 +22,9 @@ import org.springframework.core.type.AnnotatedTypeMetadata; /** - * A {@link Condition} to signal that the ServingService should get - * feature definitions and metadata from the Registry object. This is needed for versions of the feature store - * written by feast 0.10+. + * A {@link Condition} to signal that the ServingService should get feature definitions and metadata + * from the Registry object. This is needed for versions of the feature store written by feast + * 0.10+. */ public class RegistryCondition implements Condition { diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 02aff4d..d1ac636 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -139,7 +139,7 @@ public ServingServiceV2 registryBasedServingServiceV2( final FeastProperties.Store store = feastProperties.getActiveStore(); OnlineRetrieverV2 retrieverV2; - //TODO: Support more store types, and potentially use a plugin model here. + // TODO: Support more store types, and potentially use a plugin model here. switch (store.getType()) { case REDIS_CLUSTER: RedisClientAdapter redisClusterClient = diff --git a/serving/src/main/java/feast/serving/registry/RegistryRepository.java b/serving/src/main/java/feast/serving/registry/RegistryRepository.java index 9f9ed40..79634ee 100644 --- a/serving/src/main/java/feast/serving/registry/RegistryRepository.java +++ b/serving/src/main/java/feast/serving/registry/RegistryRepository.java @@ -22,8 +22,8 @@ import feast.proto.serving.ServingAPIProto; /** - * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry object. - * This approach is needed for a feature store created using feast 0.10+. + * RegistryRepository allows the ServingService to retrieve feature definitions from a Registry + * object. This approach is needed for a feature store created using feast 0.10+. */ public interface RegistryRepository { RegistryProto.Registry getRegistry();