Skip to content

Commit 306b146

Browse files
Chen Zhilingfeast-ci-bot
Chen Zhiling
authored and
feast-ci-bot
committed
Break up queries for point in time correctness joins (#347)
* Break up queries for point in time correctness joins; plus some gentle refactoring Cleanup; Remove lombok dependency Add end to end tests for batch retrieval Only make necessary functions transactional Fix typo for e2e test file Fix sorting for windowing Add left join so that rows are consistent Remove UUID * Replace listener with ExecutorService Clean up threads when error is thrown
1 parent 15b0445 commit 306b146

47 files changed

Lines changed: 1089 additions & 1034 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.prow/scripts/test-end-to-end.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ ORIGINAL_DIR=$(pwd)
206206
cd tests/e2e
207207

208208
set +e
209-
pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
209+
pytest basic-ingest-redis-serving.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
210210
TEST_EXIT_CODE=$?
211211

212212
cd ${ORIGINAL_DIR}

‎core/src/main/java/feast/core/grpc/CoreServiceImpl.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ public void getFeastCoreVersion(
7272
}
7373

7474
@Override
75-
@Transactional
7675
public void getFeatureSet(
7776
GetFeatureSetRequest request, StreamObserver<GetFeatureSetResponse> responseObserver) {
7877
try {
@@ -86,7 +85,6 @@ public void getFeatureSet(
8685
}
8786

8887
@Override
89-
@Transactional
9088
public void listFeatureSets(
9189
ListFeatureSetsRequest request, StreamObserver<ListFeatureSetsResponse> responseObserver) {
9290
try {
@@ -100,7 +98,6 @@ public void listFeatureSets(
10098
}
10199

102100
@Override
103-
@Transactional
104101
public void listStores(
105102
ListStoresRequest request, StreamObserver<ListStoresResponse> responseObserver) {
106103
try {
@@ -114,7 +111,6 @@ public void listStores(
114111
}
115112

116113
@Override
117-
@Transactional
118114
public void applyFeatureSet(
119115
ApplyFeatureSetRequest request, StreamObserver<ApplyFeatureSetResponse> responseObserver) {
120116
try {
@@ -153,7 +149,6 @@ public void applyFeatureSet(
153149
}
154150

155151
@Override
156-
@Transactional
157152
public void updateStore(
158153
UpdateStoreRequest request, StreamObserver<UpdateStoreResponse> responseObserver) {
159154
try {

‎core/src/main/java/feast/core/service/JobCoordinatorService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import lombok.extern.slf4j.Slf4j;
4242
import org.springframework.beans.factory.annotation.Autowired;
4343
import org.springframework.stereotype.Service;
44+
import org.springframework.transaction.annotation.Transactional;
4445

4546
@Slf4j
4647
@Service
@@ -60,6 +61,7 @@ public JobCoordinatorService(JobInfoRepository jobInfoRepository, JobManager job
6061
* there has been no change in the featureSet, and there is a running job for the featureSet, this
6162
* method will do nothing.
6263
*/
64+
@Transactional
6365
public JobInfo startOrUpdateJob(
6466
List<FeatureSetSpec> featureSetSpecs, SourceProto.Source sourceSpec, StoreProto.Store store) {
6567
Source source = Source.fromProto(sourceSpec);

‎core/src/main/java/feast/core/service/SpecService.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import lombok.extern.slf4j.Slf4j;
5151
import org.springframework.beans.factory.annotation.Autowired;
5252
import org.springframework.stereotype.Service;
53+
import org.springframework.transaction.annotation.Transactional;
5354

5455
/**
5556
* Facilitates management of specs within the Feast registry. This includes getting existing specs
@@ -164,6 +165,7 @@ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter fil
164165
* @param filter filter containing the desired store name
165166
* @return ListStoresResponse containing list of stores found matching the filter
166167
*/
168+
@Transactional
167169
public ListStoresResponse listStores(ListStoresRequest.Filter filter) {
168170
try {
169171
String name = filter.getName();
@@ -241,6 +243,7 @@ public ApplyFeatureSetResponse applyFeatureSet(FeatureSetSpec newFeatureSetSpec)
241243
* @return UpdateStoreResponse containing the new store definition
242244
* @throws InvalidProtocolBufferException
243245
*/
246+
@Transactional
244247
public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest)
245248
throws InvalidProtocolBufferException {
246249
StoreProto.Store newStoreProto = updateStoreRequest.getStore();

‎serving/pom.xml‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,11 +203,18 @@
203203
<artifactId>google-cloud-storage</artifactId>
204204
</dependency>
205205

206-
<!--compileOnly 'org.projectlombok:lombok:1.18.2'-->
207206
<dependency>
208-
<groupId>org.projectlombok</groupId>
209-
<artifactId>lombok</artifactId>
207+
<groupId>com.google.auto.value</groupId>
208+
<artifactId>auto-value-annotations</artifactId>
209+
<version>1.6.6</version>
210210
</dependency>
211+
<dependency>
212+
<groupId>com.google.auto.value</groupId>
213+
<artifactId>auto-value</artifactId>
214+
<version>1.6.6</version>
215+
<scope>provided</scope>
216+
</dependency>
217+
211218

212219
<!--testCompile "io.grpc:grpc-testing:${grpcVersion}"-->
213220
<dependency>

‎serving/src/main/java/feast/serving/FeastProperties.java‎

Lines changed: 120 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,8 @@
2222
// https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties
2323

2424
import java.util.Map;
25-
import lombok.Getter;
26-
import lombok.Setter;
2725
import org.springframework.boot.context.properties.ConfigurationProperties;
2826

29-
@Getter
30-
@Setter
3127
@ConfigurationProperties(prefix = "feast")
3228
public class FeastProperties {
3329
private String version;
@@ -37,27 +33,141 @@ public class FeastProperties {
3733
private JobProperties jobs;
3834
private TracingProperties tracing;
3935

40-
@Setter
41-
@Getter
36+
public String getVersion() {
37+
return this.version;
38+
}
39+
40+
public String getCoreHost() {
41+
return this.coreHost;
42+
}
43+
44+
public int getCoreGrpcPort() {
45+
return this.coreGrpcPort;
46+
}
47+
48+
public StoreProperties getStore() {
49+
return this.store;
50+
}
51+
52+
public JobProperties getJobs() {
53+
return this.jobs;
54+
}
55+
56+
public TracingProperties getTracing() {
57+
return this.tracing;
58+
}
59+
60+
public void setVersion(String version) {
61+
this.version = version;
62+
}
63+
64+
public void setCoreHost(String coreHost) {
65+
this.coreHost = coreHost;
66+
}
67+
68+
public void setCoreGrpcPort(int coreGrpcPort) {
69+
this.coreGrpcPort = coreGrpcPort;
70+
}
71+
72+
public void setStore(StoreProperties store) {
73+
this.store = store;
74+
}
75+
76+
public void setJobs(JobProperties jobs) {
77+
this.jobs = jobs;
78+
}
79+
80+
public void setTracing(TracingProperties tracing) {
81+
this.tracing = tracing;
82+
}
83+
4284
public static class StoreProperties {
4385
private String configPath;
4486
private int redisPoolMaxSize;
4587
private int redisPoolMaxIdle;
88+
89+
public String getConfigPath() {
90+
return this.configPath;
91+
}
92+
93+
public int getRedisPoolMaxSize() {
94+
return this.redisPoolMaxSize;
95+
}
96+
97+
public int getRedisPoolMaxIdle() {
98+
return this.redisPoolMaxIdle;
99+
}
100+
101+
public void setConfigPath(String configPath) {
102+
this.configPath = configPath;
103+
}
104+
105+
public void setRedisPoolMaxSize(int redisPoolMaxSize) {
106+
this.redisPoolMaxSize = redisPoolMaxSize;
107+
}
108+
109+
public void setRedisPoolMaxIdle(int redisPoolMaxIdle) {
110+
this.redisPoolMaxIdle = redisPoolMaxIdle;
111+
}
46112
}
47113

48-
@Setter
49-
@Getter
50114
public static class JobProperties {
51115
private String stagingLocation;
52116
private String storeType;
53117
private Map<String, String> storeOptions;
118+
119+
public String getStagingLocation() {
120+
return this.stagingLocation;
121+
}
122+
123+
public String getStoreType() {
124+
return this.storeType;
125+
}
126+
127+
public Map<String, String> getStoreOptions() {
128+
return this.storeOptions;
129+
}
130+
131+
public void setStagingLocation(String stagingLocation) {
132+
this.stagingLocation = stagingLocation;
133+
}
134+
135+
public void setStoreType(String storeType) {
136+
this.storeType = storeType;
137+
}
138+
139+
public void setStoreOptions(Map<String, String> storeOptions) {
140+
this.storeOptions = storeOptions;
141+
}
54142
}
55143

56-
@Setter
57-
@Getter
58144
public static class TracingProperties {
59145
private boolean enabled;
60146
private String tracerName;
61147
private String serviceName;
148+
149+
public boolean isEnabled() {
150+
return this.enabled;
151+
}
152+
153+
public String getTracerName() {
154+
return this.tracerName;
155+
}
156+
157+
public String getServiceName() {
158+
return this.serviceName;
159+
}
160+
161+
public void setEnabled(boolean enabled) {
162+
this.enabled = enabled;
163+
}
164+
165+
public void setTracerName(String tracerName) {
166+
this.tracerName = tracerName;
167+
}
168+
169+
public void setServiceName(String serviceName) {
170+
this.serviceName = serviceName;
171+
}
62172
}
63173
}

‎serving/src/main/java/feast/serving/configuration/ServingServiceConfig.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,17 @@
3636
import feast.serving.service.ServingService;
3737
import io.opentracing.Tracer;
3838
import java.util.Map;
39-
import lombok.extern.slf4j.Slf4j;
39+
import org.slf4j.Logger;
4040
import org.springframework.context.annotation.Bean;
4141
import org.springframework.context.annotation.Configuration;
4242
import redis.clients.jedis.JedisPool;
4343
import redis.clients.jedis.JedisPoolConfig;
4444

45-
@Slf4j
4645
@Configuration
4746
public class ServingServiceConfig {
4847

48+
private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfig.class);
49+
4950
@Bean(name = "JobStore")
5051
public Store jobStoreDefinition(FeastProperties feastProperties) {
5152
JobProperties jobProperties = feastProperties.getJobs();

‎serving/src/main/java/feast/serving/configuration/SpecServiceConfig.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@
2424
import java.util.concurrent.Executors;
2525
import java.util.concurrent.ScheduledExecutorService;
2626
import java.util.concurrent.TimeUnit;
27-
import lombok.extern.slf4j.Slf4j;
27+
import org.slf4j.Logger;
2828
import org.springframework.beans.factory.annotation.Autowired;
2929
import org.springframework.context.annotation.Bean;
3030
import org.springframework.context.annotation.Configuration;
3131

32-
@Slf4j
3332
@Configuration
3433
public class SpecServiceConfig {
3534

35+
private static final Logger log = org.slf4j.LoggerFactory.getLogger(SpecServiceConfig.class);
3636
private String feastCoreHost;
3737
private int feastCorePort;
3838
private static final int CACHE_REFRESH_RATE_MINUTES = 1;

‎serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@
3232
import io.opentracing.Scope;
3333
import io.opentracing.Span;
3434
import io.opentracing.Tracer;
35-
import lombok.extern.slf4j.Slf4j;
3635
import org.lognet.springboot.grpc.GRpcService;
36+
import org.slf4j.Logger;
3737
import org.springframework.beans.factory.annotation.Autowired;
3838

39-
@Slf4j
4039
@GRpcService
4140
public class ServingServiceGRpcController extends ServingServiceImplBase {
41+
42+
private static final Logger log =
43+
org.slf4j.LoggerFactory.getLogger(ServingServiceGRpcController.class);
4244
private final ServingService servingService;
4345
private final String version;
4446
private final Tracer tracer;

0 commit comments

Comments
 (0)