Skip to content

Commit 977f659

Browse files
woopwoop
authored
Remove Job Controller and Beam Ingestion (#1163)
* Remove job controller and ingestion java code Signed-off-by: Willem Pienaar <git@willem.co> * Remove feature statistics from Feast Core Signed-off-by: Willem Pienaar <git@willem.co> * Remove JC and ingestion dependencies in Serving and Core Signed-off-by: Willem Pienaar <git@willem.co> * Remove unused statistics and job controller protobufs and related code Signed-off-by: Willem Pienaar <git@willem.co> * Remove Docker Pom addition for serving and core Signed-off-by: Willem Pienaar <git@willem.co> * Remove BQ tests and beam dependencies Signed-off-by: Willem Pienaar <git@willem.co> * Remove statistics test Signed-off-by: Willem Pienaar <git@willem.co> * Add missing test dependencies Signed-off-by: Willem Pienaar <git@willem.co> * Add BigQueryDataSource generator back Signed-off-by: Willem Pienaar <git@willem.co> * Add okhttp into serving dependencies Signed-off-by: Willem Pienaar <git@willem.co> * Add createBigQueryDataSourceSpec Signed-off-by: Willem Pienaar <git@willem.co>
1 parent 5962756 commit 977f659

File tree

205 files changed

+31
-21527
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

205 files changed

+31
-21527
lines changed

‎common-test/src/main/java/feast/common/it/DataGenerator.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat;
2727
import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat;
2828
import feast.proto.core.DataSourceProto.DataSource;
29-
import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions;
3029
import feast.proto.core.DataSourceProto.DataSource.FileOptions;
3130
import feast.proto.core.DataSourceProto.DataSource.KafkaOptions;
3231
import feast.proto.core.DataSourceProto.DataSource.KinesisOptions;
@@ -107,10 +106,6 @@ public static StoreProto.Store createStore(
107106
StoreProto.Store.RedisConfig redisConfig =
108107
StoreProto.Store.RedisConfig.newBuilder().build();
109108
return builder.setRedisConfig(redisConfig).build();
110-
case BIGQUERY:
111-
StoreProto.Store.BigQueryConfig bqConfig =
112-
StoreProto.Store.BigQueryConfig.newBuilder().build();
113-
return builder.setBigqueryConfig(bqConfig).build();
114109
case REDIS_CLUSTER:
115110
StoreProto.Store.RedisClusterConfig redisClusterConfig =
116111
StoreProto.Store.RedisClusterConfig.newBuilder().build();
@@ -289,7 +284,8 @@ public static DataSource createBigQueryDataSourceSpec(
289284
String bigQueryTableRef, String timestampColumn, String datePartitionColumn) {
290285
return DataSource.newBuilder()
291286
.setType(DataSource.SourceType.BATCH_BIGQUERY)
292-
.setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build())
287+
.setBigqueryOptions(
288+
DataSource.BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build())
293289
.setEventTimestampColumn(timestampColumn)
294290
.setDatePartitionColumn(datePartitionColumn)
295291
.build();

‎common-test/src/main/java/feast/common/it/SimpleJcClient.java‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

‎core/pom.xml‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,6 @@
8080
<version>${project.version}</version>
8181
<scope>test</scope>
8282
</dependency>
83-
<dependency>
84-
<groupId>dev.feast</groupId>
85-
<artifactId>feast-storage-connector-bigquery</artifactId>
86-
<version>${project.version}</version>
87-
<scope>compile</scope>
88-
</dependency>
8983

9084
<!-- Hot reloading for Spring Boot. spring-boot-maven-plugin removes
9185
this automatically when packaging. -->
@@ -312,5 +306,21 @@
312306
<version>2.27.0</version>
313307
<scope>test</scope>
314308
</dependency>
309+
<dependency>
310+
<groupId>org.apache.avro</groupId>
311+
<artifactId>avro</artifactId>
312+
<version>1.8.2</version>
313+
<scope>test</scope>
314+
</dependency>
315+
<dependency>
316+
<groupId>com.squareup.okhttp</groupId>
317+
<artifactId>okhttp</artifactId>
318+
<version>2.7.4</version>
319+
<scope>test</scope>
320+
</dependency>
321+
<dependency>
322+
<groupId>io.grpc</groupId>
323+
<artifactId>grpc-testing</artifactId>
324+
</dependency>
315325
</dependencies>
316326
</project>

‎core/src/main/java/feast/core/controller/CoreServiceRestController.java‎

Lines changed: 1 addition & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,7 @@
2222
import feast.core.model.Project;
2323
import feast.core.service.ProjectService;
2424
import feast.core.service.SpecService;
25-
import feast.core.service.StatsService;
2625
import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse;
27-
import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest;
28-
import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest.Builder;
29-
import feast.proto.core.CoreServiceProto.GetFeatureStatisticsResponse;
3026
import feast.proto.core.CoreServiceProto.ListEntitiesRequest;
3127
import feast.proto.core.CoreServiceProto.ListEntitiesResponse;
3228
import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest;
@@ -36,7 +32,6 @@
3632
import feast.proto.core.CoreServiceProto.ListFeaturesRequest;
3733
import feast.proto.core.CoreServiceProto.ListFeaturesResponse;
3834
import feast.proto.core.CoreServiceProto.ListProjectsResponse;
39-
import java.io.IOException;
4035
import java.time.LocalDate;
4136
import java.time.LocalTime;
4237
import java.time.ZoneOffset;
@@ -64,18 +59,13 @@ public class CoreServiceRestController {
6459

6560
private final FeastProperties feastProperties;
6661
private SpecService specService;
67-
private StatsService statsService;
6862
private ProjectService projectService;
6963

7064
@Autowired
7165
public CoreServiceRestController(
72-
FeastProperties feastProperties,
73-
SpecService specService,
74-
StatsService statsService,
75-
ProjectService projectService) {
66+
FeastProperties feastProperties, SpecService specService, ProjectService projectService) {
7667
this.feastProperties = feastProperties;
7768
this.specService = specService;
78-
this.statsService = statsService;
7969
this.projectService = projectService;
8070
}
8171

@@ -133,58 +123,6 @@ public ListFeaturesResponse listFeatures(
133123
return specService.listFeatures(filterBuilder.build());
134124
}
135125

136-
/**
137-
* GET /feature-statistics : Fetches statistics for a dataset speficied by the parameters. Either
138-
* both (start_date, end_date) need to be given or ingestion_ids are required. If both are given,
139-
* (start_date, end_date) will be ignored.
140-
*
141-
* @param ingestionIds Request Parameter: List of ingestion IDs. If missing, both startDate and
142-
* endDate should be provided.
143-
* @param startDate Request Parameter: UTC+0 starting date (inclusive) in the ISO format, from
144-
* <code>0001-01-01</code> to <code>9999-12-31</code>. Time given will be ignored. This
145-
* parameter will be ignored if any ingestionIds is provided.
146-
* @param endDate Request Parameter: UTC+0 ending date (exclusive) in the ISO format, from <code>
147-
* 0001-01-01</code> to <code>9999-12-31</code>. Time given will be ignored. This parameter
148-
* will be ignored if any ingestionIds is provided.
149-
* @param store Request Parameter: The name of the historical store used in Feast Serving. Online
150-
* store is not allowed.
151-
* @param featureSetId Request Parameter: Feature set ID, which has the form of <code>
152-
* project/feature_set_name</code>.
153-
* @param forceRefresh Request Parameter: whether to override the values in the cache. Accepts
154-
* <code>true</code>, <code>false</code>.
155-
* @param features (Optional) Request Parameter: List of features. If none provided, all features
156-
* in the feature set will be used for statistics.
157-
* @return (200 OK) Returns {@link GetFeatureStatisticsResponse} in JSON.
158-
*/
159-
@RequestMapping(value = "/v1/feature-statistics", method = RequestMethod.GET)
160-
public GetFeatureStatisticsResponse getFeatureStatistics(
161-
@RequestParam(name = "feature_set_id") String featureSetId,
162-
@RequestParam(required = false) Optional<String[]> features,
163-
@RequestParam String store,
164-
@RequestParam(name = "start_date", required = false) Optional<String> startDate,
165-
@RequestParam(name = "end_date", required = false) Optional<String> endDate,
166-
@RequestParam(name = "ingestion_ids", required = false) Optional<String[]> ingestionIds,
167-
@RequestParam(name = "force_refresh") boolean forceRefresh)
168-
throws IOException {
169-
170-
Builder requestBuilder =
171-
GetFeatureStatisticsRequest.newBuilder()
172-
.setForceRefresh(forceRefresh)
173-
.setFeatureSetId(featureSetId)
174-
.setStore(store);
175-
176-
// set optional request parameters if they are provided
177-
features.ifPresent(theFeatures -> requestBuilder.addAllFeatures(Arrays.asList(theFeatures)));
178-
startDate.ifPresent(
179-
startDateStr -> requestBuilder.setStartDate(utcTimeStringToTimestamp(startDateStr)));
180-
endDate.ifPresent(
181-
endDateStr -> requestBuilder.setEndDate(utcTimeStringToTimestamp(endDateStr)));
182-
ingestionIds.ifPresent(
183-
theIngestionIds -> requestBuilder.addAllIngestionIds(Arrays.asList(theIngestionIds)));
184-
185-
return statsService.getFeatureStatistics(requestBuilder.build());
186-
}
187-
188126
/**
189127
* GET /projects : Get the list of existing feast projects.
190128
*

‎core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎core/src/main/java/feast/core/grpc/CoreServiceImpl.java‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import feast.core.model.Project;
2626
import feast.core.service.ProjectService;
2727
import feast.core.service.SpecService;
28-
import feast.core.service.StatsService;
2928
import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase;
3029
import feast.proto.core.CoreServiceProto.*;
3130
import feast.proto.core.EntityProto.EntitySpecV2;
@@ -49,21 +48,18 @@ public class CoreServiceImpl extends CoreServiceImplBase {
4948

5049
private final FeastProperties feastProperties;
5150
private SpecService specService;
52-
private StatsService statsService;
5351
private ProjectService projectService;
5452
private final AuthorizationService authorizationService;
5553

5654
@Autowired
5755
public CoreServiceImpl(
5856
SpecService specService,
5957
ProjectService projectService,
60-
StatsService statsService,
6158
FeastProperties feastProperties,
6259
AuthorizationService authorizationService) {
6360
this.specService = specService;
6461
this.projectService = projectService;
6562
this.feastProperties = feastProperties;
66-
this.statsService = statsService;
6763
this.authorizationService = authorizationService;
6864
}
6965

@@ -188,32 +184,6 @@ public void listEntities(
188184
}
189185
}
190186

191-
@Override
192-
public void getFeatureStatistics(
193-
GetFeatureStatisticsRequest request,
194-
StreamObserver<GetFeatureStatisticsResponse> responseObserver) {
195-
try {
196-
GetFeatureStatisticsResponse response = statsService.getFeatureStatistics(request);
197-
responseObserver.onNext(response);
198-
responseObserver.onCompleted();
199-
} catch (IllegalArgumentException e) {
200-
log.error("Illegal arguments provided to GetFeatureStatistics method: ", e);
201-
responseObserver.onError(
202-
Status.INVALID_ARGUMENT
203-
.withDescription(e.getMessage())
204-
.withCause(e)
205-
.asRuntimeException());
206-
} catch (RetrievalException e) {
207-
log.error("Unable to fetch feature set requested in GetFeatureStatistics method: ", e);
208-
responseObserver.onError(
209-
Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException());
210-
} catch (Exception e) {
211-
log.error("Exception has occurred in GetFeatureStatistics method: ", e);
212-
responseObserver.onError(
213-
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
214-
}
215-
}
216-
217187
@Override
218188
public void updateFeatureSetStatus(
219189
UpdateFeatureSetStatusRequest request,

0 commit comments

Comments
 (0)