Skip to content

Commit 5b69331

Browse files
woopzhilingc
authored and
feast-ci-bot
committed
Project Namespacing (feast-dev#393)
* Implement project namespacing (without auth) * Update Protos, Java SDK, Golang SDK to support namespacing * Fixed Python SDK to support project namespacing protos * Add integration with projects, update code to be compliant with new protos * Move name, version and project back to spec * Update Feast Core and Feast Ingestion to support project namespacing * Update Core and Ingestion based on refactored FeatureSet proto * Remove entity dataset validation * Register feature sets first to speed up tests * Apply PR feast-dev#392 * Apply spotless * Order test output Co-authored-by: Chen Zhiling <chnzhlng@gmail.com>
1 parent 51cbdc7 commit 5b69331

139 files changed

Lines changed: 4666 additions & 2855 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.prow/scripts/test-end-to-end-batch.sh‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,9 @@ feast:
123123
124124
spring:
125125
jpa:
126-
properties.hibernate.format_sql: true
126+
properties.hibernate:
127+
format_sql: true
128+
event.merge.entity_copy_observer: allow
127129
hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
128130
hibernate.ddl-auto: update
129131
datasource:
@@ -167,7 +169,8 @@ bigquery_config:
167169
datasetId: $DATASET_NAME
168170
subscriptions:
169171
- name: "*"
170-
version: ">0"
172+
version: "*"
173+
project: "*"
171174
EOF
172175

173176
cat <<EOF > /tmp/serving.warehouse.application.yml

‎.prow/scripts/test-end-to-end.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,12 @@ feast:
115115
116116
spring:
117117
jpa:
118-
properties.hibernate.format_sql: true
118+
properties.hibernate:
119+
format_sql: true
120+
event.merge.entity_copy_observer: allow
119121
hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
120122
hibernate.ddl-auto: update
123+
121124
datasource:
122125
url: jdbc:postgresql://localhost:5432/postgres
123126
username: postgres
@@ -153,7 +156,8 @@ redis_config:
153156
port: 6379
154157
subscriptions:
155158
- name: "*"
156-
version: ">0"
159+
version: "*"
160+
project: "*"
157161
EOF
158162

159163
cat <<EOF > /tmp/serving.online.application.yml
@@ -182,6 +186,7 @@ grpc:
182186
spring:
183187
main:
184188
web-environment: false
189+
185190
EOF
186191

187192
nohup java -jar serving/target/feast-serving-$REVISION.jar \

‎CONTRIBUTING.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@ store {
6464
name: "SERVING"
6565
type: REDIS
6666
subscriptions {
67+
project: "*"
6768
name: "*"
68-
version: ">0"
69+
version: "*"
6970
}
7071
redis_config {
7172
host: "localhost"
@@ -76,8 +77,9 @@ store {
7677
name: "WAREHOUSE"
7778
type: BIGQUERY
7879
subscriptions {
80+
project: "*"
7981
name: "*"
80-
version: ">0"
82+
version: "*"
8183
}
8284
bigquery_config {
8385
project_id: "my-google-project-id"

‎core/src/main/java/feast/core/config/FeatureStreamConfig.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ public Source getDefaultSource(FeastProperties feastProperties) {
5252
String topicName = streamProperties.getOptions().get("topic");
5353
Map<String, Object> map = new HashMap<>();
5454
map.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
55-
map.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG,
56-
DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG);
55+
map.put(
56+
AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG);
5757
AdminClient client = AdminClient.create(map);
5858

5959
NewTopic newTopic =

‎core/src/main/java/feast/core/dao/FeatureSetRepository.java‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,28 +19,31 @@
1919
import feast.core.model.FeatureSet;
2020
import java.util.List;
2121
import org.springframework.data.jpa.repository.JpaRepository;
22-
import org.springframework.data.jpa.repository.Query;
2322

2423
/** JPA repository supplying FeatureSet objects keyed by id. */
2524
public interface FeatureSetRepository extends JpaRepository<FeatureSet, String> {
2625

2726
long count();
2827

29-
// Find feature set by name and version
30-
FeatureSet findFeatureSetByNameAndVersion(String name, Integer version);
28+
// Find single feature set by project, name, and version
29+
FeatureSet findFeatureSetByNameAndProject_NameAndVersion(
30+
String name, String project, Integer version);
3131

32-
// Find latest version of a feature set by name
33-
FeatureSet findFirstFeatureSetByNameOrderByVersionDesc(String name);
34-
35-
// find all versions of featureSets matching the given name.
36-
List<FeatureSet> findByName(String name);
37-
38-
// find all versions of featureSets with names matching the regex
39-
@Query(
40-
nativeQuery = true,
41-
value = "SELECT * FROM feature_sets " + "WHERE name LIKE ?1 ORDER BY name ASC, version ASC")
42-
List<FeatureSet> findByNameWithWildcardOrderByNameAscVersionAsc(String name);
32+
// Find single latest version of a feature set by project and name (LIKE)
33+
FeatureSet findFirstFeatureSetByNameLikeAndProject_NameOrderByVersionDesc(
34+
String name, String project);
4335

4436
// find all feature sets and order by name and version
4537
List<FeatureSet> findAllByOrderByNameAscVersionAsc();
38+
39+
// find all feature sets within a project and order by name and version
40+
List<FeatureSet> findAllByProject_NameOrderByNameAscVersionAsc(String project_name);
41+
42+
// find all versions of feature sets matching the given name pattern with a specific project.
43+
List<FeatureSet> findAllByNameLikeAndProject_NameOrderByNameAscVersionAsc(
44+
String name, String project_name);
45+
46+
// find all versions of feature sets matching the given name pattern and project pattern
47+
List<FeatureSet> findAllByNameLikeAndProject_NameLikeOrderByNameAscVersionAsc(
48+
String name, String project_name);
4649
}

‎core/src/main/java/feast/core/dao/ProjectRepository.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2019 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.core.dao;
18+
19+
import feast.core.model.Project;
20+
import java.util.List;
21+
import org.springframework.data.jpa.repository.JpaRepository;
22+
23+
/** JPA repository supplying Project objects keyed by id. */
24+
public interface ProjectRepository extends JpaRepository<Project, String> {
25+
26+
List<Project> findAllByArchivedIsFalse();
27+
}

‎core/src/main/java/feast/core/grpc/CoreServiceImpl.java‎

Lines changed: 78 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,35 @@
1616
*/
1717
package feast.core.grpc;
1818

19-
import com.google.protobuf.InvalidProtocolBufferException;
2019
import feast.core.CoreServiceGrpc.CoreServiceImplBase;
2120
import feast.core.CoreServiceProto.ApplyFeatureSetRequest;
2221
import feast.core.CoreServiceProto.ApplyFeatureSetResponse;
22+
import feast.core.CoreServiceProto.ArchiveProjectRequest;
23+
import feast.core.CoreServiceProto.ArchiveProjectResponse;
24+
import feast.core.CoreServiceProto.CreateProjectRequest;
25+
import feast.core.CoreServiceProto.CreateProjectResponse;
2326
import feast.core.CoreServiceProto.GetFeastCoreVersionRequest;
2427
import feast.core.CoreServiceProto.GetFeastCoreVersionResponse;
2528
import feast.core.CoreServiceProto.GetFeatureSetRequest;
2629
import feast.core.CoreServiceProto.GetFeatureSetResponse;
2730
import feast.core.CoreServiceProto.ListFeatureSetsRequest;
2831
import feast.core.CoreServiceProto.ListFeatureSetsResponse;
32+
import feast.core.CoreServiceProto.ListProjectsRequest;
33+
import feast.core.CoreServiceProto.ListProjectsResponse;
2934
import feast.core.CoreServiceProto.ListStoresRequest;
3035
import feast.core.CoreServiceProto.ListStoresResponse;
3136
import feast.core.CoreServiceProto.UpdateStoreRequest;
3237
import feast.core.CoreServiceProto.UpdateStoreResponse;
3338
import feast.core.exception.RetrievalException;
3439
import feast.core.grpc.interceptors.MonitoringInterceptor;
40+
import feast.core.model.Project;
41+
import feast.core.service.AccessManagementService;
3542
import feast.core.service.SpecService;
43+
import io.grpc.Status;
3644
import io.grpc.StatusRuntimeException;
3745
import io.grpc.stub.StreamObserver;
46+
import java.util.List;
47+
import java.util.stream.Collectors;
3848
import lombok.extern.slf4j.Slf4j;
3949
import org.lognet.springboot.grpc.GRpcService;
4050
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,10 +55,12 @@
4555
public class CoreServiceImpl extends CoreServiceImplBase {
4656

4757
private SpecService specService;
58+
private AccessManagementService accessManagementService;
4859

4960
@Autowired
50-
public CoreServiceImpl(SpecService specService) {
61+
public CoreServiceImpl(SpecService specService, AccessManagementService accessManagementService) {
5162
this.specService = specService;
63+
this.accessManagementService = accessManagementService;
5264
}
5365

5466
@Override
@@ -65,9 +77,10 @@ public void getFeatureSet(
6577
GetFeatureSetResponse response = specService.getFeatureSet(request);
6678
responseObserver.onNext(response);
6779
responseObserver.onCompleted();
68-
} catch (RetrievalException | InvalidProtocolBufferException | StatusRuntimeException e) {
80+
} catch (RetrievalException | StatusRuntimeException e) {
6981
log.error("Exception has occurred in GetFeatureSet method: ", e);
70-
responseObserver.onError(e);
82+
responseObserver.onError(
83+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
7184
}
7285
}
7386

@@ -78,9 +91,10 @@ public void listFeatureSets(
7891
ListFeatureSetsResponse response = specService.listFeatureSets(request.getFilter());
7992
responseObserver.onNext(response);
8093
responseObserver.onCompleted();
81-
} catch (RetrievalException | InvalidProtocolBufferException e) {
94+
} catch (RetrievalException | IllegalArgumentException e) {
8295
log.error("Exception has occurred in ListFeatureSet method: ", e);
83-
responseObserver.onError(e);
96+
responseObserver.onError(
97+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
8498
}
8599
}
86100

@@ -93,7 +107,8 @@ public void listStores(
93107
responseObserver.onCompleted();
94108
} catch (RetrievalException e) {
95109
log.error("Exception has occurred in ListStores method: ", e);
96-
responseObserver.onError(e);
110+
responseObserver.onError(
111+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
97112
}
98113
}
99114

@@ -104,9 +119,17 @@ public void applyFeatureSet(
104119
ApplyFeatureSetResponse response = specService.applyFeatureSet(request.getFeatureSet());
105120
responseObserver.onNext(response);
106121
responseObserver.onCompleted();
122+
} catch (org.hibernate.exception.ConstraintViolationException e) {
123+
log.error(
124+
"Unable to persist this feature set due to a constraint violation. Please ensure that"
125+
+ " field names are unique within the project namespace: ",
126+
e);
127+
responseObserver.onError(
128+
Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException());
107129
} catch (Exception e) {
108130
log.error("Exception has occurred in ApplyFeatureSet method: ", e);
109-
responseObserver.onError(e);
131+
responseObserver.onError(
132+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
110133
}
111134
}
112135

@@ -119,7 +142,53 @@ public void updateStore(
119142
responseObserver.onCompleted();
120143
} catch (Exception e) {
121144
log.error("Exception has occurred in UpdateStore method: ", e);
122-
responseObserver.onError(e);
145+
responseObserver.onError(
146+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
147+
}
148+
}
149+
150+
@Override
151+
public void createProject(
152+
CreateProjectRequest request, StreamObserver<CreateProjectResponse> responseObserver) {
153+
try {
154+
accessManagementService.createProject(request.getName());
155+
responseObserver.onNext(CreateProjectResponse.getDefaultInstance());
156+
responseObserver.onCompleted();
157+
} catch (Exception e) {
158+
log.error("Exception has occurred in the createProject method: ", e);
159+
responseObserver.onError(
160+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
161+
}
162+
}
163+
164+
@Override
165+
public void archiveProject(
166+
ArchiveProjectRequest request, StreamObserver<ArchiveProjectResponse> responseObserver) {
167+
try {
168+
accessManagementService.archiveProject(request.getName());
169+
responseObserver.onNext(ArchiveProjectResponse.getDefaultInstance());
170+
responseObserver.onCompleted();
171+
} catch (Exception e) {
172+
log.error("Exception has occurred in the createProject method: ", e);
173+
responseObserver.onError(
174+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
175+
}
176+
}
177+
178+
@Override
179+
public void listProjects(
180+
ListProjectsRequest request, StreamObserver<ListProjectsResponse> responseObserver) {
181+
try {
182+
List<Project> projects = accessManagementService.listProjects();
183+
responseObserver.onNext(
184+
ListProjectsResponse.newBuilder()
185+
.addAllProjects(projects.stream().map(Project::getName).collect(Collectors.toList()))
186+
.build());
187+
responseObserver.onCompleted();
188+
} catch (Exception e) {
189+
log.error("Exception has occurred in the listProjects method: ", e);
190+
responseObserver.onError(
191+
Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException());
123192
}
124193
}
125194
}

0 commit comments

Comments
 (0)