Skip to content

Commit fc53df1

Browse files
terryyylimTerence
terryyylim
and
Terence
authored
Add Common module (#801)
* Add common module * Refactor parts of codebase to use common module * Add blacklist functionality and test * Update common module pom * Address PR comments * Address PR comments * Remove subscriptions blacklist changes * Update param comment Co-authored-by: Terence <terence.limxp@go-jek.com>
1 parent ec98916 commit fc53df1

24 files changed

Lines changed: 479 additions & 116 deletions

File tree

‎common/pom.xml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright 2018-2020 The Feast Authors
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ https://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
~
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
23+
<parent>
24+
<artifactId>feast-parent</artifactId>
25+
<groupId>dev.feast</groupId>
26+
<version>${revision}</version>
27+
</parent>
28+
29+
<name>Feast Common</name>
30+
<description>Feast common module with functionality that can be reused</description>
31+
<artifactId>feast-common</artifactId>
32+
33+
<build>
34+
<plugins>
35+
<plugin>
36+
<groupId>org.apache.maven.plugins</groupId>
37+
<artifactId>maven-surefire-plugin</artifactId>
38+
<version>3.0.0-M4</version>
39+
<configuration>
40+
<argLine>-Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
41+
</configuration>
42+
</plugin>
43+
</plugins>
44+
</build>
45+
46+
<dependencies>
47+
<dependency>
48+
<groupId>dev.feast</groupId>
49+
<artifactId>datatypes-java</artifactId>
50+
<version>${project.version}</version>
51+
<scope>compile</scope>
52+
</dependency>
53+
<dependency>
54+
<groupId>junit</groupId>
55+
<artifactId>junit</artifactId>
56+
<version>4.12</version>
57+
<scope>test</scope>
58+
</dependency>
59+
</dependencies>
60+
</project>

serving/src/main/java/feast/serving/util/RefUtil.java renamed to common/src/main/java/feast/common/models/Feature.java

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* SPDX-License-Identifier: Apache-2.0
3-
* Copyright 2018-2019 The Feast Authors
3+
* Copyright 2018-2020 The Feast Authors
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.
@@ -14,25 +14,29 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package feast.serving.util;
17+
package feast.common.models;
1818

19-
import feast.proto.core.FeatureSetProto.FeatureSetSpec;
2019
import feast.proto.serving.ServingAPIProto.FeatureReference;
2120

22-
public class RefUtil {
23-
public static String generateFeatureStringRef(FeatureReference featureReference) {
21+
public class Feature {
22+
23+
/**
24+
* Accepts FeatureReference object and returns its reference in String
25+
* "project/featureset_name:feature_name".
26+
*
27+
* @param featureReference {@link FeatureReference}
28+
* @param ignoreProject Flag whether to return FeatureReference with project name
29+
* @return String format of FeatureReference
30+
*/
31+
public static String getFeatureStringRef(
32+
FeatureReference featureReference, boolean ignoreProject) {
2433
String ref = featureReference.getName();
2534
if (!featureReference.getFeatureSet().isEmpty()) {
2635
ref = featureReference.getFeatureSet() + ":" + ref;
2736
}
28-
if (!featureReference.getProject().isEmpty()) {
37+
if (!featureReference.getProject().isEmpty() && !ignoreProject) {
2938
ref = featureReference.getProject() + "/" + ref;
3039
}
3140
return ref;
3241
}
33-
34-
public static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) {
35-
String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName());
36-
return ref;
37-
}
3842
}

‎common/src/main/java/feast/common/models/FeatureSet.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.models;
18+
19+
import feast.proto.core.FeatureSetProto.FeatureSetSpec;
20+
import feast.proto.core.FeatureSetReferenceProto.FeatureSetReference;
21+
22+
public class FeatureSet {
23+
24+
/**
25+
* Accepts FeatureSetSpec object and returns its reference in String "project/featureset_name".
26+
*
27+
* @param featureSetSpec {@link FeatureSetSpec}
28+
* @return String format of FeatureSetReference
29+
*/
30+
public static String getFeatureSetStringRef(FeatureSetSpec featureSetSpec) {
31+
return String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName());
32+
}
33+
34+
/**
35+
* Accepts FeatureSetReference object and returns its reference in String
36+
* "project/featureset_name".
37+
*
38+
* @param featureSetReference {@link FeatureSetReference}
39+
* @return String format of FeatureSetReference
40+
*/
41+
public static String getFeatureSetStringRef(FeatureSetReference featureSetReference) {
42+
return String.format("%s/%s", featureSetReference.getProject(), featureSetReference.getName());
43+
}
44+
}

‎common/src/main/java/feast/common/models/Store.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.models;
18+
19+
import feast.proto.core.StoreProto.Store.Subscription;
20+
import java.util.List;
21+
import java.util.regex.Pattern;
22+
23+
public class Store {
24+
25+
/**
26+
* Accepts a Subscription class object and returns it in string format
27+
*
28+
* @param subscription Subscription class to be converted to string format
29+
* @return String formatted Subscription class
30+
*/
31+
public static String parseSubscriptionFrom(Subscription subscription) {
32+
if (subscription.getName().isEmpty() || subscription.getProject().isEmpty()) {
33+
throw new IllegalArgumentException(
34+
String.format("Missing arguments in subscription string: %s", subscription.toString()));
35+
}
36+
37+
return String.format("%s:%s", subscription.getProject(), subscription.getName());
38+
}
39+
40+
/**
41+
* Accepts a exclude parameter to determine whether to return subscriptions that are excluded.
42+
*
43+
* @param subscription String formatted Subscription to be converted to Subscription class
44+
* @return Subscription class with its respective attributes
45+
*/
46+
public static Subscription convertStringToSubscription(String subscription) {
47+
if (subscription.equals("")) {
48+
return Subscription.newBuilder().build();
49+
}
50+
String[] split = subscription.split(":");
51+
return Subscription.newBuilder().setProject(split[0]).setName(split[1]).build();
52+
}
53+
54+
/**
55+
* The current use of this function is to determine whether a FeatureRow is subscribed to a
56+
* Featureset.
57+
*
58+
* @param subscriptions List of Subscriptions available in Store
59+
* @param projectName Project name used for matching Subscription's Project
60+
* @param featureSetName Featureset name used for matching Subscription's Featureset
61+
* @return boolean flag to signify if FeatureRow is subscribed to Featureset
62+
*/
63+
public static boolean isSubscribedToFeatureSet(
64+
List<Subscription> subscriptions, String projectName, String featureSetName) {
65+
for (Subscription sub : subscriptions) {
66+
// If configuration missing, fail
67+
if (sub.getProject().isEmpty() || sub.getName().isEmpty()) {
68+
throw new IllegalArgumentException(
69+
String.format("Subscription is missing arguments: %s", sub.toString()));
70+
}
71+
72+
// If all wildcards, subscribe to everything
73+
if (sub.getProject().equals("*") || sub.getName().equals("*")) {
74+
return true;
75+
}
76+
77+
// Match project name
78+
if (!projectName.equals(sub.getProject())) {
79+
continue;
80+
}
81+
82+
// Convert wildcard to regex
83+
String subName = sub.getName();
84+
if (!sub.getName().contains(".*")) {
85+
subName = subName.replace("*", ".*");
86+
}
87+
88+
// Match feature set name to pattern
89+
Pattern pattern = Pattern.compile(subName);
90+
if (!pattern.matcher(featureSetName).matches()) {
91+
continue;
92+
}
93+
return true;
94+
}
95+
96+
return false;
97+
}
98+
}

‎common/src/test/java/feast/common/models/FeatureSetTest.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.common.models;
18+
19+
import static org.hamcrest.MatcherAssert.assertThat;
20+
import static org.hamcrest.core.IsEqual.equalTo;
21+
22+
import feast.proto.core.FeatureSetProto.EntitySpec;
23+
import feast.proto.core.FeatureSetProto.FeatureSetSpec;
24+
import feast.proto.core.FeatureSetProto.FeatureSpec;
25+
import feast.proto.core.FeatureSetReferenceProto;
26+
import feast.proto.types.ValueProto;
27+
import java.util.Arrays;
28+
import java.util.List;
29+
import org.junit.Before;
30+
import org.junit.Test;
31+
import org.tensorflow.metadata.v0.*;
32+
33+
public class FeatureSetTest {
34+
35+
private List<EntitySpec> entitySpecs;
36+
private List<FeatureSpec> featureSpecs;
37+
38+
@Before
39+
public void setUp() {
40+
// Entity Specs
41+
EntitySpec entitySpec1 =
42+
EntitySpec.newBuilder()
43+
.setName("entity1")
44+
.setValueType(ValueProto.ValueType.Enum.INT64)
45+
.build();
46+
EntitySpec entitySpec2 =
47+
EntitySpec.newBuilder()
48+
.setName("entity2")
49+
.setValueType(ValueProto.ValueType.Enum.INT64)
50+
.build();
51+
52+
// Feature Specs
53+
FeatureSpec featureSpec1 =
54+
FeatureSpec.newBuilder()
55+
.setName("feature1")
56+
.setValueType(ValueProto.ValueType.Enum.INT64)
57+
.setPresence(FeaturePresence.getDefaultInstance())
58+
.setShape(FixedShape.getDefaultInstance())
59+
.setDomain("mydomain")
60+
.build();
61+
FeatureSpec featureSpec2 =
62+
FeatureSpec.newBuilder()
63+
.setName("feature2")
64+
.setValueType(ValueProto.ValueType.Enum.INT64)
65+
.setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance())
66+
.setValueCount(ValueCount.getDefaultInstance())
67+
.setIntDomain(IntDomain.getDefaultInstance())
68+
.build();
69+
70+
entitySpecs = Arrays.asList(entitySpec1, entitySpec2);
71+
featureSpecs = Arrays.asList(featureSpec1, featureSpec2);
72+
}
73+
74+
@Test
75+
public void shouldReturnFeatureSetStringRef() {
76+
FeatureSetSpec featureSetSpec =
77+
FeatureSetSpec.newBuilder()
78+
.setProject("project1")
79+
.setName("featureSetWithConstraints")
80+
.addAllEntities(entitySpecs)
81+
.addAllFeatures(featureSpecs)
82+
.build();
83+
84+
FeatureSetReferenceProto.FeatureSetReference featureSetReference =
85+
FeatureSetReferenceProto.FeatureSetReference.newBuilder()
86+
.setName(featureSetSpec.getName())
87+
.setProject(featureSetSpec.getProject())
88+
.build();
89+
90+
String actualFeatureSetStringRef1 = FeatureSet.getFeatureSetStringRef(featureSetSpec);
91+
String actualFeatureSetStringRef2 = FeatureSet.getFeatureSetStringRef(featureSetReference);
92+
String expectedFeatureSetStringRef = "project1/featureSetWithConstraints";
93+
94+
assertThat(actualFeatureSetStringRef1, equalTo(expectedFeatureSetStringRef));
95+
assertThat(actualFeatureSetStringRef2, equalTo(expectedFeatureSetStringRef));
96+
}
97+
}

0 commit comments

Comments
 (0)