Skip to content

Commit cd50a3f

Browse files
committed
Use proto hash and remove hash from db
Signed-off-by: Terence <terencelimxp@gmail.com>
1 parent ccf4e9a commit cd50a3f

3 files changed

Lines changed: 31 additions & 35 deletions

File tree

core/src/main/java/feast/core/model/FeatureTable.java

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616
*/
1717
package feast.core.model;
1818

19-
import com.google.common.primitives.Longs;
20-
import com.google.protobuf.ByteString;
19+
import com.google.common.hash.Hashing;
2120
import com.google.protobuf.Duration;
2221
import com.google.protobuf.Timestamp;
2322
import feast.core.dao.EntityRepository;
@@ -26,12 +25,7 @@
2625
import feast.proto.core.FeatureProto.FeatureSpecV2;
2726
import feast.proto.core.FeatureTableProto;
2827
import feast.proto.core.FeatureTableProto.FeatureTableSpec;
29-
import java.util.Collection;
30-
import java.util.List;
31-
import java.util.Map;
32-
import java.util.Objects;
33-
import java.util.Optional;
34-
import java.util.Set;
28+
import java.util.*;
3529
import java.util.stream.Collectors;
3630
import javax.persistence.CascadeType;
3731
import javax.persistence.Column;
@@ -112,9 +106,6 @@ public class FeatureTable extends AbstractTimestampEntity {
112106
@Column(name = "is_deleted", nullable = false)
113107
private boolean isDeleted;
114108

115-
@Column(name = "metadata_hash", nullable = false)
116-
private long metadataHash;
117-
118109
public FeatureTable() {};
119110

120111
/**
@@ -155,13 +146,6 @@ public static FeatureTable fromProto(
155146
table.setStreamSource(DataSource.fromProto(spec.getStreamSource()));
156147
}
157148

158-
table.setMetadataHash(
159-
Objects.hash(
160-
table.getEntities(),
161-
table.getFeatures(),
162-
table.getBatchSource(),
163-
table.getStreamSource()));
164-
165149
return table;
166150
}
167151

@@ -217,11 +201,6 @@ public void updateFromProto(
217201
this.streamSource = null;
218202
}
219203

220-
// Update hash
221-
this.setMetadataHash(
222-
Objects.hash(
223-
this.getEntities(), this.getFeatures(), this.getBatchSource(), this.getStreamSource()));
224-
225204
// Set isDeleted to false
226205
this.setDeleted(false);
227206

@@ -234,14 +213,7 @@ public FeatureTableProto.FeatureTable toProto() {
234213
// Convert field types to Protobuf compatible types
235214
Timestamp creationTime = TypeConversion.convertTimestamp(getCreated());
236215
Timestamp updatedTime = TypeConversion.convertTimestamp(getLastUpdated());
237-
ByteString metadataHashBytes =
238-
ByteString.copyFrom(
239-
Longs.toByteArray(
240-
Objects.hash(
241-
this.getEntities(),
242-
this.getFeatures(),
243-
this.getBatchSource(),
244-
this.getStreamSource())));
216+
String metadataHashBytes = this.protoHash();
245217

246218
List<FeatureSpecV2> featureSpecs =
247219
getFeatures().stream().map(FeatureV2::toProto).collect(Collectors.toList());
@@ -316,6 +288,32 @@ public void delete() {
316288
this.setRevision(0);
317289
}
318290

291+
public String protoHash() {
292+
List<String> sortedEntities =
293+
this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toList());
294+
Collections.sort(sortedEntities);
295+
296+
List<FeatureV2> sortedFeatures = new ArrayList(this.getFeatures());
297+
List<FeatureSpecV2> sortedFeatureSpecs =
298+
sortedFeatures.stream().map(featureV2 -> featureV2.toProto()).collect(Collectors.toList());
299+
sortedFeatures.sort(Comparator.comparing(FeatureV2::getName));
300+
301+
DataSourceProto.DataSource streamSource = DataSourceProto.DataSource.getDefaultInstance();
302+
if (getStreamSource() != null) {
303+
streamSource = getStreamSource().toProto();
304+
}
305+
306+
FeatureTableSpec featureTableSpec =
307+
FeatureTableSpec.newBuilder()
308+
.addAllEntities(sortedEntities)
309+
.addAllFeatures(sortedFeatureSpecs)
310+
.setBatchSource(getBatchSource().toProto())
311+
.setStreamSource(streamSource)
312+
.setMaxAge(Duration.newBuilder().setSeconds(getMaxAgeSecs()).build())
313+
.build();
314+
return Hashing.murmur3_32().hashBytes(featureTableSpec.toByteArray()).toString();
315+
}
316+
319317
@Override
320318
public int hashCode() {
321319
return Objects.hash(
Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
1-
ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL;
2-
3-
ALTER TABLE feature_tables ADD COLUMN metadata_hash bigint NOT NULL;
1+
ALTER TABLE feature_tables ADD COLUMN is_deleted boolean NOT NULL;

protos/feast/core/FeatureTable.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,5 +79,5 @@ message FeatureTableMeta {
7979

8080
// Hash entities, features, batch_source and stream_source to inform JobService if
8181
// jobs should be restarted should hash change
82-
bytes hash = 4;
82+
string hash = 4;
8383
}

0 commit comments

Comments
 (0)