Skip to content

Commit df4b057

Browse files
pyalexpyalex
committed
keep store proto in JobStore
1 parent 8e5f32d commit df4b057

15 files changed

Lines changed: 211 additions & 188 deletions

‎core/src/main/java/feast/core/dao/JobRepository.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,5 @@ public interface JobRepository extends JpaRepository<Job, String> {
4141
List<Job> findByFeatureSetJobStatusesIn(List<FeatureSetJobStatus> featureSetsJobStatuses);
4242

4343
// find jobs by feast store name
44-
List<Job> findByStoresName(String storeName);
44+
List<Job> findByJobStoresIdStoreName(String storeName);
4545
}

‎core/src/main/java/feast/core/job/ConsolidatedJobStrategy.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,12 @@ public Job getOrCreateJob(Source source, Set<Store> stores) {
5050
.findFirstBySourceTypeAndSourceConfigAndStoreNameAndStatusNotInOrderByLastUpdatedDesc(
5151
source.getType(), source.getConfig(), null, JobStatus.getTerminalStates())
5252
.orElseGet(
53-
() ->
54-
Job.builder()
55-
.setSource(source)
56-
.setStores(stores)
57-
.setFeatureSetJobStatuses(new HashSet<>())
58-
.build());
53+
() -> {
54+
Job job =
55+
Job.builder().setSource(source).setFeatureSetJobStatuses(new HashSet<>()).build();
56+
job.setStores(stores);
57+
return job;
58+
});
5959
}
6060

6161
@Override

‎core/src/main/java/feast/core/job/JobPerStoreStrategy.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,16 @@ public Job getOrCreateJob(Source source, Set<Store> stores) {
5555
.findFirstBySourceTypeAndSourceConfigAndStoreNameAndStatusNotInOrderByLastUpdatedDesc(
5656
source.getType(), source.getConfig(), store.getName(), JobStatus.getTerminalStates())
5757
.orElseGet(
58-
() ->
59-
Job.builder()
60-
.setSource(source)
61-
.setStoreName(store.getName())
62-
.setStores(stores)
63-
.setFeatureSetJobStatuses(new HashSet<>())
64-
.build());
58+
() -> {
59+
Job job =
60+
Job.builder()
61+
.setSource(source)
62+
.setStoreName(store.getName())
63+
.setFeatureSetJobStatuses(new HashSet<>())
64+
.build();
65+
job.setStores(stores);
66+
return job;
67+
});
6568
}
6669

6770
@Override

‎core/src/main/java/feast/core/model/Job.java‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -76,16 +76,8 @@ public JobBuilder setSource(Source source) {
7676
private String sourceConfig;
7777

7878
// Sinks
79-
@ManyToMany
80-
@JoinTable(
81-
name = "jobs_stores",
82-
joinColumns = @JoinColumn(name = "job_id"),
83-
inverseJoinColumns = @JoinColumn(name = "store_name"),
84-
indexes = {
85-
@Index(name = "idx_jobs_stores_job_id", columnList = "job_id"),
86-
@Index(name = "idx_jobs_stores_store_name", columnList = "store_name")
87-
})
88-
private Set<Store> stores;
79+
@OneToMany(mappedBy = "job", cascade = CascadeType.ALL)
80+
private Set<JobStore> jobStores = new HashSet<>();
8981

9082
@Deprecated
9183
@Column(name = "store_name")
@@ -144,6 +136,20 @@ public void addAllFeatureSets(Set<FeatureSet> featureSets) {
144136
}
145137
}
146138

139+
public Set<Store> getStores() {
140+
return getJobStores().stream()
141+
.map(JobStore::getStoreProto)
142+
.map(Store::fromProto)
143+
.collect(Collectors.toSet());
144+
}
145+
146+
public void setStores(Set<Store> stores) {
147+
jobStores = new HashSet<>();
148+
for (Store store : stores) {
149+
jobStores.add(new JobStore(this, store));
150+
}
151+
}
152+
147153
/**
148154
* Convert a job model to ingestion job proto
149155
*
@@ -177,21 +183,21 @@ public IngestionJobProto.IngestionJob toProto() throws InvalidProtocolBufferExce
177183
public Job clone() {
178184
Job job =
179185
Job.builder()
180-
.setStores(getStores())
181186
.setStoreName(getStoreName())
182187
.setSourceConfig(getSourceConfig())
183188
.setSourceType(getSourceType())
184189
.setFeatureSetJobStatuses(new HashSet<>())
185190
.setRunner(getRunner())
186191
.setStatus(JobStatus.UNKNOWN)
187192
.build();
193+
job.setStores(getStores());
188194
job.addAllFeatureSets(getFeatureSets());
189195
return job;
190196
}
191197

192198
@Override
193199
public int hashCode() {
194-
return Objects.hash(getSource(), this.stores, this.runner);
200+
return Objects.hash(getSource(), getStores(), this.runner);
195201
}
196202

197203
@Override
@@ -204,7 +210,7 @@ public boolean equals(Object obj) {
204210
return false;
205211
} else if (!getSource().equals(other.getSource())) {
206212
return false;
207-
} else if (!stores.equals(other.stores)) {
213+
} else if (!getStores().equals(other.getStores())) {
208214
return false;
209215
}
210216
return true;

‎core/src/main/java/feast/core/model/JobStore.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright 2018-2020 The Feast Authors
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package feast.core.model;
18+
19+
import com.google.common.base.Objects;
20+
import com.google.protobuf.InvalidProtocolBufferException;
21+
import feast.proto.core.StoreProto;
22+
import java.io.ByteArrayOutputStream;
23+
import java.io.IOException;
24+
import java.io.Serializable;
25+
import javax.persistence.*;
26+
import javax.persistence.Entity;
27+
import lombok.AllArgsConstructor;
28+
import lombok.EqualsAndHashCode;
29+
import lombok.Getter;
30+
import lombok.Setter;
31+
32+
@Entity
33+
@Table(
34+
name = "jobs_stores",
35+
indexes = {
36+
@Index(name = "idx_jobs_stores_job_id", columnList = "job_id"),
37+
@Index(name = "idx_jobs_stores_store_name", columnList = "store_name")
38+
})
39+
@Getter
40+
@Setter
41+
public class JobStore {
42+
@Embeddable
43+
@EqualsAndHashCode
44+
@AllArgsConstructor
45+
public static class JobStoreKey implements Serializable {
46+
public JobStoreKey() {}
47+
48+
@Column(name = "job_id")
49+
String jobId;
50+
51+
@Column(name = "store_name")
52+
String storeName;
53+
}
54+
55+
@EmbeddedId private JobStoreKey id = new JobStoreKey();
56+
57+
@ManyToOne
58+
@MapsId("jobId")
59+
@JoinColumn(name = "job_id")
60+
private Job job;
61+
62+
@Column(name = "store_proto", nullable = false)
63+
@Lob
64+
private byte[] storeProto;
65+
66+
public JobStore() {}
67+
68+
public JobStore(Job job, Store store) {
69+
this.job = job;
70+
this.id.storeName = store.getName();
71+
try {
72+
setStoreProto(store.toProto());
73+
} catch (InvalidProtocolBufferException e) {
74+
throw new RuntimeException("Couldn't convert Store to proto. Reason: %s", e.getCause());
75+
}
76+
}
77+
78+
public StoreProto.Store getStoreProto() {
79+
try {
80+
return StoreProto.Store.parseFrom(storeProto);
81+
} catch (InvalidProtocolBufferException e) {
82+
return StoreProto.Store.newBuilder().build();
83+
}
84+
}
85+
86+
public void setStoreProto(StoreProto.Store storeProto) {
87+
ByteArrayOutputStream output = new ByteArrayOutputStream();
88+
try {
89+
storeProto.writeTo(output);
90+
} catch (IOException e) {
91+
throw new RuntimeException(
92+
String.format("Couldn't write StoreProto to byteArray: %s", e.getCause()));
93+
}
94+
95+
this.storeProto = output.toByteArray();
96+
}
97+
98+
@Override
99+
public boolean equals(Object o) {
100+
if (this == o) return true;
101+
if (o == null || getClass() != o.getClass()) return false;
102+
JobStore jobStore = (JobStore) o;
103+
return Objects.equal(id, jobStore.id) && Objects.equal(storeProto, jobStore.storeProto);
104+
}
105+
106+
@Override
107+
public int hashCode() {
108+
return Objects.hashCode(id, storeProto);
109+
}
110+
}

‎core/src/main/java/feast/core/model/Store.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
@AllArgsConstructor
4848
@Entity
4949
@Table(name = "stores")
50-
public class Store extends AbstractTimestampEntity {
50+
public class Store {
5151

5252
// Name of the store. Must be unique
5353
@Id

‎core/src/main/java/feast/core/service/JobCoordinatorService.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -227,10 +227,6 @@ private boolean jobRequiresUpgrade(Job job, Set<Store> stores) {
227227
return true;
228228
}
229229

230-
if (stores.stream().anyMatch(s -> s.getLastUpdated().after(job.getCreated()))) {
231-
return true;
232-
}
233-
234230
return false;
235231
}
236232

‎core/src/main/java/feast/core/service/JobService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public ListIngestionJobsResponse listJobs(ListIngestionJobsRequest request)
108108
// multiple filters can apply together in an 'and' operation
109109
if (!filter.getStoreName().isEmpty()) {
110110
// find jobs by name
111-
List<Job> jobs = this.jobRepository.findByStoresName(filter.getStoreName());
111+
List<Job> jobs = this.jobRepository.findByJobStoresIdStoreName(filter.getStoreName());
112112
Set<String> jobIds = jobs.stream().map(Job::getId).collect(Collectors.toSet());
113113
matchingJobIds = this.mergeResults(matchingJobIds, jobIds);
114114
}

‎core/src/main/resources/db/migration/V2.4__Store_Timestamps.sql‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎core/src/main/resources/db/migration/V2.4__Store_proto.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE jobs_stores ADD COLUMN store_proto oid not null;

0 commit comments

Comments
 (0)