Skip to content

Commit 7ce74c6

Browse files
author
Oleksii Moskalenko
authored
Database Schema migration for RELEASE 0.6 with Flyway (#810)
* create baseline flyway migration * gradual schema migration * lint * first migration as baseline if db is not empty * rename migration * docs
1 parent fc53df1 commit 7ce74c6

10 files changed

Lines changed: 423 additions & 8 deletions

File tree

‎core/pom.xml‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
~
1717
-->
1818
<project xmlns="http://maven.apache.org/POM/4.0.0"
19-
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
2121
<modelVersion>4.0.0</modelVersion>
2222

2323
<parent>
@@ -60,6 +60,13 @@
6060
</execution>
6161
</executions>
6262
</plugin>
63+
64+
<plugin>
65+
<groupId>org.flywaydb</groupId>
66+
<artifactId>flyway-maven-plugin</artifactId>
67+
<version>${flyway.version}</version>
68+
</plugin>
69+
6370
</plugins>
6471
</build>
6572

@@ -84,9 +91,9 @@
8491
</dependency>
8592

8693
<dependency>
87-
<groupId>javax.inject</groupId>
88-
<artifactId>javax.inject</artifactId>
89-
<version>1</version>
94+
<groupId>javax.inject</groupId>
95+
<artifactId>javax.inject</artifactId>
96+
<version>1</version>
9097
</dependency>
9198
<!--compile "org.springframework.boot:spring-boot-starter-web:${springBootVersion}"-->
9299
<dependency>
@@ -246,5 +253,10 @@
246253
<version>6.1.2.Final</version>
247254
</dependency>
248255

256+
<dependency>
257+
<groupId>org.flywaydb</groupId>
258+
<artifactId>flyway-core</artifactId>
259+
<version>${flyway.version}</version>
260+
</dependency>
249261
</dependencies>
250262
</project>

‎core/src/main/java/feast/core/model/FeatureSet.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,13 @@ public class FeatureSet extends AbstractTimestampEntity {
7474

7575
// Source on which feature rows can be found
7676
@ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.EAGER)
77-
@JoinColumn(name = "source")
77+
@JoinColumn(name = "source_id", referencedColumnName = "pk")
7878
private Source source;
7979

80+
@Deprecated
81+
@Column(name = "source")
82+
private String deprecatedSource;
83+
8084
// Status of the feature set
8185
@Enumerated(EnumType.STRING)
8286
@Column(name = "status")

‎core/src/main/java/feast/core/model/Job.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ public JobBuilder setSource(Source source) {
6666
private Runner runner;
6767

6868
// Source type and config, derieved from job's source and stored as inline fields.
69+
@Enumerated(EnumType.STRING)
6970
@Column(name = "source_type")
7071
private SourceType sourceType;
7172

‎core/src/main/java/feast/core/model/Source.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,22 @@
3737
public class Source {
3838

3939
/** Source Id. Internal use only, do not use to identify the source. */
40-
@Id @GeneratedValue private long id;
40+
@Id
41+
@GeneratedValue
42+
@Column(name = "pk")
43+
private Integer id;
44+
45+
@Deprecated
46+
@Column(name = "id")
47+
private String deprecatedId;
48+
49+
@Deprecated
50+
@Column(name = "bootstrap_servers")
51+
private String bootstrapServers;
52+
53+
@Deprecated
54+
@Column(name = "topics")
55+
private String topics;
4156

4257
/** Type of the source */
4358
@Enumerated(EnumType.STRING)
@@ -55,6 +70,19 @@ public Source() {
5570
super();
5671
}
5772

73+
public String getConfig() {
74+
if ((config == null || config.isEmpty()) && bootstrapServers != null && topics != null) {
75+
config =
76+
KafkaSourceConfig.newBuilder()
77+
.setBootstrapServers(bootstrapServers)
78+
.setTopic(topics)
79+
.build()
80+
.toString();
81+
}
82+
83+
return config;
84+
}
85+
5886
/**
5987
* Construct a source facade object from a given proto object.
6088
*

‎core/src/main/resources/application.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,12 +91,14 @@ spring:
9191
merge:
9292
entity_copy_observer: allow
9393
hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
94-
hibernate.ddl-auto: update
94+
hibernate.ddl-auto: validate
9595
datasource:
9696
driverClassName: org.postgresql.Driver
9797
url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
9898
username: ${DB_USERNAME:postgres}
9999
password: ${DB_PASSWORD:password}
100+
flyway:
101+
baseline-on-migrate: true
100102

101103
management:
102104
metrics:

‎core/src/main/resources/db/migration/V1__Baseline.sql‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
--
2+
-- Dump of Feast Database (as of RELEASE 0.5)
3+
-- Baseline dump for migrating to flyway
4+
--
5+
6+
7+
SET statement_timeout = 0;
8+
SET lock_timeout = 0;
9+
SET idle_in_transaction_session_timeout = 0;
10+
SET client_encoding = 'UTF8';
11+
SET default_tablespace = '';
12+
13+
14+
CREATE TABLE entities (
15+
id bigint NOT NULL,
16+
name character varying(255),
17+
type character varying(255),
18+
feature_set_id bigint
19+
);
20+
21+
22+
CREATE TABLE feature_sets (
23+
id bigint NOT NULL,
24+
created timestamp without time zone NOT NULL,
25+
last_updated timestamp without time zone NOT NULL,
26+
labels text,
27+
max_age bigint,
28+
name character varying(255) NOT NULL,
29+
status character varying(255),
30+
project_name character varying(255),
31+
source character varying(255)
32+
);
33+
34+
35+
CREATE TABLE features (
36+
id bigint NOT NULL,
37+
archived boolean NOT NULL,
38+
bool_domain bytea,
39+
domain character varying(255),
40+
float_domain bytea,
41+
group_presence bytea,
42+
image_domain bytea,
43+
int_domain bytea,
44+
labels text,
45+
mid_domain bytea,
46+
name character varying(255),
47+
natural_language_domain bytea,
48+
presence bytea,
49+
shape bytea,
50+
string_domain bytea,
51+
struct_domain bytea,
52+
time_domain bytea,
53+
time_of_day_domain bytea,
54+
type character varying(255),
55+
url_domain bytea,
56+
value_count bytea,
57+
feature_set_id bigint
58+
);
59+
60+
61+
62+
CREATE SEQUENCE hibernate_sequence
63+
START WITH 1
64+
INCREMENT BY 1
65+
NO MINVALUE
66+
NO MAXVALUE
67+
CACHE 1;
68+
69+
70+
CREATE TABLE jobs (
71+
id character varying(255) NOT NULL,
72+
created timestamp without time zone NOT NULL,
73+
last_updated timestamp without time zone NOT NULL,
74+
ext_id character varying(255),
75+
runner character varying(255),
76+
status character varying(16),
77+
source_id character varying(255),
78+
store_name character varying(255)
79+
);
80+
81+
82+
CREATE TABLE jobs_feature_sets (
83+
job_id character varying(255) NOT NULL,
84+
feature_sets_id bigint NOT NULL
85+
);
86+
87+
88+
CREATE TABLE projects (
89+
name character varying(255) NOT NULL,
90+
archived boolean NOT NULL
91+
);
92+
93+
94+
CREATE TABLE sources (
95+
id character varying(255) NOT NULL,
96+
bootstrap_servers character varying(255),
97+
is_default boolean,
98+
topics character varying(255),
99+
type character varying(255) NOT NULL
100+
);
101+
102+
103+
CREATE TABLE stores (
104+
name character varying(255) NOT NULL,
105+
config oid NOT NULL,
106+
subscriptions character varying(255),
107+
type character varying(255) NOT NULL
108+
);
109+
110+
111+
ALTER TABLE ONLY entities
112+
ADD CONSTRAINT entities_pkey PRIMARY KEY (id);
113+
114+
115+
ALTER TABLE ONLY feature_sets
116+
ADD CONSTRAINT feature_sets_pkey PRIMARY KEY (id);
117+
118+
119+
ALTER TABLE ONLY features
120+
ADD CONSTRAINT features_pkey PRIMARY KEY (id);
121+
122+
123+
ALTER TABLE ONLY jobs
124+
ADD CONSTRAINT jobs_pkey PRIMARY KEY (id);
125+
126+
127+
ALTER TABLE ONLY projects
128+
ADD CONSTRAINT projects_pkey PRIMARY KEY (name);
129+
130+
131+
ALTER TABLE ONLY sources
132+
ADD CONSTRAINT sources_pkey PRIMARY KEY (id);
133+
134+
135+
ALTER TABLE ONLY stores
136+
ADD CONSTRAINT stores_pkey PRIMARY KEY (name);
137+
138+
139+
ALTER TABLE ONLY entities
140+
ADD CONSTRAINT uk4hredqqfh86prhp1hf08nofvk UNIQUE (name, feature_set_id);
141+
142+
143+
ALTER TABLE ONLY features
144+
ADD CONSTRAINT ukedouxmpcoev743cmstfwq25yp UNIQUE (name, feature_set_id);
145+
146+
147+
ALTER TABLE ONLY feature_sets
148+
ADD CONSTRAINT ukoajkc7tn9nwhodjrbcjri5jix UNIQUE (name, project_name);
149+
150+
151+
CREATE INDEX idx_jobs_feature_sets_feature_sets_id ON jobs_feature_sets USING btree (feature_sets_id);
152+
153+
CREATE INDEX idx_jobs_feature_sets_job_id ON jobs_feature_sets USING btree (job_id);
154+
155+
156+
ALTER TABLE ONLY feature_sets
157+
ADD CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh FOREIGN KEY (source) REFERENCES sources(id);
158+
159+
160+
ALTER TABLE ONLY jobs_feature_sets
161+
ADD CONSTRAINT fk2qt5yj45cr02spdhp59h4wpeg FOREIGN KEY (job_id) REFERENCES jobs(id);
162+
163+
164+
ALTER TABLE ONLY jobs
165+
ADD CONSTRAINT fk3dwuno3phk8j3iwdl4cckdqqd FOREIGN KEY (store_name) REFERENCES stores(name);
166+
167+
168+
ALTER TABLE ONLY features
169+
ADD CONSTRAINT fkfxcpsscvj0g89o4p5dx4insb1 FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id);
170+
171+
172+
ALTER TABLE ONLY jobs
173+
ADD CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f FOREIGN KEY (source_id) REFERENCES sources(id);
174+
175+
176+
ALTER TABLE ONLY entities
177+
ADD CONSTRAINT fkhyblh5sfunv00a8ums8ms9otq FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id);
178+
179+
180+
ALTER TABLE ONLY feature_sets
181+
ADD CONSTRAINT fkiiqcdeuuq9mf0tmt7jtnln3oa FOREIGN KEY (project_name) REFERENCES projects(name);
182+
183+
184+
ALTER TABLE ONLY jobs_feature_sets
185+
ADD CONSTRAINT fkroca9etjw89c48e8jays6jl4l FOREIGN KEY (feature_sets_id) REFERENCES feature_sets(id);

0 commit comments

Comments
 (0)