diff --git a/core/pom.xml b/core/pom.xml
index c4b3c0f4aa5..838f46fda75 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -16,8 +16,8 @@
~
-->
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
4.0.0
@@ -60,6 +60,13 @@
+
+
+ org.flywaydb
+ flyway-maven-plugin
+ ${flyway.version}
+
+
@@ -79,9 +86,9 @@
- javax.inject
- javax.inject
- 1
+ javax.inject
+ javax.inject
+ 1
@@ -241,5 +248,10 @@
6.1.2.Final
+
+ org.flywaydb
+ flyway-core
+ ${flyway.version}
+
diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java
index e032c712fce..f24a5cc958e 100644
--- a/core/src/main/java/feast/core/model/FeatureSet.java
+++ b/core/src/main/java/feast/core/model/FeatureSet.java
@@ -74,9 +74,13 @@ public class FeatureSet extends AbstractTimestampEntity {
// Source on which feature rows can be found
@ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.EAGER)
- @JoinColumn(name = "source")
+ @JoinColumn(name = "source_id", referencedColumnName = "pk")
private Source source;
+ @Deprecated
+ @Column(name = "source")
+ private String deprecatedSource;
+
// Status of the feature set
@Enumerated(EnumType.STRING)
@Column(name = "status")
diff --git a/core/src/main/java/feast/core/model/Job.java b/core/src/main/java/feast/core/model/Job.java
index de2508d671b..fb8b071b81d 100644
--- a/core/src/main/java/feast/core/model/Job.java
+++ b/core/src/main/java/feast/core/model/Job.java
@@ -66,6 +66,7 @@ public JobBuilder setSource(Source source) {
private Runner runner;
// Source type and config, derieved from job's source and stored as inline fields.
+ @Enumerated(EnumType.STRING)
@Column(name = "source_type")
private SourceType sourceType;
diff --git a/core/src/main/java/feast/core/model/Source.java b/core/src/main/java/feast/core/model/Source.java
index 0b9f2a3e4c7..9ccbf5d490c 100644
--- a/core/src/main/java/feast/core/model/Source.java
+++ b/core/src/main/java/feast/core/model/Source.java
@@ -37,7 +37,22 @@
public class Source {
/** Source Id. Internal use only, do not use to identify the source. */
- @Id @GeneratedValue private long id;
+ @Id
+ @GeneratedValue
+ @Column(name = "pk")
+ private Integer id;
+
+ @Deprecated
+ @Column(name = "id")
+ private String deprecatedId;
+
+ @Deprecated
+ @Column(name = "bootstrap_servers")
+ private String bootstrapServers;
+
+ @Deprecated
+ @Column(name = "topics")
+ private String topics;
/** Type of the source */
@Enumerated(EnumType.STRING)
@@ -55,6 +70,19 @@ public Source() {
super();
}
+ public String getConfig() {
+ if ((config == null || config.isEmpty()) && bootstrapServers != null && topics != null) {
+ config =
+ KafkaSourceConfig.newBuilder()
+ .setBootstrapServers(bootstrapServers)
+ .setTopic(topics)
+ .build()
+ .toString();
+ }
+
+ return config;
+ }
+
/**
* Construct a source facade object from a given proto object.
*
diff --git a/core/src/main/resources/application.yml b/core/src/main/resources/application.yml
index c34af21495e..35390e7163e 100644
--- a/core/src/main/resources/application.yml
+++ b/core/src/main/resources/application.yml
@@ -91,12 +91,14 @@ spring:
merge:
entity_copy_observer: allow
hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
- hibernate.ddl-auto: update
+ hibernate.ddl-auto: validate
datasource:
driverClassName: org.postgresql.Driver
url: jdbc:postgresql://${DB_HOST:127.0.0.1}:${DB_PORT:5432}/${DB_DATABASE:postgres}
username: ${DB_USERNAME:postgres}
password: ${DB_PASSWORD:password}
+ flyway:
+ baseline-on-migrate: true
management:
metrics:
diff --git a/core/src/main/resources/db/migration/V1__Baseline.sql b/core/src/main/resources/db/migration/V1__Baseline.sql
new file mode 100644
index 00000000000..262e0586ef2
--- /dev/null
+++ b/core/src/main/resources/db/migration/V1__Baseline.sql
@@ -0,0 +1,185 @@
+--
+-- Dump of Feast Database (as of RELEASE 0.5)
+-- Baseline dump for migrating to flyway
+--
+
+
+SET statement_timeout = 0;
+SET lock_timeout = 0;
+SET idle_in_transaction_session_timeout = 0;
+SET client_encoding = 'UTF8';
+SET default_tablespace = '';
+
+
+CREATE TABLE entities (
+ id bigint NOT NULL,
+ name character varying(255),
+ type character varying(255),
+ feature_set_id bigint
+);
+
+
+CREATE TABLE feature_sets (
+ id bigint NOT NULL,
+ created timestamp without time zone NOT NULL,
+ last_updated timestamp without time zone NOT NULL,
+ labels text,
+ max_age bigint,
+ name character varying(255) NOT NULL,
+ status character varying(255),
+ project_name character varying(255),
+ source character varying(255)
+);
+
+
+CREATE TABLE features (
+ id bigint NOT NULL,
+ archived boolean NOT NULL,
+ bool_domain bytea,
+ domain character varying(255),
+ float_domain bytea,
+ group_presence bytea,
+ image_domain bytea,
+ int_domain bytea,
+ labels text,
+ mid_domain bytea,
+ name character varying(255),
+ natural_language_domain bytea,
+ presence bytea,
+ shape bytea,
+ string_domain bytea,
+ struct_domain bytea,
+ time_domain bytea,
+ time_of_day_domain bytea,
+ type character varying(255),
+ url_domain bytea,
+ value_count bytea,
+ feature_set_id bigint
+);
+
+
+
+CREATE SEQUENCE hibernate_sequence
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+CREATE TABLE jobs (
+ id character varying(255) NOT NULL,
+ created timestamp without time zone NOT NULL,
+ last_updated timestamp without time zone NOT NULL,
+ ext_id character varying(255),
+ runner character varying(255),
+ status character varying(16),
+ source_id character varying(255),
+ store_name character varying(255)
+);
+
+
+CREATE TABLE jobs_feature_sets (
+ job_id character varying(255) NOT NULL,
+ feature_sets_id bigint NOT NULL
+);
+
+
+CREATE TABLE projects (
+ name character varying(255) NOT NULL,
+ archived boolean NOT NULL
+);
+
+
+CREATE TABLE sources (
+ id character varying(255) NOT NULL,
+ bootstrap_servers character varying(255),
+ is_default boolean,
+ topics character varying(255),
+ type character varying(255) NOT NULL
+);
+
+
+CREATE TABLE stores (
+ name character varying(255) NOT NULL,
+ config oid NOT NULL,
+ subscriptions character varying(255),
+ type character varying(255) NOT NULL
+);
+
+
+ALTER TABLE ONLY entities
+ ADD CONSTRAINT entities_pkey PRIMARY KEY (id);
+
+
+ALTER TABLE ONLY feature_sets
+ ADD CONSTRAINT feature_sets_pkey PRIMARY KEY (id);
+
+
+ALTER TABLE ONLY features
+ ADD CONSTRAINT features_pkey PRIMARY KEY (id);
+
+
+ALTER TABLE ONLY jobs
+ ADD CONSTRAINT jobs_pkey PRIMARY KEY (id);
+
+
+ALTER TABLE ONLY projects
+ ADD CONSTRAINT projects_pkey PRIMARY KEY (name);
+
+
+ALTER TABLE ONLY sources
+ ADD CONSTRAINT sources_pkey PRIMARY KEY (id);
+
+
+ALTER TABLE ONLY stores
+ ADD CONSTRAINT stores_pkey PRIMARY KEY (name);
+
+
+ALTER TABLE ONLY entities
+ ADD CONSTRAINT uk4hredqqfh86prhp1hf08nofvk UNIQUE (name, feature_set_id);
+
+
+ALTER TABLE ONLY features
+ ADD CONSTRAINT ukedouxmpcoev743cmstfwq25yp UNIQUE (name, feature_set_id);
+
+
+ALTER TABLE ONLY feature_sets
+ ADD CONSTRAINT ukoajkc7tn9nwhodjrbcjri5jix UNIQUE (name, project_name);
+
+
+CREATE INDEX idx_jobs_feature_sets_feature_sets_id ON jobs_feature_sets USING btree (feature_sets_id);
+
+CREATE INDEX idx_jobs_feature_sets_job_id ON jobs_feature_sets USING btree (job_id);
+
+
+ALTER TABLE ONLY feature_sets
+ ADD CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh FOREIGN KEY (source) REFERENCES sources(id);
+
+
+ALTER TABLE ONLY jobs_feature_sets
+ ADD CONSTRAINT fk2qt5yj45cr02spdhp59h4wpeg FOREIGN KEY (job_id) REFERENCES jobs(id);
+
+
+ALTER TABLE ONLY jobs
+ ADD CONSTRAINT fk3dwuno3phk8j3iwdl4cckdqqd FOREIGN KEY (store_name) REFERENCES stores(name);
+
+
+ALTER TABLE ONLY features
+ ADD CONSTRAINT fkfxcpsscvj0g89o4p5dx4insb1 FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id);
+
+
+ALTER TABLE ONLY jobs
+ ADD CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f FOREIGN KEY (source_id) REFERENCES sources(id);
+
+
+ALTER TABLE ONLY entities
+ ADD CONSTRAINT fkhyblh5sfunv00a8ums8ms9otq FOREIGN KEY (feature_set_id) REFERENCES feature_sets(id);
+
+
+ALTER TABLE ONLY feature_sets
+ ADD CONSTRAINT fkiiqcdeuuq9mf0tmt7jtnln3oa FOREIGN KEY (project_name) REFERENCES projects(name);
+
+
+ALTER TABLE ONLY jobs_feature_sets
+ ADD CONSTRAINT fkroca9etjw89c48e8jays6jl4l FOREIGN KEY (feature_sets_id) REFERENCES feature_sets(id);
diff --git a/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql b/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql
new file mode 100644
index 00000000000..230a5f5c8c1
--- /dev/null
+++ b/core/src/main/resources/db/migration/V2__RELEASE_0.6_Generalizing_Source_AND_Extending_FeatureSetJobStatus_AND_Feature_Statistics.sql
@@ -0,0 +1,124 @@
+--- Feast Release 0.6
+
+--- New fields from FeatureSetJobStatus (version & deliveryStatus)
+
+ALTER TABLE jobs_feature_sets
+ ADD column version int4 default 0;
+
+ALTER TABLE jobs_feature_sets
+ ADD column delivery_status varchar(255);
+
+ALTER TABLE feature_sets
+ ADD column version int4 default 0;
+
+UPDATE feature_sets SET version = 1;
+
+
+--- FeatureStatistics Creation
+
+CREATE TABLE feature_statistics
+(
+ id integer NOT NULL,
+ average_length real NOT NULL,
+ avg_bytes real NOT NULL,
+ avg_num_values real NOT NULL,
+ count bigint NOT NULL,
+ dataset_id character varying(255),
+ date timestamp without time zone,
+ feature_type character varying(255),
+ max double precision NOT NULL,
+ max_bytes real NOT NULL,
+ max_num_values bigint NOT NULL,
+ mean double precision NOT NULL,
+ median double precision NOT NULL,
+ min double precision NOT NULL,
+ min_bytes real NOT NULL,
+ min_num_values bigint NOT NULL,
+ num_missing bigint NOT NULL,
+ num_values_histogram bytea,
+ numeric_value_histogram bytea,
+ numeric_value_quantiles bytea,
+ rank_histogram bytea,
+ stdev double precision NOT NULL,
+ top_values bytea,
+ total_num_values bigint NOT NULL,
+ n_unique bigint,
+ zeroes bigint NOT NULL,
+ feature_id bigint
+);
+
+
+ALTER TABLE ONLY feature_statistics
+ ADD CONSTRAINT feature_statistics_pkey PRIMARY KEY (id);
+
+CREATE INDEX idx_feature_statistics_dataset_id ON public.feature_statistics USING btree (dataset_id);
+
+CREATE INDEX idx_feature_statistics_date ON public.feature_statistics USING btree (date);
+
+CREATE INDEX idx_feature_statistics_feature ON public.feature_statistics USING btree (feature_id);
+
+ALTER TABLE ONLY feature_statistics
+ ADD CONSTRAINT feature_statistics_feature_fkey FOREIGN KEY (feature_id) REFERENCES public.features (id);
+
+
+--- Releasing previous PK in Source
+
+ALTER TABLE feature_sets
+ DROP CONSTRAINT fk2di8f74x6wir076hrfbyi1qfh;
+ALTER TABLE jobs
+ DROP CONSTRAINT fkhkfwvhc2gei0wqw5h4mfvsy9f;
+ALTER TABLE sources
+ DROP CONSTRAINT sources_pkey;
+ALTER TABLE sources
+ ALTER COLUMN id DROP NOT NULL;
+
+--- Migrating to auto-incremental Source primary key
+
+ALTER TABLE sources
+ ADD column pk SERIAL PRIMARY KEY;
+
+-- ALTER TABLE sources
+-- ALTER column type Type int4 USING ('{"KAFKA": 1}'::json ->> type)::INTEGER;
+ALTER TABLE sources
+ ADD column config varchar(255);
+
+
+--- Update all related to Source tables
+
+ALTER TABLE feature_sets
+ ADD COLUMN source_id int4;
+
+-- foreign key on source(id) -> source(pk)
+
+ALTER TABLE feature_sets
+ ADD CONSTRAINT feature_sets_to_sources_fkey
+ FOREIGN KEY (source_id) REFERENCES sources (pk);
+
+
+ALTER TABLE jobs
+ ADD COLUMN source_type varchar(255); -- Enum SourceType
+ALTER TABLE jobs
+ ADD COLUMN source_config varchar(255);
+
+
+
+--- Migrate Data
+--- creating sources with identical primary keys as in feature_sets
+
+SELECT feature_sets.id,
+ sources.bootstrap_servers,
+ sources.topics,
+ sources.is_default,
+ sources.type
+INTO TEMP TABLE converted_sources
+FROM feature_sets,
+ sources
+WHERE feature_sets.source = sources.id;
+
+DELETE FROM sources;
+
+INSERT INTO sources (pk, bootstrap_servers, topics, is_default, type)
+ (SELECT * FROM converted_sources);
+
+UPDATE feature_sets
+SET source_id = id;
diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index ae4a61f3e2e..78ae5df8279 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -35,6 +35,7 @@
## Administration
* [Troubleshooting](administration/troubleshooting.md)
+* [Feast Upgrading](administration/upgrading.md)
## Reference
diff --git a/docs/administration/upgrading.md b/docs/administration/upgrading.md
new file mode 100644
index 00000000000..71c6ada1212
--- /dev/null
+++ b/docs/administration/upgrading.md
@@ -0,0 +1,57 @@
+# Upgrading Feast
+
+
+## Migration 0.5 -> 0.6
+### Database schema
+In Release 0.6 we introduced [Flyway](https://flywaydb.org/) to handle schema migrations in PostgreSQL.
+Flyway is integrated into `core` and for now on all migrations will be run automatically on
+`core` start. It uses table `flyway_schema_history` in the same database (also created automatically) to keep track of already applied migrations.
+So no specific maintenance should be needed.
+
+If you already have existing deployment of feast 0.5 - Flyway will detect existing tables and omit first baseline migration.
+
+After `core` started you should have `flyway_schema_history` look
+like this
+ ```
+>> select version, description, script, checksum from flyway_schema_history
+
+version | description | script | checksum
+--------+-----------------------------------------+-----------------------------------------+------------
+ 1 | << Flyway Baseline >> | << Flyway Baseline >> |
+ 2 | RELEASE 0.6 Generalizing Source AND ... | V2__RELEASE_0.6_Generalizing_Source_... | 1537500232
+```
+
+In this release next major schema changes were done:
+
+* Source is not shared between FeatureSets anymore. It's changed to 1:1 relation
+and source's primary key is now auto-incremented number.
+* Due to generalization of Source `sources.topics` & `sources.bootstrap_servers` columns were deprecated.
+They will be replaced with `sources.config`. Data migration handled by code when respected Source is used.
+`topics` and `bootstrap_servers` will be deleted in the next release.
+* Job (table `jobs`) is no longer connected to `Source` (table `sources`) since it uses consolidated source for optimization purposes.
+All data required by Job would be embedded in its table.
+
+New Models (tables):
+* feature_statistics
+
+Minor changes:
+* FeatureSet has new column version (see [proto](https://github.com/feast-dev/feast/blob/master/protos/feast/core/FeatureSet.proto) for details)
+* Connecting table `jobs_feature_sets` in many-to-many relation between jobs & feature sets
+has now `version` and `delivery_status`.
+
+## Migration 0.4 -> 0.6
+
+### Database
+For all versions earlier than 0.5 seamless migration is not feasible due to earlier breaking changes and
+creation of new database will be required.
+
+Since database will be empty - first (baseline) migration would be applied:
+
+ ```
+>> select version, description, script, checksum from flyway_schema_history
+
+version | description | script | checksum
+--------+-----------------------------------------+-----------------------------------------+------------
+ 1 | Baseline | V1__Baseline.sql | 1091472110
+ 2 | RELEASE 0.6 Generalizing Source AND ... | V2__RELEASE_0.6_Generalizing_Source_... | 1537500232
+```
diff --git a/pom.xml b/pom.xml
index 38ff13cf71d..b37cd0a7b6d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,6 +61,7 @@
0.21.0
2.12.1
+ 5.2.4