import React, { useState } from "react"; import { EuiFormRow, EuiFieldText, EuiSelect, EuiSpacer, EuiHorizontalRule, EuiText, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTextArea, EuiTitle, } from "@elastic/eui"; import { feast } from "../protos"; import FormModal from "./forms/FormModal"; import TagsEditor, { TagEntry } from "./forms/TagsEditor"; import { DATA_SOURCE_TYPES } from "../pages/data-sources/DataSourceCatalog"; const SOURCE_TYPE_OPTIONS = [ { value: String(feast.core.DataSource.SourceType.BATCH_FILE), text: "File (Parquet / CSV)", }, { value: String(feast.core.DataSource.SourceType.BATCH_BIGQUERY), text: "BigQuery", }, { value: String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE), text: "Snowflake", }, { value: String(feast.core.DataSource.SourceType.BATCH_REDSHIFT), text: "Redshift", }, { value: String(feast.core.DataSource.SourceType.BATCH_SPARK), text: "Spark", }, { value: String(feast.core.DataSource.SourceType.BATCH_TRINO), text: "Trino", }, { value: String(feast.core.DataSource.SourceType.BATCH_ATHENA), text: "AWS Athena", }, { value: String(feast.core.DataSource.SourceType.STREAM_KAFKA), text: "Kafka", }, { value: String(feast.core.DataSource.SourceType.STREAM_KINESIS), text: "AWS Kinesis", }, { value: String(feast.core.DataSource.SourceType.REQUEST_SOURCE), text: "Request Source", }, { value: String(feast.core.DataSource.SourceType.PUSH_SOURCE), text: "Push Source", }, { value: String(feast.core.DataSource.SourceType.CUSTOM_SOURCE), text: "Custom Source", }, { value: "RAY_SOURCE", text: "Ray" }, { value: "POSTGRES_SOURCE", text: "PostgreSQL" }, { value: "MONGODB_SOURCE", text: "MongoDB" }, { value: "CLICKHOUSE_SOURCE", text: "ClickHouse" }, { value: "MSSQL_SOURCE", text: "SQL Server" }, { value: "ORACLE_SOURCE", text: "Oracle" }, { value: "COUCHBASE_SOURCE", text: "Couchbase" }, ]; interface DataSourceFormData { name: string; description: string; owner: string; sourceType: string; timestampField: string; createdTimestampColumn: string; tags: TagEntry[]; fileUri: string; bigqueryTable: string; bigqueryQuery: string; snowflakeTable: string; snowflakeDatabase: string; snowflakeSchema: string; redshiftTable: string; redshiftDatabase: string; redshiftSchema: string; kafkaBootstrapServers: string; kafkaTopic: string; sparkTable: string; sparkPath: string; kinesisRegion: string; kinesisStreamName: string; trinoTable: string; trinoQuery: string; athenaTable: string; athenaQuery: string; athenaDatabase: string; athenaDataSource: string; customSourceClassName: string; customSourceConfig: string; // Contrib source fields rayReaderType: string; rayPath: string; rayReaderOptions: string; postgresTable: string; postgresQuery: string; mongodbCollection: string; clickhouseTable: string; clickhouseQuery: string; mssqlTable: string; mssqlConnectionStr: string; oracleTable: string; oracleConnectionStr: string; couchbaseDatabase: string; couchbaseScope: string; couchbaseCollection: string; couchbaseQuery: string; } interface DataSourceFormModalProps { onClose: () => void; onSubmit: (data: DataSourceFormData) => void; initialData?: DataSourceFormData; isEdit?: boolean; isSubmitting?: boolean; submitError?: string | null; } const EMPTY_FORM: DataSourceFormData = { name: "", description: "", owner: "", sourceType: String(feast.core.DataSource.SourceType.BATCH_FILE), timestampField: "", createdTimestampColumn: "", tags: [], fileUri: "", bigqueryTable: "", bigqueryQuery: "", snowflakeTable: "", snowflakeDatabase: "", snowflakeSchema: "", redshiftTable: "", redshiftDatabase: "", redshiftSchema: "", kafkaBootstrapServers: "", kafkaTopic: "", sparkTable: "", sparkPath: "", kinesisRegion: "", kinesisStreamName: "", trinoTable: "", trinoQuery: "", athenaTable: "", athenaQuery: "", athenaDatabase: "", athenaDataSource: "", customSourceClassName: "", customSourceConfig: "", rayReaderType: "parquet", rayPath: "", rayReaderOptions: "", postgresTable: "", postgresQuery: "", mongodbCollection: "", clickhouseTable: "", clickhouseQuery: "", mssqlTable: "", mssqlConnectionStr: "", oracleTable: "", oracleConnectionStr: "", couchbaseDatabase: "", couchbaseScope: "", couchbaseCollection: "", couchbaseQuery: "", }; const BATCH_SOURCE_TYPES = new Set([ String(feast.core.DataSource.SourceType.BATCH_FILE), String(feast.core.DataSource.SourceType.BATCH_BIGQUERY), String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE), String(feast.core.DataSource.SourceType.BATCH_REDSHIFT), String(feast.core.DataSource.SourceType.BATCH_SPARK), String(feast.core.DataSource.SourceType.BATCH_TRINO), String(feast.core.DataSource.SourceType.BATCH_ATHENA), "RAY_SOURCE", "POSTGRES_SOURCE", "MONGODB_SOURCE", "CLICKHOUSE_SOURCE", "MSSQL_SOURCE", "ORACLE_SOURCE", "COUCHBASE_SOURCE", ]); const RAY_READER_OPTIONS = [ { value: "parquet", text: "Parquet" }, { value: "csv", text: "CSV" }, { value: "json", text: "JSON" }, { value: "text", text: "Text" }, { value: "images", text: "Images" }, { value: "binary_files", text: "Binary Files" }, { value: "tfrecords", text: "TFRecords" }, { value: "webdataset", text: "WebDataset" }, { value: "huggingface", text: "HuggingFace" }, { value: "mongo", text: "MongoDB (via Ray)" }, { value: "sql", text: "SQL (via Ray)" }, ]; const DataSourceFormModal: React.FC = ({ onClose, onSubmit, initialData, isEdit = false, isSubmitting = false, submitError, }) => { const [formData, setFormData] = useState( initialData || EMPTY_FORM, ); const [errors, setErrors] = useState>({}); const [submitted, setSubmitted] = useState(false); const isBatchSource = BATCH_SOURCE_TYPES.has(formData.sourceType); const isPreselected = !!initialData?.sourceType; const catalogEntry = DATA_SOURCE_TYPES.find( (ds) => ds.sourceType === formData.sourceType, ); const validate = (): boolean => { const newErrors: Record = {}; const st = formData.sourceType; if (!formData.name.trim()) { newErrors.name = "Data source name is required."; } else if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(formData.name)) { newErrors.name = "Must start with a letter or underscore, and contain only letters, numbers, and underscores."; } if (st === String(feast.core.DataSource.SourceType.BATCH_FILE)) { if (!formData.fileUri.trim()) { newErrors.fileUri = "File URI is required."; } else if ( !/^(s3|gs|gcs|hdfs|abfs|file):\/\/\S+$/.test(formData.fileUri.trim()) ) { newErrors.fileUri = "Must be a valid URI (e.g. s3://bucket/path, gs://bucket/path, file:///local/path)."; } } else if (st === String(feast.core.DataSource.SourceType.BATCH_BIGQUERY)) { if (!formData.bigqueryTable.trim() && !formData.bigqueryQuery.trim()) { newErrors.bigqueryTable = "Either a table reference or a query is required."; } } else if ( st === String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE) ) { if (!formData.snowflakeTable.trim()) { newErrors.snowflakeTable = "Table name is required for Snowflake."; } if (!formData.snowflakeDatabase.trim()) { newErrors.snowflakeDatabase = "Database is required for Snowflake."; } } else if (st === String(feast.core.DataSource.SourceType.BATCH_REDSHIFT)) { if (!formData.redshiftTable.trim()) { newErrors.redshiftTable = "Table name is required for Redshift."; } if (!formData.redshiftDatabase.trim()) { newErrors.redshiftDatabase = "Database is required for Redshift."; } } else if (st === String(feast.core.DataSource.SourceType.BATCH_SPARK)) { if (!formData.sparkTable.trim() && !formData.sparkPath.trim()) { newErrors.sparkTable = "Either a table reference or a path is required for Spark."; } } else if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { if (!formData.trinoTable.trim() && !formData.trinoQuery.trim()) { newErrors.trinoTable = "Either a table reference or a query is required for Trino."; } } else if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { if (!formData.athenaTable.trim() && !formData.athenaQuery.trim()) { newErrors.athenaTable = "Either a table reference or a query is required for Athena."; } if (!formData.athenaDatabase.trim()) { newErrors.athenaDatabase = "Database is required for Athena."; } } else if (st === String(feast.core.DataSource.SourceType.STREAM_KAFKA)) { if (!formData.kafkaBootstrapServers.trim()) { newErrors.kafkaBootstrapServers = "Bootstrap servers are required."; } else if ( !/^[\w.-]+:\d+(,[\w.-]+:\d+)*$/.test( formData.kafkaBootstrapServers.trim(), ) ) { newErrors.kafkaBootstrapServers = "Must be in host:port format (e.g. localhost:9092)."; } if (!formData.kafkaTopic.trim()) { newErrors.kafkaTopic = "Topic is required."; } } else if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { if (!formData.kinesisRegion.trim()) { newErrors.kinesisRegion = "AWS region is required."; } if (!formData.kinesisStreamName.trim()) { newErrors.kinesisStreamName = "Stream name is required."; } } else if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { if (!formData.customSourceClassName.trim()) { newErrors.customSourceClassName = "Class name is required."; } } else if (st === "RAY_SOURCE") { if ( !formData.rayPath.trim() && !["huggingface", "mongo", "sql"].includes(formData.rayReaderType) ) { newErrors.rayPath = "Path is required for this reader type."; } } else if (st === "POSTGRES_SOURCE") { if (!formData.postgresTable.trim() && !formData.postgresQuery.trim()) { newErrors.postgresTable = "Either a table or query is required."; } } else if (st === "CLICKHOUSE_SOURCE") { if ( !formData.clickhouseTable.trim() && !formData.clickhouseQuery.trim() ) { newErrors.clickhouseTable = "Either a table or query is required."; } } else if (st === "MSSQL_SOURCE") { if (!formData.mssqlTable.trim()) { newErrors.mssqlTable = "Table reference is required."; } } else if (st === "ORACLE_SOURCE") { if (!formData.oracleTable.trim()) { newErrors.oracleTable = "Table reference is required."; } } else if (st === "COUCHBASE_SOURCE") { if ( !formData.couchbaseCollection.trim() && !formData.couchbaseQuery.trim() ) { newErrors.couchbaseCollection = "Either a collection or query is required."; } } if (isBatchSource && !formData.timestampField.trim()) { newErrors.timestampField = "Timestamp field is required for batch sources."; } else if ( formData.timestampField.trim() && !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(formData.timestampField.trim()) ) { newErrors.timestampField = "Must be a valid column name (letters, numbers, underscores)."; } const tagKeys = formData.tags.map((t) => t.key).filter((k) => k.trim()); if (new Set(tagKeys).size !== tagKeys.length) { newErrors.tags = "Tag keys must be unique."; } setErrors(newErrors); return Object.keys(newErrors).length === 0; }; const handleSubmit = () => { setSubmitted(true); if (validate()) { const cleanedData = { ...formData, tags: formData.tags.filter((t) => t.key.trim()), }; onSubmit(cleanedData); } }; const updateField = ( field: K, value: DataSourceFormData[K], ) => { setFormData((prev) => ({ ...prev, [field]: value })); if (submitted) { setErrors((prev) => { const next = { ...prev }; delete next[field]; return next; }); } }; const renderFileSourceFields = () => ( updateField("fileUri", e.target.value)} isInvalid={!!errors.fileUri} placeholder="s3://bucket/path/to/data.parquet" /> ); const renderSourceTypeHeader = () => { if (!isPreselected || !catalogEntry) return null; const IconComponent = catalogEntry.icon; return (
{catalogEntry.name} {catalogEntry.description}
); }; const renderSourceTypeFields = () => { const st = formData.sourceType; if (st === String(feast.core.DataSource.SourceType.BATCH_FILE)) { return renderFileSourceFields(); } if (st === String(feast.core.DataSource.SourceType.BATCH_BIGQUERY)) { return ( <> updateField("bigqueryTable", e.target.value)} isInvalid={!!errors.bigqueryTable} placeholder="project:dataset.table" /> updateField("bigqueryQuery", e.target.value)} placeholder="SELECT * FROM `project.dataset.table` WHERE ..." rows={3} /> ); } if (st === String(feast.core.DataSource.SourceType.BATCH_SNOWFLAKE)) { return ( <> updateField("snowflakeDatabase", e.target.value) } isInvalid={!!errors.snowflakeDatabase} placeholder="MY_DATABASE" /> updateField("snowflakeSchema", e.target.value) } placeholder="PUBLIC" /> updateField("snowflakeTable", e.target.value)} isInvalid={!!errors.snowflakeTable} placeholder="MY_TABLE" /> ); } if (st === String(feast.core.DataSource.SourceType.BATCH_REDSHIFT)) { return ( <> updateField("redshiftDatabase", e.target.value) } isInvalid={!!errors.redshiftDatabase} placeholder="my_database" /> updateField("redshiftSchema", e.target.value) } placeholder="public" /> updateField("redshiftTable", e.target.value)} isInvalid={!!errors.redshiftTable} placeholder="my_table" /> ); } if (st === String(feast.core.DataSource.SourceType.STREAM_KAFKA)) { return ( <> updateField("kafkaBootstrapServers", e.target.value) } isInvalid={!!errors.kafkaBootstrapServers} placeholder="broker1:9092,broker2:9092" /> updateField("kafkaTopic", e.target.value)} isInvalid={!!errors.kafkaTopic} placeholder="my-feature-topic" /> ); } if (st === String(feast.core.DataSource.SourceType.BATCH_SPARK)) { return ( <> updateField("sparkTable", e.target.value)} isInvalid={!!errors.sparkTable} placeholder="catalog.database.table" /> updateField("sparkPath", e.target.value)} placeholder="s3://bucket/path/" /> ); } if (st === String(feast.core.DataSource.SourceType.BATCH_TRINO)) { return ( <> updateField("trinoTable", e.target.value)} isInvalid={!!errors.trinoTable} placeholder="catalog.schema.table" /> updateField("trinoQuery", e.target.value)} placeholder="SELECT * FROM catalog.schema.table" rows={3} /> ); } if (st === String(feast.core.DataSource.SourceType.BATCH_ATHENA)) { return ( <> updateField("athenaDatabase", e.target.value) } isInvalid={!!errors.athenaDatabase} placeholder="my_database" /> updateField("athenaDataSource", e.target.value) } placeholder="AwsDataCatalog" /> updateField("athenaTable", e.target.value)} isInvalid={!!errors.athenaTable} placeholder="my_table" /> updateField("athenaQuery", e.target.value)} placeholder="SELECT * FROM my_table" rows={3} /> ); } if (st === String(feast.core.DataSource.SourceType.STREAM_KINESIS)) { return ( <> updateField("kinesisRegion", e.target.value)} isInvalid={!!errors.kinesisRegion} placeholder="us-east-1" /> updateField("kinesisStreamName", e.target.value)} isInvalid={!!errors.kinesisStreamName} placeholder="my-feature-stream" /> ); } if (st === String(feast.core.DataSource.SourceType.CUSTOM_SOURCE)) { return ( <> updateField("customSourceClassName", e.target.value) } isInvalid={!!errors.customSourceClassName} placeholder="mymodule.MyCustomDataSource" /> updateField("customSourceConfig", e.target.value) } placeholder='{"key": "value"}' rows={3} /> ); } if (st === "RAY_SOURCE") { return ( <> updateField("rayReaderType", e.target.value)} /> updateField("rayPath", e.target.value)} isInvalid={!!errors.rayPath} placeholder="s3://bucket/images/" /> updateField("rayReaderOptions", e.target.value)} placeholder='{"dataset_name": "org/name", "split": "train"}' rows={3} /> ); } if (st === "POSTGRES_SOURCE") { return ( <> updateField("postgresTable", e.target.value)} isInvalid={!!errors.postgresTable} placeholder="public.my_features" /> updateField("postgresQuery", e.target.value)} placeholder="SELECT * FROM my_features WHERE ..." rows={3} /> ); } if (st === "MONGODB_SOURCE") { return ( updateField("mongodbCollection", e.target.value)} isInvalid={!!errors.mongodbCollection} placeholder="features_collection" /> ); } if (st === "CLICKHOUSE_SOURCE") { return ( <> updateField("clickhouseTable", e.target.value)} isInvalid={!!errors.clickhouseTable} placeholder="default.my_features" /> updateField("clickhouseQuery", e.target.value)} placeholder="SELECT * FROM default.my_features" rows={3} /> ); } if (st === "MSSQL_SOURCE") { return ( <> updateField("mssqlTable", e.target.value)} isInvalid={!!errors.mssqlTable} placeholder="dbo.my_features" /> updateField("mssqlConnectionStr", e.target.value) } placeholder="mssql+pyodbc://user:pass@host/db" // pragma: allowlist secret /> ); } if (st === "ORACLE_SOURCE") { return ( <> updateField("oracleTable", e.target.value)} isInvalid={!!errors.oracleTable} placeholder="SCHEMA.MY_FEATURES" /> updateField("oracleConnectionStr", e.target.value) } placeholder="oracle+cx_oracle://user:pass@host:1521/service" // pragma: allowlist secret /> ); } if (st === "COUCHBASE_SOURCE") { return ( <> updateField("couchbaseDatabase", e.target.value) } placeholder="Default" /> updateField("couchbaseScope", e.target.value) } placeholder="Default" /> updateField("couchbaseCollection", e.target.value) } isInvalid={!!errors.couchbaseCollection} placeholder="my_collection" /> updateField("couchbaseQuery", e.target.value)} placeholder="SELECT * FROM `collection`" rows={3} /> ); } if ( st === String(feast.core.DataSource.SourceType.REQUEST_SOURCE) || st === String(feast.core.DataSource.SourceType.PUSH_SOURCE) ) { return ( No connection configuration needed. This source type receives data at request time or via push ingestion. ); } return null; }; const sourceTypeName = SOURCE_TYPE_OPTIONS.find((o) => o.value === formData.sourceType)?.text || "Data Source"; return ( {submitError && ( <>

{submitError}

)} {isPreselected && renderSourceTypeHeader()} {isPreselected && } {/* Section: Identity */}

Identity

updateField("name", e.target.value)} isInvalid={!!errors.name} disabled={isEdit} placeholder="e.g. customer_transactions" /> updateField("owner", e.target.value)} placeholder="team@company.com" /> updateField("description", e.target.value)} placeholder="Brief description of this data source..." /> {!isPreselected && ( <> { updateField("sourceType", e.target.value); setErrors({}); }} disabled={isEdit} /> )} {/* Section: Connection */}

Connection Details

{renderSourceTypeFields()} {/* Section: Timestamp (for batch sources) */} {isBatchSource && ( <>

Time Configuration

updateField("timestampField", e.target.value) } isInvalid={!!errors.timestampField} placeholder="event_timestamp" /> updateField("createdTimestampColumn", e.target.value) } placeholder="created_at" /> )} {/* Section: Tags */}

Tags (optional)

updateField("tags", tags)} error={errors.tags} />
); }; export default DataSourceFormModal; export type { DataSourceFormData };