From c42db955567e2afcb6889fb7acc4b08c5eb48543 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Wed, 5 Aug 2026 12:34:26 +0530 Subject: [PATCH 1/3] feat: Enhancements to Feast OpenLineage UI Signed-off-by: ntkathole --- sdk/python/feast/feature_store.py | 50 +- .../feast/infra/compute_engines/base.py | 19 +- .../spark_application/compute.py | 46 +- .../compute_engines/spark_application/job.py | 19 + sdk/python/feast/openlineage/__init__.py | 17 + sdk/python/feast/openlineage/emitter.py | 162 ++++-- sdk/python/feast/openlineage/facets.py | 42 ++ sdk/python/feast/openlineage/identity.py | 140 ++++++ sdk/python/feast/openlineage/mappers.py | 125 ++++- sdk/python/feast/openlineage/processor.py | 99 +++- sdk/python/feast/openlineage/store.py | 41 +- .../compute_engines/test_spark_application.py | 25 + .../tests/unit/openlineage/test_identity.py | 53 ++ .../tests/unit/openlineage/test_processor.py | 43 +- .../tests/unit/openlineage/test_teardown.py | 79 +-- ui/src/components/LineageJobsList.tsx | 363 ++++++++++++++ ui/src/components/OpenLineageGraph.tsx | 461 +++++++++++++++--- ui/src/pages/lineage/Index.tsx | 7 +- ui/src/queries/useLoadOpenLineageGraph.ts | 28 ++ 19 files changed, 1586 insertions(+), 233 deletions(-) create mode 100644 sdk/python/feast/openlineage/identity.py create mode 100644 sdk/python/tests/unit/openlineage/test_identity.py create mode 100644 ui/src/components/LineageJobsList.tsx diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 5e6f520032a..3a80b504178 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -532,6 +532,7 @@ def _submit_and_process_materialization_jobs( regular_fvs: list, previous_states: dict, date_range: "_MaterializationDateRange", + openlineage_run_id: Optional[str] = None, ) -> None: """ Submit all tasks to the engine in one call and process the results. @@ -544,8 +545,22 @@ def _submit_and_process_materialization_jobs( ) batch_start = time.monotonic() + materialize_kwargs: Dict[str, Any] = {} + if openlineage_run_id and self.openlineage_emitter is not None: + from feast.openlineage.identity import ( + LineageParentContext, + materialize_job_name, + ) + + materialize_kwargs["lineage_parent"] = LineageParentContext( + job_namespace=self.openlineage_emitter.namespace_for(self.project), + job_name=materialize_job_name(self.project), + run_id=openlineage_run_id, + ) try: - jobs = provider.batch_engine.materialize(self.registry, tasks) + jobs = provider.batch_engine.materialize( + self.registry, tasks, **materialize_kwargs + ) except Exception: self._rollback_fv_states(regular_fvs, previous_states) raise @@ -602,6 +617,7 @@ def _materialize_fvs_batch( end_date: datetime, tqdm_builder, disable_event_timestamp: bool = False, + openlineage_run_id: Optional[str] = None, ) -> None: """Batch path: collect all FVs, submit to engine in one call. @@ -638,6 +654,7 @@ def _materialize_fvs_batch( regular_fvs, previous_states, date_range, + openlineage_run_id=openlineage_run_id, ) @property @@ -1915,21 +1932,9 @@ def teardown(self): def _teardown_openlineage(self): """Clean up OpenLineage data for this project's namespace during teardown.""" try: - if ( - hasattr(self.config, "openlineage") - and self.config.openlineage is not None - and self.config.openlineage.enabled - ): - ol_config = self.config.openlineage.to_openlineage_config() - consumer_cfg = getattr(ol_config, "consumer", None) - if consumer_cfg and getattr(consumer_cfg, "enabled", False): - conn_str = getattr(consumer_cfg, "connection_string", None) - if conn_str: - from feast.openlineage.store import OpenLineageStore - - ol_store = OpenLineageStore(connection_string=conn_str) - namespace = f"{self.project}/{self.project}" - ol_store.purge_namespace(namespace) + emitter = self.openlineage_emitter + if emitter is not None: + emitter.teardown_project(self.project) except Exception as e: warnings.warn(f"Failed to clean up OpenLineage data during teardown: {e}") @@ -2565,6 +2570,7 @@ def tqdm_builder(length): regular_fvs_with_dates, end_date_tz, tqdm_builder, + openlineage_run_id=ol_run_id, ) else: for feature_view, start_date in regular_fvs_with_dates: @@ -2757,6 +2763,7 @@ def tqdm_builder(length): end_date, tqdm_builder, disable_event_timestamp=disable_event_timestamp, + openlineage_run_id=ol_run_id, ) else: for feature_view, fv_start in regular_fvs_with_dates: @@ -2882,7 +2889,11 @@ def _emit_openlineage_materialize_start( return None try: run_id, success = self.openlineage_emitter.emit_materialize_start( - feature_views, start_date, end_date, self.project + feature_views, + start_date, + end_date, + self.project, + online_store=getattr(self.config, "online_store", None), ) # Return run_id only if START was successfully emitted # This prevents orphaned COMPLETE/FAIL events @@ -2901,7 +2912,10 @@ def _emit_openlineage_materialize_complete( return try: self.openlineage_emitter.emit_materialize_complete( - run_id, feature_views, self.project + run_id, + feature_views, + self.project, + online_store=getattr(self.config, "online_store", None), ) except Exception as e: warnings.warn(f"Failed to emit OpenLineage materialize complete event: {e}") diff --git a/sdk/python/feast/infra/compute_engines/base.py b/sdk/python/feast/infra/compute_engines/base.py index a99907a82b2..028bac44aac 100644 --- a/sdk/python/feast/infra/compute_engines/base.py +++ b/sdk/python/feast/infra/compute_engines/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import List, Sequence, Union +from typing import TYPE_CHECKING, List, Optional, Sequence, Union import pyarrow as pa @@ -19,6 +19,9 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.stream_feature_view import StreamFeatureView +if TYPE_CHECKING: + from feast.openlineage.identity import LineageParentContext + class ComputeEngine(ABC): """ @@ -104,10 +107,24 @@ def materialize( self, registry: BaseRegistry, tasks: Union[MaterializationTask, List[MaterializationTask]], + *, + lineage_parent: Optional["LineageParentContext"] = None, **kwargs, ) -> List[MaterializationJob]: + """Materialize features for the given tasks. + + Args: + registry: Feature registry + tasks: One or more materialization tasks + lineage_parent: Optional OpenLineage parent run. Engines that emit + their own lineage (e.g. SparkApplication) should attach this as + a parentRun; others ignore it. + **kwargs: Engine-specific options + """ if isinstance(tasks, MaterializationTask): tasks = [tasks] + if lineage_parent is not None: + kwargs["lineage_parent"] = lineage_parent return [self._materialize_one(registry, task, **kwargs) for task in tasks] def _materialize_one( diff --git a/sdk/python/feast/infra/compute_engines/spark_application/compute.py b/sdk/python/feast/infra/compute_engines/spark_application/compute.py index d5fbf7493ce..7dd3aeb866e 100644 --- a/sdk/python/feast/infra/compute_engines/spark_application/compute.py +++ b/sdk/python/feast/infra/compute_engines/spark_application/compute.py @@ -1,7 +1,7 @@ import logging import time import uuid -from typing import List, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union import pyarrow as pa import yaml @@ -173,6 +173,11 @@ def materialize( tasks = [tasks] job_id = uuid.uuid4().hex[:8] + from feast.openlineage.identity import coerce_lineage_parent + + lineage_parent = coerce_lineage_parent( + kwargs.get("lineage_parent") or kwargs.get("openlineage_parent") + ) try: self._create_with_retry( @@ -193,7 +198,7 @@ def materialize( return [job for _ in tasks] try: - cr = self._build_spark_application_cr(job_id) + cr = self._build_spark_application_cr(job_id, lineage_parent=lineage_parent) self._create_with_retry( lambda: self.custom_api.create_namespaced_custom_object( group="sparkoperator.k8s.io", @@ -223,6 +228,13 @@ def materialize( ) try: self._wait_for_completion(job) + # Freeze success before finally-cleanup deletes the CR; otherwise + # FeatureStore's later job.status() poll gets 404 → false FAIL. + if ( + job.error() is None + and job.status() == MaterializationJobStatus.SUCCEEDED + ): + job.mark_succeeded() return self._build_per_fv_jobs(registry, tasks, job_id, job) finally: self._cleanup(job_id) @@ -264,7 +276,14 @@ def _build_per_fv_jobs( SparkApplication does not pollute the status of succeeded FVs. """ if len(tasks) <= 1: - return [job for _ in tasks] + # Prefer a non-polling stub once the SparkApplication finished so + # post-cleanup status() checks cannot race with CR deletion. + if ( + job.error() is None + and job.status() == MaterializationJobStatus.SUCCEEDED + ): + return [CompletedMaterializationJob(job_id)] + return [job] jobs: List[MaterializationJob] = [] for task in tasks: @@ -320,7 +339,9 @@ def _create_configmap(self, job_id: str, tasks: List[MaterializationTask]): namespace=self.config.namespace, body=manifest ) - def _build_spark_application_cr(self, job_id: str) -> dict: + def _build_spark_application_cr( + self, job_id: str, lineage_parent: Optional[Any] = None + ) -> dict: driver_env_conf = { "spark.kubernetes.driverEnv.FEAST_CONFIGMAP_NAME": f"feast-sa-{job_id}", "spark.kubernetes.driverEnv.FEAST_CONFIGMAP_NAMESPACE": self.config.namespace, @@ -332,6 +353,22 @@ def _build_spark_application_cr(self, job_id: str) -> dict: entry["value"] ) + from feast.openlineage.identity import ( + coerce_lineage_parent, + spark_compute_job_name, + ) + + # K8s names stay unique per run; OL job identity is stable. + k8s_app_name = f"feast-sa-{job_id}" + project = getattr(self.repo_config, "project", None) or "feast" + parent_conf: Dict[str, str] = { + "spark.app.name": k8s_app_name, + "spark.openlineage.appName": spark_compute_job_name(project), + } + parent = coerce_lineage_parent(lineage_parent) + if parent: + parent_conf.update(parent.to_spark_openlineage_conf()) + spec = { "type": "Python", "mode": "cluster", @@ -343,6 +380,7 @@ def _build_spark_application_cr(self, job_id: str) -> dict: "sparkConf": { "spark.scheduler.mode": "FAIR", **(self.config.spark_conf or {}), + **parent_conf, **driver_env_conf, }, "restartPolicy": { diff --git a/sdk/python/feast/infra/compute_engines/spark_application/job.py b/sdk/python/feast/infra/compute_engines/spark_application/job.py index ef05585b061..73cfe8f2466 100644 --- a/sdk/python/feast/infra/compute_engines/spark_application/job.py +++ b/sdk/python/feast/infra/compute_engines/spark_application/job.py @@ -90,8 +90,16 @@ def __init__( self.namespace = namespace self.custom_api = custom_api self._error: Optional[BaseException] = error + # Once we observe SUCCEEDED/ERROR on the CR, keep it. Feast (and the + # Spark Operator TTL) delete the CR after success; a later status() + # poll must not turn that into a false "not found" failure. + self._terminal_status: Optional[MaterializationJobStatus] = None + if error is not None: + self._terminal_status = MaterializationJobStatus.ERROR def status(self) -> MaterializationJobStatus: + if self._terminal_status is not None: + return self._terminal_status if self._error is not None: return MaterializationJobStatus.ERROR @@ -112,8 +120,16 @@ def status(self) -> MaterializationJobStatus: .get("errorMessage", f"SparkApplication failed: {state}") ) self._error = Exception(msg) + self._terminal_status = MaterializationJobStatus.ERROR + elif result == MaterializationJobStatus.SUCCEEDED: + self._terminal_status = MaterializationJobStatus.SUCCEEDED return result + def mark_succeeded(self) -> None: + """Record successful completion before the CR is deleted/TTL'd.""" + self._error = None + self._terminal_status = MaterializationJobStatus.SUCCEEDED + def _get_cr_with_retry(self) -> Optional[dict]: """Fetch SparkApplication CR with exponential backoff on transient errors.""" last_exc = None @@ -128,6 +144,9 @@ def _get_cr_with_retry(self) -> Optional[dict]: ) except ApiException as e: if e.status == 404: + # Already completed jobs are cleaned up; don't invent a failure. + if self._terminal_status == MaterializationJobStatus.SUCCEEDED: + return None self._error = Exception( f"SparkApplication feast-sa-{self._job_id} not found" ) diff --git a/sdk/python/feast/openlineage/__init__.py b/sdk/python/feast/openlineage/__init__.py index b3c44849788..4edf71789a2 100644 --- a/sdk/python/feast/openlineage/__init__.py +++ b/sdk/python/feast/openlineage/__init__.py @@ -66,9 +66,18 @@ FeastEntityFacet, FeastFeatureServiceFacet, FeastFeatureViewFacet, + FeastJobKindFacet, FeastMaterializationFacet, + FeastOnlineStoreFacet, FeastProjectFacet, ) +from feast.openlineage.identity import ( + FeastJobKind, + LineageParentContext, + materialize_job_name, + resolve_namespace, + spark_compute_job_name, +) __all__ = [ # Main classes (used internally by native integration) @@ -76,11 +85,19 @@ "FeastOpenLineageEmitter", "OpenLineageConfig", "OpenLineageConsumerConfig", + # Identity / context + "FeastJobKind", + "LineageParentContext", + "materialize_job_name", + "resolve_namespace", + "spark_compute_job_name", # Facets (custom Feast metadata in lineage events) "FeastFeatureViewFacet", "FeastFeatureServiceFacet", "FeastDataSourceFacet", "FeastEntityFacet", + "FeastOnlineStoreFacet", "FeastMaterializationFacet", "FeastProjectFacet", + "FeastJobKindFacet", ] diff --git a/sdk/python/feast/openlineage/emitter.py b/sdk/python/feast/openlineage/emitter.py index b20168186c8..72cc1005a89 100644 --- a/sdk/python/feast/openlineage/emitter.py +++ b/sdk/python/feast/openlineage/emitter.py @@ -97,25 +97,44 @@ def namespace(self) -> str: """Get the default namespace.""" return self._config.namespace - def _get_namespace(self, project: str) -> str: - """ - Get the OpenLineage namespace for a project. + def namespace_for(self, project: str) -> str: + """Public OpenLineage namespace for a Feast project.""" + from feast.openlineage.identity import resolve_namespace - By default, uses the Feast project name as the namespace. - If an explicit namespace is configured (not the default "feast"), - it will be used as a prefix: {namespace}/{project} + return resolve_namespace(self._config.namespace, project) - Args: - project: Feast project name + def _get_namespace(self, project: str) -> str: + """Backward-compatible alias for :meth:`namespace_for`.""" + return self.namespace_for(project) + + def _job_kind_facets(self, kind: str, project: str) -> Dict[str, Any]: + from feast.openlineage.facets import FeastJobKindFacet + from feast.openlineage.identity import FeastJobKind + + kind_value = kind.value if isinstance(kind, FeastJobKind) else str(kind) + return { + "feast_jobKind": FeastJobKindFacet( + kind=kind_value, + feast_project=project, + ) + } - Returns: - OpenLineage namespace string + def teardown_project(self, project: str) -> None: + """Purge consumer-store lineage for this project's namespace. + + No-op when the OpenLineage consumer is disabled or has no connection. """ - # If namespace is default "feast", just use project name - if self._config.namespace == "feast": - return project - # If custom namespace is configured, use it as prefix - return f"{self._config.namespace}/{project}" + consumer_cfg = getattr(self._config, "consumer", None) + if not consumer_cfg or not getattr(consumer_cfg, "enabled", False): + return + conn_str = getattr(consumer_cfg, "connection_string", None) + if not conn_str: + return + from feast.openlineage.store import OpenLineageStore + + OpenLineageStore(connection_string=conn_str).purge_namespace( + self.namespace_for(project) + ) def emit_registry_lineage( self, @@ -474,6 +493,8 @@ def emit_materialize_start( end_date: datetime, project: str, run_id: Optional[str] = None, + online_store_type: str = "online", + online_store: Any = None, ) -> Tuple[str, bool]: """ Emit a START event for a materialization run. @@ -484,6 +505,10 @@ def emit_materialize_start( end_date: End of materialization window project: Project name run_id: Optional run ID (will be generated if not provided) + online_store_type: Backend type for the online store sink (redis, …) + online_store: Optional RepoConfig.online_store; when set, type is + resolved via :func:`resolve_online_store_type` and overrides + ``online_store_type``. Returns: Tuple of (run_id, success) @@ -492,21 +517,38 @@ def emit_materialize_start( return "", False from feast.openlineage.facets import FeastMaterializationFacet + from feast.openlineage.identity import ( + FeastJobKind, + materialize_job_name, + ) from feast.openlineage.mappers import ( data_source_to_dataset, + feature_view_to_dataset, online_store_to_dataset, + resolve_online_store_type, ) + if online_store is not None: + online_store_type = resolve_online_store_type(online_store) + run_id = run_id or str(uuid.uuid4()) try: - namespace = self._get_namespace(project) + namespace = self.namespace_for(project) # Build inputs (data sources) - include both batch and stream sources inputs = [] seen_sources = set() # Track source names to avoid duplicates for fv in feature_views: + # FeatureView as input — same dataset name as feast apply, with + # feast_featureView facet so mapping/metadata survive materialize. + if fv.name and fv.name not in seen_sources: + seen_sources.add(fv.name) + inputs.append( + feature_view_to_dataset(fv, namespace=namespace, as_input=True) + ) + # Add batch source if hasattr(fv, "batch_source") and fv.batch_source: source_name = getattr(fv.batch_source, "name", None) @@ -533,8 +575,10 @@ def emit_materialize_start( ) ) - # Add entities as inputs (use direct name for consistency with emit_apply) + # Add entities as inputs with feast_entity facet for mapping if hasattr(fv, "entities") and fv.entities: + from feast.openlineage.facets import FeastEntityFacet + for entity_name in fv.entities: if entity_name and entity_name != "__dummy": if entity_name not in seen_sources: @@ -543,13 +587,23 @@ def emit_materialize_start( InputDataset( namespace=namespace, name=entity_name, + facets={ + "feast_entity": FeastEntityFacet( + name=entity_name, + join_keys=[], + value_type="STRING", + description="", + owner="", + tags={}, + ) + }, ) ) - # Build outputs (online store entries) + # Build outputs — physical online-store sinks (not FeatureView nodes) outputs = [ online_store_to_dataset( - store_type="online_store", + store_type=online_store_type, feature_view_name=fv.name, namespace=namespace, ) @@ -563,15 +617,18 @@ def emit_materialize_start( start_date=start_date.isoformat() if start_date else None, end_date=end_date.isoformat() if end_date else None, project=project, + online_store_type=online_store_type or "", ) } + job_facets = self._job_kind_facets(FeastJobKind.TRANSFORM, project) success = self._client.emit_run_event( - job_name=f"materialize_{project}", + job_name=materialize_job_name(project), run_id=run_id, event_type=RunState.START, inputs=inputs, outputs=outputs, + job_facets=job_facets, run_facets=run_facets, namespace=namespace, ) @@ -587,6 +644,8 @@ def emit_materialize_complete( feature_views: List["FeatureView"], project: str, rows_written: Optional[int] = None, + online_store_type: str = "online", + online_store: Any = None, ) -> bool: """ Emit a COMPLETE event for a materialization run. @@ -596,6 +655,8 @@ def emit_materialize_complete( feature_views: Feature views that were materialized project: Project name rows_written: Optional count of rows written + online_store_type: Backend type for the online store sink (redis, …) + online_store: Optional RepoConfig.online_store; overrides type when set. Returns: True if successful, False otherwise @@ -604,14 +665,32 @@ def emit_materialize_complete( return False from feast.openlineage.facets import FeastMaterializationFacet - from feast.openlineage.mappers import online_store_to_dataset + from feast.openlineage.identity import ( + FeastJobKind, + materialize_job_name, + ) + from feast.openlineage.mappers import ( + feature_view_to_dataset, + online_store_to_dataset, + resolve_online_store_type, + ) + + if online_store is not None: + online_store_type = resolve_online_store_type(online_store) try: - namespace = self._get_namespace(project) + namespace = self.namespace_for(project) + + # Keep FeatureView as input on COMPLETE so edges stay consistent with START. + inputs = [ + feature_view_to_dataset(fv, namespace=namespace, as_input=True) + for fv in feature_views + if fv.name + ] outputs = [ online_store_to_dataset( - store_type="online_store", + store_type=online_store_type, feature_view_name=fv.name, namespace=namespace, ) @@ -623,14 +702,18 @@ def emit_materialize_complete( feature_views=[fv.name for fv in feature_views], project=project, rows_written=rows_written, + online_store_type=online_store_type or "", ) } + job_facets = self._job_kind_facets(FeastJobKind.TRANSFORM, project) return self._client.emit_run_event( - job_name=f"materialize_{project}", + job_name=materialize_job_name(project), run_id=run_id, event_type=RunState.COMPLETE, + inputs=inputs, outputs=outputs, + job_facets=job_facets, run_facets=run_facets, namespace=namespace, ) @@ -661,18 +744,25 @@ def emit_materialize_fail( try: from openlineage.client.facet_v2 import error_message_run - namespace = self._get_namespace(project) + from feast.openlineage.identity import ( + FeastJobKind, + materialize_job_name, + ) + + namespace = self.namespace_for(project) run_facets = {} if error_message: run_facets["errorMessage"] = error_message_run.ErrorMessageRunFacet( message=error_message, programmingLanguage="python", ) + job_facets = self._job_kind_facets(FeastJobKind.TRANSFORM, project) return self._client.emit_run_event( - job_name=f"materialize_{project}", + job_name=materialize_job_name(project), run_id=run_id, event_type=RunState.FAIL, + job_facets=job_facets, run_facets=run_facets, namespace=namespace, ) @@ -891,14 +981,20 @@ def emit_apply( ) # Emit Job 1: Feature Views job + from feast.openlineage.identity import ( + FeastJobKind, + feature_views_job_name, + ) + job_facets = { "feast_project": FeastProjectFacet( project_name=project, - ) + ), + **self._job_kind_facets(FeastJobKind.DEFINITION, project), } result1 = self._client.emit_run_event( - job_name=f"feast_feature_views_{project}", + job_name=feature_views_job_name(project), run_id=str(uuid.uuid4()), event_type=RunState.COMPLETE, inputs=fv_inputs, @@ -1007,14 +1103,20 @@ def emit_apply( ) # Emit a job for this specific FeatureService + from feast.openlineage.identity import ( + FeastJobKind, + feature_service_job_name, + ) + job_facets = { "feast_project": FeastProjectFacet( project_name=project, - ) + ), + **self._job_kind_facets(FeastJobKind.DEFINITION, project), } result = self._client.emit_run_event( - job_name=f"feature_service_{fs.name}", # Prefix to avoid conflict with dataset + job_name=feature_service_job_name(fs.name), run_id=str(uuid.uuid4()), event_type=RunState.COMPLETE, inputs=fs_inputs, diff --git a/sdk/python/feast/openlineage/facets.py b/sdk/python/feast/openlineage/facets.py index d350b74f0df..6720e253fd7 100644 --- a/sdk/python/feast/openlineage/facets.py +++ b/sdk/python/feast/openlineage/facets.py @@ -164,6 +164,30 @@ def _get_schema() -> str: return f"{FEAST_FACET_SCHEMA_BASE}/FeastDataSourceFacet.json" +@attr.define(kw_only=True) +class FeastOnlineStoreFacet(DatasetFacet): + """ + Custom facet for a Feast online-store sink dataset. + + Materialization writes FeatureView features into a physical online store + (Redis, DynamoDB, etc.). That sink is a separate lineage dataset from the + FeatureView definition itself. + + Attributes: + feature_view: Feature view whose features are stored here + store_type: Online store backend type (redis, sqlite, dynamodb, ...) + description: Human-readable description + """ + + feature_view: str = attr.field() + store_type: str = attr.field() + description: str = attr.field(default="") + + @staticmethod + def _get_schema() -> str: + return f"{FEAST_FACET_SCHEMA_BASE}/FeastOnlineStoreFacet.json" + + @attr.define(kw_only=True) class FeastEntityFacet(DatasetFacet): """ @@ -279,3 +303,21 @@ class FeastProjectFacet(JobFacet): @staticmethod def _get_schema() -> str: return f"{FEAST_FACET_SCHEMA_BASE}/FeastProjectFacet.json" + + +@attr.define(kw_only=True) +class FeastJobKindFacet(JobFacet): + """ + Distinguishes Feast OpenLineage jobs by semantic role. + + ``kind`` is ``definition`` (registry / apply topology) or ``transform`` + (runtime materialize / compute). Consumers and UIs should filter on this + facet rather than job-name heuristics. + """ + + kind: str = attr.field() # FeastJobKind value + feast_project: str = attr.field(default="") + + @staticmethod + def _get_schema() -> str: + return f"{FEAST_FACET_SCHEMA_BASE}/FeastJobKindFacet.json" diff --git a/sdk/python/feast/openlineage/identity.py b/sdk/python/feast/openlineage/identity.py new file mode 100644 index 00000000000..04246d641e6 --- /dev/null +++ b/sdk/python/feast/openlineage/identity.py @@ -0,0 +1,140 @@ +# Copyright 2026 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenLineage identity and lineage context for Feast. + +This module is the single source of truth for: +- Namespace resolution (Feast project ↔ OpenLineage namespace) +- Stable job names (materialize, spark compute, …) +- Parent-run context passed from FeatureStore to compute engines + +Compute engines that emit their own OpenLineage events (e.g. Spark) should +consume :class:`LineageParentContext` rather than ad-hoc kwargs/dicts. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Mapping, Optional, Union + + +class FeastJobKind(str, Enum): + """Semantic role of a Feast-emitted OpenLineage job. + + ``DEFINITION`` — registry topology (apply / feature service membership). + ``TRANSFORM`` — runtime execution (materialize, compute engines, …). + """ + + DEFINITION = "definition" + TRANSFORM = "transform" + + +def resolve_namespace( + configured_namespace: Optional[str], + project: str, +) -> str: + """Resolve the OpenLineage namespace for a Feast project. + + Rules (keep in sync with docs; do not duplicate elsewhere): + + - Empty / default ``\"feast\"`` → use the project name. + - Configured namespace equals the project (or already ends with + ``/{project}``) → use configured as-is (aligns with + ``spark.openlineage.namespace``). + - Otherwise → ``{configured}/{project}``. + """ + configured = (configured_namespace or "feast").strip() + if configured in ("", "feast"): + return project + if configured == project or configured.endswith(f"/{project}"): + return configured + return f"{configured}/{project}" + + +def materialize_job_name(project: str) -> str: + """Stable OpenLineage job name for Feast materialization.""" + return f"materialize_{project}" + + +def spark_compute_job_name(project: str) -> str: + """Stable OpenLineage job name for SparkApplication compute runs. + + Kubernetes SparkApplication / ConfigMap names remain unique per run + (operational). OpenLineage job identity must be stable so each + SparkApplication is a *run* of one job. + """ + return f"spark_compute_{project}" + + +def feature_views_job_name(project: str) -> str: + """OpenLineage job name for apply-time feature-view wiring.""" + return f"feast_feature_views_{project}" + + +def feature_service_job_name(feature_service_name: str) -> str: + """OpenLineage job name for FeatureService membership wiring.""" + return f"feature_service_{feature_service_name}" + + +@dataclass(frozen=True) +class LineageParentContext: + """Parent OpenLineage run that a compute-engine run should link to. + + Maps to the OpenLineage ``parentRun`` / Spark + ``spark.openlineage.parent*`` configuration. Engines that do not emit + OpenLineage events ignore this context. + """ + + job_namespace: str + job_name: str + run_id: str + + def to_spark_openlineage_conf(self) -> Dict[str, str]: + """Spark OpenLineage agent parent / root-parent configuration.""" + return { + "spark.openlineage.parentJobNamespace": self.job_namespace, + "spark.openlineage.parentJobName": self.job_name, + "spark.openlineage.parentRunId": self.run_id, + # Materialize is the root for Feast-driven Spark jobs today. + "spark.openlineage.rootParentJobNamespace": self.job_namespace, + "spark.openlineage.rootParentJobName": self.job_name, + "spark.openlineage.rootParentRunId": self.run_id, + } + + @classmethod + def from_mapping( + cls, data: Optional[Mapping[str, Any]] + ) -> Optional["LineageParentContext"]: + """Build from a mapping (e.g. legacy kwargs) or return None.""" + if not data: + return None + ns = data.get("jobNamespace") or data.get("job_namespace") + name = data.get("jobName") or data.get("job_name") + run_id = data.get("runId") or data.get("run_id") + if not (ns and name and run_id): + return None + return cls(job_namespace=str(ns), job_name=str(name), run_id=str(run_id)) + + +def coerce_lineage_parent( + value: Union[None, "LineageParentContext", Mapping[str, Any]], +) -> Optional[LineageParentContext]: + """Normalize lineage parent from typed context or legacy mapping.""" + if value is None: + return None + if isinstance(value, LineageParentContext): + return value + return LineageParentContext.from_mapping(value) diff --git a/sdk/python/feast/openlineage/mappers.py b/sdk/python/feast/openlineage/mappers.py index 9be9096aed2..38243b2f579 100644 --- a/sdk/python/feast/openlineage/mappers.py +++ b/sdk/python/feast/openlineage/mappers.py @@ -240,6 +240,9 @@ def feature_view_to_job( if feature_view.ttl: ttl_seconds = int(feature_view.ttl.total_seconds()) + from feast.openlineage.facets import FeastJobKindFacet + from feast.openlineage.identity import FeastJobKind + job_facets["feast_featureView"] = FeastFeatureViewFacet( name=feature_view.name, ttl_seconds=ttl_seconds, @@ -256,6 +259,10 @@ def feature_view_to_job( owner=feature_view.owner if feature_view.owner else "", tags=feature_view.tags if feature_view.tags else {}, ) + job_facets["feast_jobKind"] = FeastJobKindFacet( + kind=FeastJobKind.DEFINITION.value, + feast_project="", + ) # Add documentation if feature_view.description: @@ -344,7 +351,8 @@ def feature_service_to_job( """ _check_openlineage_available() - from feast.openlineage.facets import FeastFeatureServiceFacet + from feast.openlineage.facets import FeastFeatureServiceFacet, FeastJobKindFacet + from feast.openlineage.identity import FeastJobKind # Create job facets job_facets: Dict[str, Any] = {} @@ -377,6 +385,10 @@ def feature_service_to_job( tags=feature_service.tags if feature_service.tags else {}, logging_enabled=getattr(feature_service, "logging", None) is not None, ) + job_facets["feast_jobKind"] = FeastJobKindFacet( + kind=FeastJobKind.DEFINITION.value, + feast_project="", + ) # Add documentation if feature_service.description: @@ -429,6 +441,60 @@ def feature_service_to_job( return job, inputs, outputs +def feature_view_to_dataset( + feature_view: "Union[FeatureView, LabelView]", + namespace: str = "feast", + as_input: bool = True, +) -> Any: + """ + Convert a Feast FeatureView to an OpenLineage Dataset with Feast facets. + + Used for apply outputs and materialize inputs so FeatureView metadata is + not lost when jobs re-reference the same dataset name. + """ + _check_openlineage_available() + + from feast.openlineage.facets import FeastFeatureViewFacet + + facets: Dict[str, Any] = {} + + ttl_seconds = 0 + ttl = getattr(feature_view, "ttl", None) + if ttl is not None: + ttl_seconds = int(ttl.total_seconds()) + + facets["feast_featureView"] = FeastFeatureViewFacet( + name=feature_view.name, + ttl_seconds=ttl_seconds, + entities=list(feature_view.entities) if feature_view.entities else [], + features=[f.name for f in feature_view.features] + if feature_view.features + else [], + online_enabled=getattr(feature_view, "online", True), + offline_enabled=getattr(feature_view, "offline", False), + mode=str(feature_view.mode) + if hasattr(feature_view, "mode") and feature_view.mode + else "", + description=feature_view.description if feature_view.description else "", + owner=feature_view.owner if feature_view.owner else "", + tags=feature_view.tags if feature_view.tags else {}, + ) + + if feature_view.features: + facets["schema"] = schema_dataset.SchemaDatasetFacet( + fields=[feast_field_to_schema_field(f) for f in feature_view.features] + ) + + if feature_view.description: + facets["documentation"] = documentation_dataset.DocumentationDatasetFacet( + description=feature_view.description + ) + + if as_input: + return InputDataset(namespace=namespace, name=feature_view.name, facets=facets) + return OutputDataset(namespace=namespace, name=feature_view.name, facets=facets) + + def entity_to_dataset( entity: "Entity", namespace: str = "feast", @@ -486,32 +552,67 @@ def entity_to_dataset( ) +def resolve_online_store_type(online_store: Any = None) -> str: + """Resolve online store backend type for OpenLineage sink datasets. + + Accepts a RepoConfig ``online_store`` value (config object, dict, or type + string) and returns a short type label such as ``redis`` or ``online``. + """ + if online_store is None: + return "online" + if isinstance(online_store, str): + return online_store + if isinstance(online_store, dict): + return str(online_store.get("type") or "online") + return str(getattr(online_store, "type", None) or "online") + + def online_store_to_dataset( store_type: str, feature_view_name: str, namespace: str = "feast", ) -> "OutputDataset": """ - Create an OpenLineage OutputDataset for an online store. - - Args: - store_type: Type of online store (redis, sqlite, dynamodb, etc.) - feature_view_name: Name of the feature view being stored - namespace: OpenLineage namespace + Create an OpenLineage OutputDataset for an online store sink. - Returns: - OpenLineage OutputDataset object + This represents the *physical* online-store table/keyspace for a FeatureView, + not the FeatureView registry object itself. Dataset name stays + ``online_store_{feature_view}`` so lineage edges remain stable. """ _check_openlineage_available() + from feast.openlineage.facets import FeastOnlineStoreFacet + + # Callers historically passed store_type="online_store", which produced + # dataSource name "online_store_online_store". Normalize that. + store = (store_type or "online").strip() or "online" + if store in ("online_store", "online"): + store_label = "online" + else: + store_label = store + return OutputDataset( namespace=namespace, name=f"online_store_{feature_view_name}", facets={ "dataSource": datasource_dataset.DatasourceDatasetFacet( - name=f"{store_type}_online_store", - uri=f"{store_type}://feast/{feature_view_name}", - ) + name=store_label, + uri=f"{store_label}://feast/{feature_view_name}", + ), + "feast_onlineStore": FeastOnlineStoreFacet( + feature_view=feature_view_name, + store_type=store_label, + description=( + f"Online store sink for feature view '{feature_view_name}' " + f"({store_label})" + ), + ), + "documentation": documentation_dataset.DocumentationDatasetFacet( + description=( + f"Materialized online features for '{feature_view_name}' " + f"in {store_label}" + ) + ), }, ) diff --git a/sdk/python/feast/openlineage/processor.py b/sdk/python/feast/openlineage/processor.py index bbacf377b96..02e7227ab1d 100644 --- a/sdk/python/feast/openlineage/processor.py +++ b/sdk/python/feast/openlineage/processor.py @@ -104,15 +104,34 @@ def _process_run_event(self, event_id: str, event: Dict[str, Any]): run_facets = run.get("facets", {}) self._store.upsert_run(run_id, job_namespace, job_name, event_type, run_facets) + # Parent/child job link (e.g. Feast materialize → SparkApplication) + parent = run_facets.get("parent") or run_facets.get("parentRun") + if isinstance(parent, dict): + p_job = parent.get("job") or {} + p_ns = p_job.get("namespace") or "" + p_name = p_job.get("name") or "" + if p_ns and p_name and (p_ns != job_namespace or p_name != job_name): + self._store.upsert_lineage_edge( + source_type="job", + source_namespace=p_ns, + source_name=p_name, + target_type="job", + target_namespace=job_namespace, + target_name=job_name, + edge_type="parent", + ) + inputs = event.get("inputs", []) outputs = event.get("outputs", []) for inp in inputs: ds_namespace = inp.get("namespace", job_namespace) ds_name = inp.get("name", "") - ds_facets = inp.get("facets", {}) + ds_facets = inp.get("facets", {}) or {} - feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name) + feast_mapping = self._resolve_feast_mapping( + ds_namespace, ds_name, ds_facets + ) self._store.upsert_dataset( ds_namespace, ds_name, ds_facets, feast_mapping, producer=producer ) @@ -132,9 +151,11 @@ def _process_run_event(self, event_id: str, event: Dict[str, Any]): for out in outputs: ds_namespace = out.get("namespace", job_namespace) ds_name = out.get("name", "") - ds_facets = out.get("facets", {}) + ds_facets = out.get("facets", {}) or {} - feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name) + feast_mapping = self._resolve_feast_mapping( + ds_namespace, ds_name, ds_facets + ) self._store.upsert_dataset( ds_namespace, ds_name, ds_facets, feast_mapping, producer=producer ) @@ -162,7 +183,7 @@ def _process_dataset_event(self, event_id: str, event: Dict[str, Any]): self._store.store_event(event_id, event) - feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name) + feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name, ds_facets) self._store.upsert_dataset( ds_namespace, ds_name, ds_facets, feast_mapping, producer=producer ) @@ -184,9 +205,11 @@ def _process_job_event(self, event_id: str, event: Dict[str, Any]): for inp in inputs: ds_namespace = inp.get("namespace", job_namespace) ds_name = inp.get("name", "") - ds_facets = inp.get("facets", {}) + ds_facets = inp.get("facets", {}) or {} - feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name) + feast_mapping = self._resolve_feast_mapping( + ds_namespace, ds_name, ds_facets + ) self._store.upsert_dataset( ds_namespace, ds_name, ds_facets, feast_mapping, producer=producer ) @@ -205,9 +228,11 @@ def _process_job_event(self, event_id: str, event: Dict[str, Any]): for out in outputs: ds_namespace = out.get("namespace", job_namespace) ds_name = out.get("name", "") - ds_facets = out.get("facets", {}) + ds_facets = out.get("facets", {}) or {} - feast_mapping = self._resolve_feast_mapping(ds_namespace, ds_name) + feast_mapping = self._resolve_feast_mapping( + ds_namespace, ds_name, ds_facets + ) self._store.upsert_dataset( ds_namespace, ds_name, ds_facets, feast_mapping, producer=producer ) @@ -348,23 +373,55 @@ def _process_dataset_symlinks( ) def _resolve_feast_mapping( - self, namespace: str, dataset_name: str + self, + namespace: str, + dataset_name: str, + facets: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, str]]: """ Attempt to map an OpenLineage dataset to a Feast registry object. - Mapping rules: - 1. If the namespace matches a Feast project (directly or via namespace_mapping), - check if the dataset name matches a known Feast object naming pattern. - 2. Feast apply emits datasets with names like the FeatureView/FeatureService name. - 3. Feast materialize emits datasets named 'online_store_{fv_name}'. + Prefer Feast custom facets on the dataset (emitted on apply), then + fall back to known naming patterns (``online_store_*``, + ``request_source_*``). """ - feast_project = self._namespace_mapping.get(namespace, namespace) + facets = facets or {} + + # Namespace may be project or prefix/project; mapping keys are usually + # the logical OL namespace (e.g. customer_churn). + feast_project = self._namespace_mapping.get(namespace) + if feast_project is None and "/" in namespace: + for part in (namespace.split("/")[-1], namespace.split("/")[0]): + if part in self._namespace_mapping: + feast_project = self._namespace_mapping[part] + break + if feast_project is None: + feast_project = namespace.split("/")[-1] if "/" in namespace else namespace + + facet_type_map = ( + ("feast_onlineStore", "onlineStore"), + ("feast_featureView", "featureView"), + ("feast_featureService", "featureService"), + ("feast_entity", "entity"), + ("feast_dataSource", "dataSource"), + ) + for facet_key, obj_type in facet_type_map: + if facet_key in facets: + facet = facets.get(facet_key) or {} + name = None + if isinstance(facet, dict): + # Online store facet keys the related FV as feature_view + name = facet.get("name") or facet.get("feature_view") + return { + "type": obj_type, + "name": name or dataset_name, + "project": feast_project, + } if dataset_name.startswith("online_store_"): fv_name = dataset_name[len("online_store_") :] return { - "type": "featureView", + "type": "onlineStore", "name": fv_name, "project": feast_project, } @@ -376,8 +433,6 @@ def _resolve_feast_mapping( "project": feast_project, } - return { - "type": "unknown", - "name": dataset_name, - "project": feast_project, - } + # Bare name with no Feast facet — leave untyped so upsert does not + # clobber a previously resolved mapping with "unknown". + return None diff --git a/sdk/python/feast/openlineage/store.py b/sdk/python/feast/openlineage/store.py index 3ffc83b928e..8b9a7f86b0b 100644 --- a/sdk/python/feast/openlineage/store.py +++ b/sdk/python/feast/openlineage/store.py @@ -89,7 +89,12 @@ def upsert_job( now = int(time.time() * 1000) facets = job_data.get("facets", {}) job_type = None - if "jobType" in facets: + # Prefer Feast semantic kind over generic OL jobType processingType. + if "feast_jobKind" in facets: + kind = facets["feast_jobKind"] + if isinstance(kind, dict): + job_type = kind.get("kind") + if not job_type and "jobType" in facets: jt = facets["jobType"] job_type = jt.get("processingType", jt.get("integration")) @@ -171,20 +176,38 @@ def upsert_dataset( ) ).first() - values = { - "source_type": source_type, - "description": description, - "schema_json": schema_json, - "facets_json": json.dumps(facets) if facets else None, + values: Dict[str, Any] = { "updated_at": now, } if producer: values["producer"] = producer - if feast_obj_type: + + # Preserve richer metadata when a later event (e.g. materialize) + # re-touches the dataset without facets. + if facets: + values["facets_json"] = json.dumps(facets) + if source_type is not None: + values["source_type"] = source_type + if description is not None: + values["description"] = description + if schema_json is not None: + values["schema_json"] = schema_json + elif not existing: + values["facets_json"] = None + values["source_type"] = source_type + values["description"] = description + values["schema_json"] = schema_json + + if feast_obj_type and feast_obj_type != "unknown": + values["feast_object_type"] = feast_obj_type + if feast_obj_name: + values["feast_object_name"] = feast_obj_name + if feast_project: + values["feast_project"] = feast_project + elif not existing: + # First sighting with no resolvable Feast type values["feast_object_type"] = feast_obj_type - if feast_obj_name: values["feast_object_name"] = feast_obj_name - if feast_project: values["feast_project"] = feast_project if existing: diff --git a/sdk/python/tests/unit/infra/compute_engines/test_spark_application.py b/sdk/python/tests/unit/infra/compute_engines/test_spark_application.py index 3b080bd8993..a836ef39090 100644 --- a/sdk/python/tests/unit/infra/compute_engines/test_spark_application.py +++ b/sdk/python/tests/unit/infra/compute_engines/test_spark_application.py @@ -25,6 +25,7 @@ def _make_repo_config( ): """Build a mock RepoConfig for testing.""" config = MagicMock() + config.project = "test" config.online_store = MagicMock() config.online_store.type = online_store_type config.offline_store = MagicMock() @@ -207,6 +208,30 @@ def test_cr_driver_env_passthrough(): ) +def test_cr_openlineage_job_name_is_stable(): + """Each SparkApplication CR is unique, but OL job name is per-project.""" + from feast.openlineage.identity import LineageParentContext + + engine = _make_engine() + parent = LineageParentContext( + job_namespace="test", + job_name="materialize_test", + run_id="run-123", + ) + cr_a = engine._build_spark_application_cr("abcd1234", lineage_parent=parent) + cr_b = engine._build_spark_application_cr("efgh5678", lineage_parent=parent) + conf_a = cr_a["spec"]["sparkConf"] + conf_b = cr_b["spec"]["sparkConf"] + assert cr_a["metadata"]["name"] == "feast-sa-abcd1234" + assert cr_b["metadata"]["name"] == "feast-sa-efgh5678" + assert conf_a["spark.app.name"] == "feast-sa-abcd1234" + assert conf_b["spark.app.name"] == "feast-sa-efgh5678" + assert conf_a["spark.openlineage.appName"] == "spark_compute_test" + assert conf_b["spark.openlineage.appName"] == "spark_compute_test" + assert conf_a["spark.openlineage.parentJobName"] == "materialize_test" + assert conf_a["spark.openlineage.parentRunId"] == "run-123" + + # ── Test 9: Status mapping covers all 14 states ── diff --git a/sdk/python/tests/unit/openlineage/test_identity.py b/sdk/python/tests/unit/openlineage/test_identity.py new file mode 100644 index 00000000000..6f71890873b --- /dev/null +++ b/sdk/python/tests/unit/openlineage/test_identity.py @@ -0,0 +1,53 @@ +"""Tests for feast.openlineage.identity.""" + +from feast.openlineage.identity import ( + FeastJobKind, + LineageParentContext, + materialize_job_name, + resolve_namespace, + spark_compute_job_name, +) + + +class TestResolveNamespace: + def test_default_feast_uses_project(self): + assert resolve_namespace("feast", "my_project") == "my_project" + assert resolve_namespace("", "my_project") == "my_project" + assert resolve_namespace(None, "my_project") == "my_project" + + def test_matching_namespace_not_doubled(self): + assert resolve_namespace("customer_churn", "customer_churn") == "customer_churn" + + def test_prefix_when_distinct(self): + assert resolve_namespace("org", "proj") == "org/proj" + + def test_already_suffixed(self): + assert resolve_namespace("org/proj", "proj") == "org/proj" + + +class TestJobNames: + def test_materialize(self): + assert materialize_job_name("p") == "materialize_p" + + def test_spark_compute(self): + assert spark_compute_job_name("p") == "spark_compute_p" + + +class TestLineageParentContext: + def test_to_spark_conf(self): + ctx = LineageParentContext("ns", "materialize_p", "run-1") + conf = ctx.to_spark_openlineage_conf() + assert conf["spark.openlineage.parentJobNamespace"] == "ns" + assert conf["spark.openlineage.parentJobName"] == "materialize_p" + assert conf["spark.openlineage.parentRunId"] == "run-1" + assert conf["spark.openlineage.rootParentRunId"] == "run-1" + + def test_from_mapping(self): + ctx = LineageParentContext.from_mapping( + {"jobNamespace": "ns", "jobName": "j", "runId": "r"} + ) + assert ctx == LineageParentContext("ns", "j", "r") + + def test_job_kind_values(self): + assert FeastJobKind.DEFINITION.value == "definition" + assert FeastJobKind.TRANSFORM.value == "transform" diff --git a/sdk/python/tests/unit/openlineage/test_processor.py b/sdk/python/tests/unit/openlineage/test_processor.py index e5ed9388132..c7e3e1a0ac3 100644 --- a/sdk/python/tests/unit/openlineage/test_processor.py +++ b/sdk/python/tests/unit/openlineage/test_processor.py @@ -509,7 +509,7 @@ def test_online_store_prefix_mapped(self, processor, store): ) datasets = store.get_datasets() ds = [d for d in datasets if d["dataset_name"] == "online_store_driver_fv"][0] - assert ds["feast_object_type"] == "featureView" + assert ds["feast_object_type"] == "onlineStore" assert ds["feast_object_name"] == "driver_fv" def test_request_source_prefix_mapped(self, processor, store): @@ -538,7 +538,46 @@ def test_default_mapping(self, processor, store): ) datasets = store.get_datasets() ds = [d for d in datasets if d["dataset_name"] == "regular_dataset"][0] - assert ds["feast_object_type"] == "unknown" + # Unresolvable datasets stay untyped rather than "unknown" + assert ds["feast_object_type"] in (None, "unknown") + + def test_facet_mapping_feature_view(self, processor, store): + processor.process_event( + _run_event( + inputs=[ + { + "namespace": "test-ns", + "name": "driver_hourly_stats", + "facets": { + "feast_featureView": {"name": "driver_hourly_stats"} + }, + } + ], + ) + ) + ds = [ + d + for d in store.get_datasets() + if d["dataset_name"] == "driver_hourly_stats" + ][0] + assert ds["feast_object_type"] == "featureView" + assert ds["feast_object_name"] == "driver_hourly_stats" + + def test_facet_mapping_entity(self, processor, store): + processor.process_event( + _run_event( + inputs=[ + { + "namespace": "test-ns", + "name": "driver", + "facets": {"feast_entity": {"name": "driver"}}, + } + ], + ) + ) + ds = [d for d in store.get_datasets() if d["dataset_name"] == "driver"][0] + assert ds["feast_object_type"] == "entity" + assert ds["feast_object_name"] == "driver" def test_namespace_mapping_applied(self, processor_with_mapping, store): processor_with_mapping.process_event( diff --git a/sdk/python/tests/unit/openlineage/test_teardown.py b/sdk/python/tests/unit/openlineage/test_teardown.py index 2f754c230b5..c932ecfd0bf 100644 --- a/sdk/python/tests/unit/openlineage/test_teardown.py +++ b/sdk/python/tests/unit/openlineage/test_teardown.py @@ -1,84 +1,45 @@ """Tests for FeatureStore._teardown_openlineage().""" from contextvars import ContextVar -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest class TestTeardownOpenlineage: - def _make_feature_store( - self, ol_enabled=True, consumer_enabled=True, conn_str="sqlite:///test.db" - ): - """Create a mock FeatureStore with configurable OL settings.""" + def _make_feature_store(self, emitter=None): + """Create a mock FeatureStore with a controllable OL emitter.""" from feast.feature_store import FeatureStore fs = object.__new__(FeatureStore) fs._current_project = ContextVar("current_project", default=None) - - mock_ol_config = MagicMock() - mock_ol_config.enabled = ol_enabled - - mock_consumer = MagicMock() - mock_consumer.enabled = consumer_enabled - mock_consumer.connection_string = conn_str - - mock_full_config = MagicMock() - mock_full_config.consumer = mock_consumer - - mock_ol_config.to_openlineage_config.return_value = mock_full_config + fs._openlineage_emitter = emitter mock_config = MagicMock() mock_config.project = "test_project" - mock_config.openlineage = mock_ol_config - + mock_config.openlineage = MagicMock(enabled=True) fs.config = mock_config - return fs - def test_calls_purge_namespace(self): - """When OL consumer is configured, teardown should purge the project namespace.""" - fs = self._make_feature_store() - - with patch("feast.openlineage.store.OpenLineageStore") as mock_store_cls: - mock_instance = mock_store_cls.return_value - fs._teardown_openlineage() - mock_store_cls.assert_called_once_with( - connection_string="sqlite:///test.db" - ) - mock_instance.purge_namespace.assert_called_once_with( - "test_project/test_project" - ) - - def test_no_crash_when_ol_not_configured(self): - """When OL is not configured, teardown should silently succeed.""" - from feast.feature_store import FeatureStore - - fs = object.__new__(FeatureStore) - fs._current_project = ContextVar("current_project", default=None) - mock_config = MagicMock() - mock_config.openlineage = None - fs.config = mock_config - - fs._teardown_openlineage() - - def test_no_crash_when_consumer_disabled(self): - """When consumer is disabled, teardown should not attempt purge.""" - fs = self._make_feature_store(consumer_enabled=False) + def test_calls_emitter_teardown_project(self): + """Teardown delegates purge to the emitter.""" + emitter = MagicMock() + fs = self._make_feature_store(emitter=emitter) fs._teardown_openlineage() + emitter.teardown_project.assert_called_once_with("test_project") - def test_no_crash_when_ol_disabled(self): - """When OL is disabled entirely, teardown should not attempt purge.""" - fs = self._make_feature_store(ol_enabled=False) + def test_no_crash_when_emitter_missing(self): + """When OL emitter is absent, teardown should silently succeed.""" + fs = self._make_feature_store(emitter=None) + # Force property path: no emitter init + fs._init_openlineage_emitter = MagicMock(return_value=None) # type: ignore fs._teardown_openlineage() def test_exception_is_warning_not_error(self): """If purge fails, it should warn, not raise.""" - fs = self._make_feature_store() + emitter = MagicMock() + emitter.teardown_project.side_effect = Exception("DB error") + fs = self._make_feature_store(emitter=emitter) - with patch( - "feast.openlineage.store.OpenLineageStore", - side_effect=Exception("DB error"), - ): - with pytest.warns(match="Failed to clean up OpenLineage"): - fs._teardown_openlineage() + with pytest.warns(match="Failed to clean up OpenLineage"): + fs._teardown_openlineage() diff --git a/ui/src/components/LineageJobsList.tsx b/ui/src/components/LineageJobsList.tsx new file mode 100644 index 00000000000..e420e11c08b --- /dev/null +++ b/ui/src/components/LineageJobsList.tsx @@ -0,0 +1,363 @@ +import React, { useMemo, useState } from "react"; +import { + EuiPanel, + EuiTitle, + EuiSpacer, + EuiBasicTable, + EuiLoadingSpinner, + EuiEmptyPrompt, + EuiBadge, + EuiFieldSearch, + EuiFlexGroup, + EuiFlexItem, + EuiFlyout, + EuiFlyoutHeader, + EuiFlyoutBody, + EuiDescriptionList, + EuiHorizontalRule, +} from "@elastic/eui"; +import type { OpenLineageJob } from "../queries/useLoadOpenLineageGraph"; +import { useLoadOpenLineageJobs } from "../queries/useLoadOpenLineageGraph"; +import { useRunDetail, useRunHistory } from "../queries/useLoadRunHistory"; +import type { RunSummary } from "../queries/useLoadRunHistory"; + +const formatTimestamp = (ts: number | null | undefined) => { + if (!ts) return "—"; + return new Date(ts).toLocaleString(); +}; + +const formatDuration = (start: number | null, end: number | null): string => { + if (!start || !end) return "—"; + const ms = end - start; + if (ms < 1000) return `${ms}ms`; + const s = Math.floor(ms / 1000); + if (s < 60) return `${s}s`; + const m = Math.floor(s / 60); + return `${m}m ${s % 60}s`; +}; + +const stateColor = (state: string) => { + switch ((state || "").toUpperCase()) { + case "COMPLETE": + return "success"; + case "FAIL": + return "danger"; + case "RUNNING": + case "START": + return "primary"; + case "ABORT": + return "warning"; + default: + return "default"; + } +}; + +const JobRunHistory: React.FC<{ job: OpenLineageJob }> = ({ job }) => { + const { data, isLoading } = useRunHistory(job.job_namespace, job.job_name); + const [selectedRunId, setSelectedRunId] = useState(); + const { data: runDetail, isLoading: detailLoading } = + useRunDetail(selectedRunId); + + const runs = data?.runs || []; + + if (isLoading) { + return ; + } + + if (runs.length === 0) { + return

No runs recorded for this job yet.

; + } + + return ( + <> + +

Run History ({runs.length})

+
+ + ({ + onClick: () => setSelectedRunId(run.run_id), + style: { + cursor: "pointer", + background: + selectedRunId === run.run_id + ? "rgba(0, 119, 204, 0.08)" + : undefined, + }, + })} + columns={[ + { + field: "run_id", + name: "Run", + truncateText: true, + render: (id: string) => ( + + {id.slice(0, 8)}… + + ), + }, + { + field: "state", + name: "Status", + width: "120px", + render: (state: string) => ( + {state} + ), + }, + { + field: "started_at", + name: "Started", + render: (ts: number | null) => formatTimestamp(ts), + }, + { + name: "Duration", + render: (run: RunSummary) => + formatDuration(run.started_at, run.ended_at), + }, + ]} + /> + {selectedRunId && ( + <> + + +

Run Detail

+
+ + {detailLoading && } + {!detailLoading && runDetail && ( + <> + + {runDetail.state} + + ), + }, + { + title: "Started", + description: formatTimestamp(runDetail.started_at), + }, + { + title: "Ended", + description: formatTimestamp(runDetail.ended_at), + }, + ]} + /> + +
+ Inputs + {(runDetail.inputs || []).length === 0 ? ( +

None

+ ) : ( +
    + {runDetail.inputs.map((io) => ( +
  • + {io.name} +
  • + ))} +
+ )} + Outputs + {(runDetail.outputs || []).length === 0 ? ( +

None

+ ) : ( +
    + {runDetail.outputs.map((io) => ( +
  • + {io.name} +
  • + ))} +
+ )} +
+ + )} + + )} + + ); +}; + +const LineageJobsList: React.FC = () => { + const [search, setSearch] = useState(""); + const [selectedJob, setSelectedJob] = useState(null); + const { data, isLoading, isError } = useLoadOpenLineageJobs(); + + const jobs = useMemo(() => { + const all = (data?.jobs || []).filter( + // Spark OL often emits a bootstrap START with job name "unknown" + // before spark.app.name is resolved — hide those stub jobs. + (j) => (j.job_name || "").toLowerCase() !== "unknown", + ); + const q = search.trim().toLowerCase(); + if (!q) return all; + return all.filter( + (j) => + j.job_name.toLowerCase().includes(q) || + j.job_namespace.toLowerCase().includes(q) || + (j.producer || "").toLowerCase().includes(q) || + (j.job_type || "").toLowerCase().includes(q), + ); + }, [data, search]); + + if (isLoading) { + return ( + +
+ +
+
+ ); + } + + if (isError) { + return ( + + Failed to load jobs} + body={

Could not fetch OpenLineage jobs from the consumer.

} + /> +
+ ); + } + + if (!data?.jobs?.length) { + return ( + + No Jobs Yet} + body={ +

+ Jobs appear here when OpenLineage producers (Feast materialize, + Spark, Airflow, etc.) emit events to this consumer. +

+ } + /> +
+ ); + } + + return ( + <> + + + + +

Jobs ({jobs.length})

+
+

+ Catalog of all OpenLineage jobs. Runtime transforms (materialize, + Spark) also appear on the Lineage graph; apply/definition jobs + (e.g. feature_service_*) are listed here only. +

+
+ + setSearch(e.target.value)} + isClearable + aria-label="Search jobs" + /> + +
+ + ({ + onClick: () => setSelectedJob(job), + style: { cursor: "pointer" }, + })} + columns={[ + { + field: "job_name", + name: "Job", + truncateText: true, + }, + { + field: "job_namespace", + name: "Namespace", + truncateText: true, + width: "220px", + }, + { + field: "job_type", + name: "Type", + width: "120px", + render: (val?: string | null) => val || "—", + }, + { + field: "producer", + name: "Producer", + width: "140px", + render: (val?: string | null) => + val ? {val} : "—", + }, + { + field: "updated_at", + name: "Last Seen", + width: "200px", + render: (ts: number) => formatTimestamp(ts), + }, + ]} + /> +
+ + {selectedJob && ( + setSelectedJob(null)} + aria-labelledby="job-flyout-title" + > + + +

{selectedJob.job_name}

+
+
+ + + {selectedJob.description && ( + <> + +

{selectedJob.description}

+ + )} + + +
+
+ )} + + ); +}; + +export default LineageJobsList; diff --git a/ui/src/components/OpenLineageGraph.tsx b/ui/src/components/OpenLineageGraph.tsx index a95c6c20b08..2636e3e66ad 100644 --- a/ui/src/components/OpenLineageGraph.tsx +++ b/ui/src/components/OpenLineageGraph.tsx @@ -41,24 +41,25 @@ const nodeHeight = 65; // ── Producer-based colors (generated dynamically) ── -const normalizeProducer = (producer?: string | null): string => { +/** + * Label for producer badges / filters. + * User-configured names are shown as-is. Long OpenLineage producer URLs + * (e.g. …/integration/spark) are shortened to the last path segment so the + * UI stays readable. + */ +const displayProducer = (producer?: string | null): string => { if (!producer) return "unknown"; - const p = producer.toLowerCase().trim(); - - // Extract the last meaningful path segment from URLs like - // "https://github.com/OpenLineage/OpenLineage/tree/1.0.0/integration/airflow" + const trimmed = producer.trim(); try { - const url = new URL(p); + const url = new URL(trimmed); const segments = url.pathname.split("/").filter(Boolean); if (segments.length > 0) { - return segments[segments.length - 1].replace(/-/g, "_"); + return segments[segments.length - 1]; } } catch { - // not a URL + // not a URL — keep configured value unchanged } - - // For plain names like "feast" or "my-custom-producer", just clean up - return p.replace(/-/g, "_").replace(/\s+/g, "_"); + return trimmed; }; const hashString = (str: string): number => { @@ -82,7 +83,7 @@ const generateProducerColor = ( const producerColorCache: Record = {}; const getProducerColors = (producer?: string | null) => { - const key = normalizeProducer(producer); + const key = displayProducer(producer); if (!producerColorCache[key]) { producerColorCache[key] = generateProducerColor(key); } @@ -93,6 +94,179 @@ const getNodeIcon = (type: string) => { return type === "job" ? "\u2699" : "\u2B21"; }; +const nodeKey = (type: string, ns: string, name: string) => + `${type}:${ns}:${name}`; + +/** + * Runtime transform jobs appear as pills on the Lineage graph. + * Prefer server-provided job_type (from feast_jobKind facet). Fall back to + * known Feast emit conventions for events emitted before jobKind existed. + */ +const isRuntimeTransformJob = ( + name: string, + producer?: string | null, + jobType?: string | null, +): boolean => { + const kind = (jobType || "").toLowerCase(); + if (kind === "transform") return true; + if (kind === "definition") return false; + + const n = (name || "").toLowerCase(); + if ( + n.startsWith("feature_service_") || + n.startsWith("feast_feature_views_") || + n.startsWith("feast_feature_services_") + ) { + return false; + } + if (n.startsWith("materialize_")) return true; + if (n.startsWith("spark_compute_")) return true; + if (n.startsWith("stream_")) return true; + if (n.startsWith("on_demand_feature_view_")) return true; + const p = (producer || "").toLowerCase(); + if (p.includes("/integration/spark")) return true; + return false; +}; + +/** + * Lineage graph: datasets + runtime transform jobs (materialize, Spark, …). + * Definition relationships (source→FV, FV→FeatureService) stay as derived + * dataset edges. Derived shortcuts are omitted only when a *transform* job + * already connects the same pair. + */ +const buildLineageGraphSlice = ( + olData: OpenLineageGraphData, +): { nodes: OpenLineageNode[]; edges: OpenLineageGraphData["edges"] } => { + const makeId = (type: string, ns: string, name: string) => + nodeKey(type, ns, name); + + const jobById = new Map( + olData.nodes + .filter((n) => n.type === "job") + .map((n) => [makeId(n.type, n.namespace, n.name), n]), + ); + + const ioEdges = olData.edges.filter( + (e) => e.edge_type === "input" || e.edge_type === "output", + ); + const parentEdges = olData.edges.filter((e) => e.edge_type === "parent"); + const symlinkEdges = olData.edges.filter((e) => e.edge_type === "symlink"); + const derivedEdges = olData.edges.filter((e) => e.edge_type === "derived"); + + const isTransformJobId = (id: string) => { + const job = jobById.get(id); + if (!job) return false; + return isRuntimeTransformJob(job.name, job.producer, job.job_type); + }; + + const jobsOnPath = new Set(); + for (const e of ioEdges) { + if (e.source_type === "job") { + const id = makeId(e.source_type, e.source_namespace, e.source_name); + if (isTransformJobId(id)) jobsOnPath.add(id); + } + if (e.target_type === "job") { + const id = makeId(e.target_type, e.target_namespace, e.target_name); + if (isTransformJobId(id)) jobsOnPath.add(id); + } + } + + // Pull in parent/child compute jobs (e.g. materialize → Spark) + let grew = true; + while (grew) { + grew = false; + for (const e of parentEdges) { + const src = makeId(e.source_type, e.source_namespace, e.source_name); + const tgt = makeId(e.target_type, e.target_namespace, e.target_name); + if ( + jobsOnPath.has(src) && + !jobsOnPath.has(tgt) && + isTransformJobId(tgt) + ) { + jobsOnPath.add(tgt); + grew = true; + } + if ( + jobsOnPath.has(tgt) && + !jobsOnPath.has(src) && + isTransformJobId(src) + ) { + jobsOnPath.add(src); + grew = true; + } + } + } + + const nodes = olData.nodes.filter( + (n) => + n.type === "dataset" || + (n.type === "job" && jobsOnPath.has(makeId(n.type, n.namespace, n.name))), + ); + const nodeIds = new Set( + nodes.map((n) => makeId(n.type, n.namespace, n.name)), + ); + + // Only transform-job I/O suppresses derived dataset shortcuts + const mediatedPairs = new Set(); + const jobInputs = new Map>(); + const jobOutputs = new Map>(); + for (const e of ioEdges) { + if (e.edge_type === "input" && e.target_type === "job") { + const jid = makeId(e.target_type, e.target_namespace, e.target_name); + if (!jobsOnPath.has(jid)) continue; + const did = makeId(e.source_type, e.source_namespace, e.source_name); + if (!jobInputs.has(jid)) jobInputs.set(jid, new Set()); + jobInputs.get(jid)!.add(did); + } + if (e.edge_type === "output" && e.source_type === "job") { + const jid = makeId(e.source_type, e.source_namespace, e.source_name); + if (!jobsOnPath.has(jid)) continue; + const did = makeId(e.target_type, e.target_namespace, e.target_name); + if (!jobOutputs.has(jid)) jobOutputs.set(jid, new Set()); + jobOutputs.get(jid)!.add(did); + } + } + for (const jid of Array.from(jobInputs.keys())) { + const inputs = jobInputs.get(jid)!; + const outputs = jobOutputs.get(jid); + if (!outputs) continue; + Array.from(inputs).forEach((inn) => { + Array.from(outputs).forEach((out) => { + mediatedPairs.add(`${inn}=>${out}`); + }); + }); + } + + const keepEdge = (e: OpenLineageGraphData["edges"][number]) => { + const src = makeId(e.source_type, e.source_namespace, e.source_name); + const tgt = makeId(e.target_type, e.target_namespace, e.target_name); + if (!nodeIds.has(src) || !nodeIds.has(tgt)) return false; + if (e.edge_type === "derived") { + return !mediatedPairs.has(`${src}=>${tgt}`); + } + // Drop I/O edges for definition jobs (those jobs are not on the graph) + if (e.edge_type === "input" || e.edge_type === "output") { + if (e.source_type === "job" && !jobsOnPath.has(src)) return false; + if (e.target_type === "job" && !jobsOnPath.has(tgt)) return false; + } + return ( + e.edge_type === "input" || + e.edge_type === "output" || + e.edge_type === "parent" || + e.edge_type === "symlink" + ); + }; + + const edges = [ + ...ioEdges, + ...parentEdges, + ...symlinkEdges, + ...derivedEdges, + ].filter(keepEdge); + + return { nodes, edges }; +}; + // ── Custom Node ── interface LineageNodeData { @@ -108,7 +282,8 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => { const [isHovered, setIsHovered] = useState(false); const colors = getProducerColors(data.producer); const icon = getNodeIcon(data.type); - const producerLabel = normalizeProducer(data.producer); + const producerLabel = displayProducer(data.producer); + const isJob = data.type === "job"; const handleClick = () => { if (data.onNodeClick && data.nodeRef) { @@ -120,10 +295,12 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => {
{ zIndex: 5, }} > - {producerLabel} + {isJob ? "transform" : producerLabel}
{data.namespace && isHovered && ( @@ -190,6 +367,7 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => { style={{ flex: 1, display: "flex", + flexDirection: "column", alignItems: "center", justifyContent: "center", padding: "0 10px", @@ -198,7 +376,21 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => { color: "#333333", }} > - {data.label} +
+ {data.label} +
+ {isJob && ( +
+ click for runs +
+ )} - - {normalizeProducer(node.producer)} - + {displayProducer(node.producer)} {node.job_type && ( {node.job_type} @@ -737,7 +930,11 @@ const NodeDetailPanel: React.FC<{ {node.feast_object_name && (
- Name:{" "} + + {node.feast_object_type === "onlineStore" + ? "Feature View:" + : "Name:"} + {" "} {node.feast_object_name}
)} @@ -750,6 +947,24 @@ const NodeDetailPanel: React.FC<{ )} + {onlineStore && ( +
+
Online Store
+ {onlineStore.store_type && ( +
+ Backend:{" "} + {onlineStore.store_type} +
+ )} + {onlineStore.feature_view && ( +
+ Feature View:{" "} + {onlineStore.feature_view} +
+ )} +
+ )} + {dataSource && (
Data Source
@@ -843,11 +1058,27 @@ const NodeDetailPanel: React.FC<{ )} {node.type === "job" && ( - + <> +
+ Materialization / compute run. Select a run below for inputs, + outputs, and status. (FeatureService membership is not a transform — + that stays as dataset links.) +
+ + )}
); @@ -860,6 +1091,12 @@ interface LineageGraphProps { olLoading: boolean; olError: boolean; feastOnlyCheckbox?: React.ReactNode; + /** + * lineage — datasets + runtime transforms (materialize, Spark); default + * objects — datasets only + * all — every consumer node/edge + */ + viewMode?: "lineage" | "objects" | "all"; } const LineageGraph: React.FC = ({ @@ -867,6 +1104,7 @@ const LineageGraph: React.FC = ({ olLoading, olError, feastOnlyCheckbox, + viewMode = "lineage", }) => { const [nodes, setNodes, onNodesChange] = useNodesState([]); const [edges, setEdges, onEdgesChange] = useEdgesState([]); @@ -878,43 +1116,70 @@ const LineageGraph: React.FC = ({ null, ); + const objectsOnly = viewMode === "objects"; + const showTypeFilter = viewMode !== "objects"; + + const { baseNodes, baseEdges } = useMemo(() => { + if (!olData) return { baseNodes: [] as OpenLineageNode[], baseEdges: [] }; + if (viewMode === "all") { + return { baseNodes: olData.nodes, baseEdges: olData.edges }; + } + if (viewMode === "objects") { + return { + baseNodes: olData.nodes.filter((n) => n.type === "dataset"), + baseEdges: olData.edges.filter( + (e) => + e.source_type === "dataset" && + e.target_type === "dataset" && + e.edge_type !== "parent", + ), + }; + } + const slice = buildLineageGraphSlice(olData); + return { baseNodes: slice.nodes, baseEdges: slice.edges }; + }, [olData, viewMode]); + const producers = useMemo(() => { - if (!olData) return []; const set = new Set(); - for (const n of olData.nodes) { - set.add(normalizeProducer(n.producer)); + for (const n of baseNodes) { + set.add(displayProducer(n.producer)); } return Array.from(set).sort(); - }, [olData]); + }, [baseNodes]); const objectOptions = useMemo(() => { - if (!olData) return []; - return olData.nodes + return baseNodes .filter((n) => { if (filterType && n.type !== filterType) return false; - if (filterProducer && normalizeProducer(n.producer) !== filterProducer) + if (filterProducer && displayProducer(n.producer) !== filterProducer) return false; return true; }) .map((n) => n.name) .filter((v, i, a) => a.indexOf(v) === i) .sort(); - }, [olData, filterType, filterProducer]); + }, [baseNodes, filterType, filterProducer]); useEffect(() => { setFilterObject(""); }, [filterType, filterProducer]); + useEffect(() => { + if (objectsOnly && filterType === "job") { + setFilterType(""); + } + }, [objectsOnly, filterType]); + useEffect(() => { if (!olData) return; - let filteredNodes = olData.nodes; + let filteredNodes = baseNodes; if (filterType) { filteredNodes = filteredNodes.filter((n) => n.type === filterType); } if (filterProducer) { filteredNodes = filteredNodes.filter( - (n) => normalizeProducer(n.producer) === filterProducer, + (n) => displayProducer(n.producer) === filterProducer, ); } @@ -929,7 +1194,7 @@ const LineageGraph: React.FC = ({ ); const connectedIds = new Set(); - for (const e of olData.edges) { + for (const e of baseEdges) { const srcId = makeId(e.source_type, e.source_namespace, e.source_name); const tgtId = makeId(e.target_type, e.target_namespace, e.target_name); if (focusIds.has(srcId)) connectedIds.add(tgtId); @@ -939,7 +1204,7 @@ const LineageGraph: React.FC = ({ const visibleIds = new Set( Array.from(focusIds).concat(Array.from(connectedIds)), ); - filteredNodes = olData.nodes.filter((n) => + filteredNodes = baseNodes.filter((n) => visibleIds.has(makeId(n.type, n.namespace, n.name)), ); } @@ -962,34 +1227,43 @@ const LineageGraph: React.FC = ({ position: { x: 0, y: 0 }, })); - const flowEdges: Edge[] = olData.edges + const flowEdges: Edge[] = baseEdges .filter((e) => { const srcId = makeId(e.source_type, e.source_namespace, e.source_name); const tgtId = makeId(e.target_type, e.target_namespace, e.target_name); return filteredNodeIds.has(srcId) && filteredNodeIds.has(tgtId); }) .map((e, i) => { - const isSymlink = e.edge_type === "symlink"; + const edgeType = e.edge_type || ""; + const isSymlink = edgeType === "symlink"; + const isParent = edgeType === "parent"; + const isDerived = edgeType === "derived"; const color = isSymlink ? "#999999" - : e.edge_type === "derived" - ? "#3366cc" - : "#e67300"; + : isParent + ? "#7a7a7a" + : isDerived + ? "#3366cc" + : "#e67300"; return { id: `ol-edge-${i}`, source: makeId(e.source_type, e.source_namespace, e.source_name), sourceHandle: "source", target: makeId(e.target_type, e.target_namespace, e.target_name), targetHandle: "target", - animated: !isSymlink, + animated: !isSymlink && !isParent, + label: isParent ? "runs on" : undefined, + labelStyle: isParent ? { fontSize: 10, fill: "#666" } : undefined, style: { - strokeWidth: isSymlink ? 1 : 2, + strokeWidth: isSymlink || isParent ? 1.5 : 2, stroke: color, strokeDasharray: isSymlink ? "3 3" - : e.edge_type === "derived" - ? "5 3" - : "none", + : isParent + ? "2 4" + : isDerived + ? "5 3" + : "none", }, type: "smoothstep", markerEnd: { @@ -1004,7 +1278,16 @@ const LineageGraph: React.FC = ({ const { nodes: ln, edges: le } = layoutGraph(flowNodes, flowEdges); setNodes(ln); setEdges(le); - }, [olData, filterType, filterProducer, filterObject, setNodes, setEdges]); + }, [ + olData, + baseNodes, + baseEdges, + filterType, + filterProducer, + filterObject, + setNodes, + setEdges, + ]); if (olLoading) { return ( @@ -1034,17 +1317,19 @@ const LineageGraph: React.FC = ({ ); } - if (olData.nodes.length === 0) { + if (baseNodes.length === 0) { return ( No Lineage Events} + title={ +

{objectsOnly ? "No Dataset Lineage" : "No Lineage Yet"}

+ } body={

- No OpenLineage events have been received yet. Configure your data - pipeline producers (Airflow, Spark, dbt, Feast) to send events to - this instance. + {objectsOnly + ? "No dataset lineage edges have been recorded yet. Materialize features or emit OpenLineage events that include datasets." + : "No OpenLineage events yet. Apply features and run materialization so datasets, jobs, and runs appear here."}

} /> @@ -1052,40 +1337,66 @@ const LineageGraph: React.FC = ({ ); } + const title = + viewMode === "objects" + ? "Dataset Lineage" + : viewMode === "lineage" + ? "Lineage" + : "OpenLineage Graph"; + return (
- -

OpenLineage Graph

-
+
+ +

{title}

+
+ {viewMode === "lineage" && ( +

+ Datasets link by definition (source → feature view → feature + service). Runtime transforms — materialize and compute engines + like Spark — appear as dashed pills; click them for run history. +

+ )} +
{feastOnlyCheckbox && ( -
+
{feastOnlyCheckbox}
)}
- - - setFilterType(e.target.value)} - aria-label="Filter by type" - /> - - + {showTypeFilter && ( + + + setFilterType(e.target.value)} + aria-label="Filter by type" + /> + + + )} = ({ - + = ({ ]} value={filterObject} onChange={(e) => setFilterObject(e.target.value)} - aria-label="Select object" + aria-label="Focus on object" /> diff --git a/ui/src/pages/lineage/Index.tsx b/ui/src/pages/lineage/Index.tsx index 40bf7f83afc..20867a6ca59 100644 --- a/ui/src/pages/lineage/Index.tsx +++ b/ui/src/pages/lineage/Index.tsx @@ -14,13 +14,15 @@ import RegistryPathContext from "../../contexts/RegistryPathContext"; import RegistryVisualizationTab from "../../components/RegistryVisualizationTab"; import { LineageGraph } from "../../components/OpenLineageGraph"; import LineageEventsList from "../../components/LineageEventsList"; +import LineageJobsList from "../../components/LineageJobsList"; import { useLoadOpenLineageGraph } from "../../queries/useLoadOpenLineageGraph"; import { useParams } from "react-router-dom"; -type ActiveTab = "lineage" | "events"; +type ActiveTab = "lineage" | "jobs" | "events"; const tabButtons = [ { id: "lineage", label: "Lineage" }, + { id: "jobs", label: "Jobs" }, { id: "events", label: "Events" }, ]; @@ -132,6 +134,7 @@ const LineagePage = () => { /> ) : ( { )} + {activeTab === "jobs" && } + {activeTab === "events" && } ) : ( diff --git a/ui/src/queries/useLoadOpenLineageGraph.ts b/ui/src/queries/useLoadOpenLineageGraph.ts index 25bd4e12389..f8154a405a6 100644 --- a/ui/src/queries/useLoadOpenLineageGraph.ts +++ b/ui/src/queries/useLoadOpenLineageGraph.ts @@ -125,8 +125,36 @@ const useLoadRegistryLineage = (project?: string) => { ); }; +export interface OpenLineageJob { + job_namespace: string; + job_name: string; + job_type?: string | null; + producer?: string | null; + description?: string | null; + latest_run_id?: string | null; + updated_at: number; + facets_json?: string | null; +} + +const useLoadOpenLineageJobs = () => { + const registryUrl = useContext(RegistryPathContext); + const { fetchOptions } = useDataMode(); + + return useQuery<{ jobs: OpenLineageJob[] }>( + ["openlineage-jobs"], + () => + restFetch<{ jobs: OpenLineageJob[] }>( + registryUrl, + "/lineage/openlineage/jobs", + fetchOptions, + ), + { enabled: !!registryUrl }, + ); +}; + export { useLoadOpenLineageGraph, useLoadOpenLineageEvents, + useLoadOpenLineageJobs, useLoadRegistryLineage, }; From ff298df8f3da0499998dbf75cd30202d873b0896 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Sun, 9 Aug 2026 21:07:39 +0530 Subject: [PATCH 2/3] feat: Add saved datasets to lineage Signed-off-by: ntkathole --- .../feast/api/registry/rest/__init__.py | 63 +++- sdk/python/feast/api/registry/rest/lineage.py | 6 + sdk/python/feast/feature_store.py | 16 + sdk/python/feast/lineage/registry_lineage.py | 169 +++++++++- sdk/python/feast/openlineage/client.py | 4 +- sdk/python/feast/openlineage/consumer.py | 52 ++- sdk/python/feast/openlineage/emitter.py | 297 ++++++++++++++++++ sdk/python/feast/openlineage/facets.py | 31 ++ sdk/python/feast/openlineage/processor.py | 1 + sdk/python/feast/openlineage/store.py | 57 +++- ui/src/components/LineageEventsList.tsx | 33 +- ui/src/components/RegistryVisualization.tsx | 34 ++ ui/src/hooks/useFCOExploreSuggestions.ts | 1 + ui/src/pages/lineage/Index.tsx | 61 +++- ui/src/parsers/parseEntityRelationships.ts | 123 ++++++++ ui/src/parsers/types.ts | 1 + ui/src/queries/useLoadOpenLineageGraph.ts | 34 +- 17 files changed, 944 insertions(+), 39 deletions(-) diff --git a/sdk/python/feast/api/registry/rest/__init__.py b/sdk/python/feast/api/registry/rest/__init__.py index 4f5712e1493..4aaf59635cd 100644 --- a/sdk/python/feast/api/registry/rest/__init__.py +++ b/sdk/python/feast/api/registry/rest/__init__.py @@ -25,6 +25,11 @@ _ol_processor: Optional[Any] = None +def get_ol_processor() -> Optional[Any]: + """Return the global OpenLineage processor, if initialized.""" + return _ol_processor + + def register_all_routes(app: FastAPI, grpc_handler, server=None, store=None): app.include_router(get_entity_router(grpc_handler)) app.include_router(get_data_source_router(grpc_handler)) @@ -133,19 +138,67 @@ def _register_openlineage_consumer(app: FastAPI, feast_store): # Wire the local processor into Feast's own OL emitter so Feast events # are also stored in the consumer DB automatically. + # The emitter is lazy-initialized, so it may be None at startup. + # Two-pronged approach: + # 1. If already initialized, wire now. + # 2. Store processor globally so _init_openlineage_emitter() can + # pick it up when the emitter is lazily created later. try: - if feast_store and hasattr(feast_store, "_openlineage_emitter"): - emitter = feast_store._openlineage_emitter - if emitter and hasattr(emitter, "_client") and emitter._client: - emitter._client.set_local_processor(processor) - logger.info("Feast OL emitter wired to local consumer processor") + emitter = getattr(feast_store, "_openlineage_emitter", None) + if emitter and hasattr(emitter, "_client") and emitter._client: + emitter._client.set_local_processor(processor) + logger.info( + "Feast OL emitter wired to local consumer processor (eager)" + ) except Exception as wire_err: logger.debug(f"Could not wire emitter to local processor: {wire_err}") + def _build_allowed_namespaces_fn(fs): + """Build a callback that derives OL namespace access from Feast RBAC. + + Uses ``permitted_resources`` with ``DESCRIBE`` on all known projects. + The projects the current user may describe become the allowed OL + namespaces (mapped through ``resolve_namespace``). + """ + + def _get_allowed(): + try: + from feast.openlineage.identity import resolve_namespace + from feast.permissions.action import AuthzedAction + from feast.permissions.security_manager import ( + get_security_manager, + permitted_resources, + ) + + sm = get_security_manager() + if sm is None: + return None + + all_projects = fs.registry.list_projects(allow_cache=True) + if not all_projects: + return None + + allowed_projects = permitted_resources( + all_projects, AuthzedAction.DESCRIBE + ) + + ol_ns_config = getattr(ol_config, "namespace", "feast") + namespaces = set() + for p in allowed_projects: + namespaces.add(resolve_namespace(ol_ns_config, p.name)) + return list(namespaces) if namespaces else None + except Exception: + return None + + return _get_allowed + + get_allowed_namespaces = _build_allowed_namespaces_fn(feast_store) + consumer_router = get_consumer_router( config=ol_config, store=ol_store, processor=processor, + get_allowed_namespaces=get_allowed_namespaces, ) app.include_router(consumer_router) diff --git a/sdk/python/feast/api/registry/rest/lineage.py b/sdk/python/feast/api/registry/rest/lineage.py index 360d6210aa6..143ec352c46 100644 --- a/sdk/python/feast/api/registry/rest/lineage.py +++ b/sdk/python/feast/api/registry/rest/lineage.py @@ -86,6 +86,7 @@ def get_object_relationships_path( "featureView", "featureService", "feature", + "savedDataset", ] if object_type not in valid_types: raise ValueError( @@ -175,6 +176,7 @@ def get_complete_registry_data( "featureServices": project_resources.get("featureServices", []), "features": project_resources.get("features", []), "labels": project_resources.get("labels", []), + "savedDatasets": project_resources.get("savedDatasets", []), }, "relationships": lineage_response.get("relationships", []), "indirectRelationships": lineage_response.get("indirectRelationships", []), @@ -186,6 +188,7 @@ def get_complete_registry_data( "featureServices": pagination.get("featureServices", {}), "features": pagination.get("features", {}), "labels": pagination.get("labels", {}), + "savedDatasets": pagination.get("savedDatasets", {}), "relationships": lineage_response.get("relationshipsPagination", {}), "indirectRelationships": lineage_response.get( "indirectRelationshipsPagination", {} @@ -274,6 +277,8 @@ def get_complete_registry_data_all( feat["project"] = project_name for lbl in project_resources.get("labels", []): lbl["project"] = project_name + for sd in project_resources.get("savedDatasets", []): + sd["project"] = project_name all_data.append( { "project": project_name, @@ -285,6 +290,7 @@ def get_complete_registry_data_all( "featureServices": project_resources.get("featureServices", []), "features": project_resources.get("features", []), "labels": project_resources.get("labels", []), + "savedDatasets": project_resources.get("savedDatasets", []), }, "relationships": lineage_response.get("relationships", []), "indirectRelationships": lineage_response.get( diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 3a80b504178..4607f84d150 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -376,6 +376,7 @@ def _init_openlineage_emitter(self) -> Optional[Any]: ol_config = self.config.openlineage.to_openlineage_config() emitter = FeastOpenLineageEmitter(ol_config) if emitter.is_enabled: + self._wire_local_processor(emitter) return emitter except ImportError: # OpenLineage not installed, silently skip @@ -384,6 +385,21 @@ def _init_openlineage_emitter(self) -> Optional[Any]: warnings.warn(f"Failed to initialize OpenLineage emitter: {e}") return None + def _wire_local_processor(self, emitter: Any) -> None: + """Wire the local OL consumer processor into the emitter so + Feast-produced events are also stored in the consumer DB.""" + try: + from feast.api.registry.rest import get_ol_processor + + processor = get_ol_processor() + if processor and hasattr(emitter, "_client") and emitter._client: + emitter._client.set_local_processor(processor) + _logger.info( + "Feast OL emitter wired to local consumer processor (lazy)" + ) + except Exception as e: + _logger.debug(f"Could not wire emitter to local processor: {e}") + def __repr__(self) -> str: # Show lazy loading status without triggering initialization registry_status = "not loaded" if self._registry is None else "loaded" diff --git a/sdk/python/feast/lineage/registry_lineage.py b/sdk/python/feast/lineage/registry_lineage.py index ce038f4d1ae..c58fbde65a3 100644 --- a/sdk/python/feast/lineage/registry_lineage.py +++ b/sdk/python/feast/lineage/registry_lineage.py @@ -2,17 +2,102 @@ Registry lineage generation for Feast objects. This module provides functionality to generate relationship graphs between -Feast objects (entities, feature views, data sources, feature services) -for lineage visualization. +Feast objects (entities, feature views, data sources, feature services, +saved datasets) for lineage visualization. """ from dataclasses import dataclass from enum import Enum -from typing import Dict, List, Tuple +from typing import Dict, List, Set, Tuple from feast.protos.feast.core.Registry_pb2 import Registry +def _extract_storage_identifiers(storage) -> Set[str]: + """Extract physical location identifiers from a SavedDatasetStorage proto. + + Returns a set of non-empty strings (URIs, table names, paths) that can + be matched against DataSource options. + """ + ids: Set[str] = set() + if hasattr(storage, "file_storage") and storage.HasField("file_storage"): + if storage.file_storage.uri: + ids.add(storage.file_storage.uri) + if hasattr(storage, "bigquery_storage") and storage.HasField("bigquery_storage"): + if storage.bigquery_storage.table: + ids.add(storage.bigquery_storage.table) + if hasattr(storage, "redshift_storage") and storage.HasField("redshift_storage"): + if storage.redshift_storage.table: + ids.add(storage.redshift_storage.table) + if hasattr(storage, "snowflake_storage") and storage.HasField("snowflake_storage"): + if storage.snowflake_storage.table: + ids.add(storage.snowflake_storage.table) + if hasattr(storage, "spark_storage") and storage.HasField("spark_storage"): + if storage.spark_storage.path: + ids.add(storage.spark_storage.path) + if storage.spark_storage.table: + ids.add(storage.spark_storage.table) + if hasattr(storage, "trino_storage") and storage.HasField("trino_storage"): + if storage.trino_storage.table: + ids.add(storage.trino_storage.table) + if hasattr(storage, "athena_storage") and storage.HasField("athena_storage"): + if storage.athena_storage.table: + ids.add(storage.athena_storage.table) + return ids + + +def _extract_datasource_identifiers(data_source) -> Set[str]: + """Extract physical location identifiers from a DataSource proto. + + Returns a set of non-empty strings (URIs, table names, paths) that can + be compared against SavedDatasetStorage identifiers. + """ + ids: Set[str] = set() + opts = ( + data_source.WhichOneof("options") + if hasattr(data_source, "WhichOneof") + else None + ) + if opts == "file_options" and data_source.file_options.uri: + ids.add(data_source.file_options.uri) + elif opts == "bigquery_options" and data_source.bigquery_options.table: + ids.add(data_source.bigquery_options.table) + elif opts == "redshift_options" and data_source.redshift_options.table: + ids.add(data_source.redshift_options.table) + elif opts == "snowflake_options" and data_source.snowflake_options.table: + ids.add(data_source.snowflake_options.table) + elif opts == "spark_options": + if data_source.spark_options.path: + ids.add(data_source.spark_options.path) + if data_source.spark_options.table: + ids.add(data_source.spark_options.table) + elif opts == "trino_options" and data_source.trino_options.table: + ids.add(data_source.trino_options.table) + elif opts == "athena_options" and data_source.athena_options.table: + ids.add(data_source.athena_options.table) + + # Also check batch_source if present (FeatureView's embedded source) + if hasattr(data_source, "batch_source") and data_source.HasField("batch_source"): + ids.update(_extract_datasource_identifiers(data_source.batch_source)) + + return ids + + +def _build_datasource_location_index(registry: Registry) -> Dict[str, str]: + """Build a reverse index: physical location → DataSource name. + + Scans all DataSources in the registry and maps each physical identifier + (URI, table, path) to the DataSource's name. + """ + location_to_name: Dict[str, str] = {} + for ds in registry.data_sources: + if not (hasattr(ds, "name") and ds.name): + continue + for loc_id in _extract_datasource_identifiers(ds): + location_to_name[loc_id] = ds.name + return location_to_name + + class FeastObjectType(Enum): DATA_SOURCE = "dataSource" ENTITY = "entity" @@ -20,6 +105,7 @@ class FeastObjectType(Enum): LABEL_VIEW = "labelView" FEATURE_SERVICE = "featureService" FEATURE = "feature" + SAVED_DATASET = "savedDataset" @dataclass @@ -390,6 +476,83 @@ def _parse_direct_relationships(self, registry: Registry) -> List[EntityRelation ) ) + # SavedDataset relationships + ds_location_index = _build_datasource_location_index(registry) + + for saved_dataset in registry.saved_datasets: + if hasattr(saved_dataset, "spec") and saved_dataset.spec: + # FeatureService -> SavedDataset (when created via a feature service) + if ( + hasattr(saved_dataset.spec, "feature_service_name") + and saved_dataset.spec.feature_service_name + ): + relationships.append( + EntityRelation( + source=EntityReference( + FeastObjectType.FEATURE_SERVICE, + saved_dataset.spec.feature_service_name, + ), + target=EntityReference( + FeastObjectType.SAVED_DATASET, + saved_dataset.spec.name, + ), + ) + ) + + # FeatureView -> SavedDataset (derived from feature refs "view:feat") + if ( + hasattr(saved_dataset.spec, "features") + and saved_dataset.spec.features + ): + from feast.utils import _parse_feature_ref + + seen_views: set = set() + for feat_ref in saved_dataset.spec.features: + try: + view_name, _, _ = _parse_feature_ref(feat_ref) + except ValueError: + continue + if view_name and view_name not in seen_views: + seen_views.add(view_name) + relationships.append( + EntityRelation( + source=EntityReference( + FeastObjectType.FEATURE_VIEW, + view_name, + ), + target=EntityReference( + FeastObjectType.SAVED_DATASET, + saved_dataset.spec.name, + ), + ) + ) + + # DataSource -> SavedDataset (matched via storage location) + if ( + hasattr(saved_dataset.spec, "storage") + and saved_dataset.spec.storage + ): + storage_ids = _extract_storage_identifiers( + saved_dataset.spec.storage + ) + matched_ds_names: set = set() + for loc_id in storage_ids: + ds_name = ds_location_index.get(loc_id) + if ds_name and ds_name not in matched_ds_names: + matched_ds_names.add(ds_name) + relationships.append( + EntityRelation( + source=EntityReference( + FeastObjectType.DATA_SOURCE, + ds_name, + ), + target=EntityReference( + FeastObjectType.SAVED_DATASET, + saved_dataset.spec.name, + ), + ) + ) + return relationships def _parse_indirect_relationships( diff --git a/sdk/python/feast/openlineage/client.py b/sdk/python/feast/openlineage/client.py index d97bc482663..63436fe26bb 100644 --- a/sdk/python/feast/openlineage/client.py +++ b/sdk/python/feast/openlineage/client.py @@ -275,6 +275,7 @@ def emit_job_event( inputs: Optional[List[Any]] = None, outputs: Optional[List[Any]] = None, job_facets: Optional[Dict[str, Any]] = None, + namespace: Optional[str] = None, ) -> bool: """ Emit a JobEvent for a Feast job definition. @@ -284,6 +285,7 @@ def emit_job_event( inputs: List of input datasets outputs: List of output datasets job_facets: Job facets + namespace: Optional namespace for the job (defaults to client namespace) Returns: True if successful, False otherwise @@ -297,7 +299,7 @@ def emit_job_event( event = JobEvent( eventTime=datetime.now(timezone.utc).isoformat(), job=Job( - namespace=self.namespace, + namespace=namespace or self.namespace, name=job_name, facets=job_facets or {}, ), diff --git a/sdk/python/feast/openlineage/consumer.py b/sdk/python/feast/openlineage/consumer.py index 995de4d131f..6dfa7a0bf90 100644 --- a/sdk/python/feast/openlineage/consumer.py +++ b/sdk/python/feast/openlineage/consumer.py @@ -257,21 +257,43 @@ def get_lineage_graph( ) return graph + @router.get("/lineage/openlineage/namespaces") + def list_namespaces(): + """List all distinct namespaces known to the lineage store.""" + ns_filter = _get_namespace_filter(get_allowed_namespaces) + all_ns = store.get_all_namespaces() + if ns_filter is not None: + all_ns = [ns for ns in all_ns if ns in ns_filter] + return {"namespaces": all_ns} + @router.get("/lineage/openlineage/graph") - def get_full_lineage_graph(): + def get_full_lineage_graph( + namespace: Optional[str] = Query(None), + limit: int = Query(0, ge=0, le=10000), + offset: int = Query(0, ge=0), + ): """ - Get all lineage edges (RBAC-filtered by namespace). + Get all lineage nodes and edges (RBAC-filtered by namespace). + + Optional query params: + - namespace: filter to a single namespace + - limit/offset: paginate nodes (0 = no limit) Includes symlink edges that connect datasets across producers when they reference the same physical data (via SymlinksDatasetFacet or matching dataSource URIs). """ ns_filter = _get_namespace_filter(get_allowed_namespaces) + if namespace: + if ns_filter is not None and namespace not in ns_filter: + return {"nodes": [], "edges": [], "symlinks": []} + ns_filter = [namespace] + edges = store.get_all_lineage_edges(namespaces=ns_filter) datasets = store.get_datasets(namespaces=ns_filter) jobs = store.get_jobs(namespaces=ns_filter) - nodes = [] + nodes: list = [] for ds in datasets: facets = _safe_parse_json(ds.get("facets_json")) schema = _safe_parse_json(ds.get("schema_json")) @@ -305,8 +327,30 @@ def get_full_lineage_graph(): ) symlinks = store.get_all_symlinks() + for sl in symlinks: + edges.append( + { + "source_type": "dataset", + "source_namespace": sl["dataset_namespace"], + "source_name": sl["dataset_name"], + "target_type": "dataset", + "target_namespace": sl["linked_namespace"], + "target_name": sl["linked_name"], + "edge_type": "symlink", + "updated_at": sl.get("updated_at"), + } + ) + + total_nodes = len(nodes) + if limit > 0: + nodes = nodes[offset : offset + limit] - return {"nodes": nodes, "edges": edges, "symlinks": symlinks} + return { + "nodes": nodes, + "edges": edges, + "symlinks": symlinks, + "total_nodes": total_nodes, + } # ── Run history endpoints ── diff --git a/sdk/python/feast/openlineage/emitter.py b/sdk/python/feast/openlineage/emitter.py index 72cc1005a89..b7a2c8c844c 100644 --- a/sdk/python/feast/openlineage/emitter.py +++ b/sdk/python/feast/openlineage/emitter.py @@ -107,6 +107,50 @@ def _get_namespace(self, project: str) -> str: """Backward-compatible alias for :meth:`namespace_for`.""" return self.namespace_for(project) + @staticmethod + def _match_storage_to_datasources( + saved_dataset: Any, + registered_data_sources: Optional[List[Any]], + ) -> List[str]: + """Match a SavedDataset's storage location against registered DataSources. + + Compares the physical identifiers (URI, table, path) from the + SavedDataset's storage with those from each registered DataSource. + + Returns: + List of matched DataSource names. + """ + if not registered_data_sources or not hasattr(saved_dataset, "storage"): + return [] + + try: + storage_proto = saved_dataset.storage.to_proto() + except Exception: + return [] + + from feast.lineage.registry_lineage import ( + _extract_datasource_identifiers, + _extract_storage_identifiers, + ) + + storage_ids = _extract_storage_identifiers(storage_proto) + if not storage_ids: + return [] + + matched: List[str] = [] + seen: set = set() + for ds in registered_data_sources: + if not hasattr(ds, "name") or not ds.name or ds.name in seen: + continue + try: + ds_ids = _extract_datasource_identifiers(ds.to_proto()) + except Exception: + continue + if storage_ids & ds_ids: + seen.add(ds.name) + matched.append(ds.name) + return matched + def _job_kind_facets(self, kind: str, project: str) -> Dict[str, Any]: from feast.openlineage.facets import FeastJobKindFacet from feast.openlineage.identity import FeastJobKind @@ -205,6 +249,28 @@ def emit_registry_lineage( except Exception as e: logger.error(f"Error emitting feature service lineage: {e}") + # Emit events for saved datasets + try: + saved_datasets = registry.list_saved_datasets( + project=project, allow_cache=allow_cache + ) + # Fetch registered data sources for storage-based matching + registered_data_sources = [] + try: + registered_data_sources = registry.list_data_sources( + project=project, allow_cache=allow_cache + ) + except Exception: + pass + + for sd in saved_datasets: + result = self.emit_saved_dataset_lineage( + sd, project, registered_data_sources=registered_data_sources + ) + results.append(result) + except Exception as e: + logger.error(f"Error emitting saved dataset lineage: {e}") + logger.info( f"Emitted {sum(results)}/{len(results)} lineage events for registry" ) @@ -486,6 +552,138 @@ def emit_feature_service_lineage( ) return False + def emit_saved_dataset_lineage( + self, + saved_dataset: Any, + project: str, + registered_data_sources: Optional[List[Any]] = None, + ) -> bool: + """ + Emit lineage for a saved dataset definition. + + Creates a definition job with inputs derived from: + - FeatureService (when feature_service_name is set) + - FeatureViews (extracted from feature refs in the format "view:feat") + - DataSources (matched by comparing storage location against registered sources) + + Args: + saved_dataset: The SavedDataset object + project: Project name + registered_data_sources: Optional list of registered DataSource objects + for storage-based matching + + Returns: + True if successful, False otherwise + """ + if not self.is_enabled: + return False + + try: + from openlineage.client.facet_v2 import schema_dataset + + from feast.openlineage.facets import ( + FeastProjectFacet, + FeastSavedDatasetFacet, + ) + from feast.openlineage.identity import FeastJobKind + + namespace = self._get_namespace(project) + + inputs = [] + if saved_dataset.feature_service_name: + inputs.append( + InputDataset( + namespace=namespace, + name=saved_dataset.feature_service_name, + ) + ) + + # FeatureView inputs from feature refs ("view_name:feature_name") + if saved_dataset.features: + from feast.utils import _parse_feature_ref + + seen_views: set = set() + for feat_ref in saved_dataset.features: + try: + view_name, _, _ = _parse_feature_ref(feat_ref) + except ValueError: + continue + if view_name and view_name not in seen_views: + seen_views.add(view_name) + inputs.append( + InputDataset( + namespace=namespace, + name=view_name, + ) + ) + + # DataSource inputs matched by storage location + matched_ds = self._match_storage_to_datasources( + saved_dataset, registered_data_sources + ) + for ds_name in matched_ds: + inputs.append( + InputDataset( + namespace=namespace, + name=ds_name, + ) + ) + + sd_facets: Dict[str, Any] = { + "feast_savedDataset": FeastSavedDatasetFacet( + name=saved_dataset.name, + features=list(saved_dataset.features) + if saved_dataset.features + else [], + join_keys=list(saved_dataset.join_keys) + if saved_dataset.join_keys + else [], + feature_service_name=saved_dataset.feature_service_name or "", + full_feature_names=saved_dataset.full_feature_names, + description=saved_dataset.description + if hasattr(saved_dataset, "description") + and saved_dataset.description + else "", + tags=saved_dataset.tags if saved_dataset.tags else {}, + ) + } + + if saved_dataset.features: + sd_facets["schema"] = schema_dataset.SchemaDatasetFacet( + fields=[ + schema_dataset.SchemaDatasetFacetFields(name=f, type="UNKNOWN") + for f in saved_dataset.features + ] + ) + + outputs = [ + OutputDataset( + namespace=namespace, + name=saved_dataset.name, + facets=sd_facets, + ) + ] + + job_facets = { + "feast_project": FeastProjectFacet(project_name=project), + **self._job_kind_facets(FeastJobKind.DEFINITION, project), + } + + return self._client.emit_run_event( + job_name=f"saved_dataset_{saved_dataset.name}", + run_id=str(uuid.uuid4()), + event_type=RunState.COMPLETE, + inputs=inputs, + outputs=outputs, + job_facets=job_facets, + namespace=namespace, + ) + except Exception as e: + logger.error( + f"Error emitting saved dataset lineage for {saved_dataset.name}: {e}" + ) + return False + def emit_materialize_start( self, feature_views: List["FeatureView"], @@ -809,6 +1007,7 @@ def emit_apply( entity_to_dataset, feast_field_to_schema_field, ) + from feast.saved_dataset import SavedDataset from feast.stream_feature_view import StreamFeatureView try: @@ -823,6 +1022,7 @@ def emit_apply( feature_views: List[Union[FeatureView, OnDemandFeatureView]] = [] on_demand_feature_views: List[OnDemandFeatureView] = [] feature_services: List[FeatureService] = [] + saved_datasets: List[SavedDataset] = [] for obj in objects: if isinstance(obj, StreamFeatureView): @@ -838,6 +1038,8 @@ def emit_apply( elif isinstance(obj, Entity): if obj.name != "__dummy": entities.append(obj) + elif isinstance(obj, SavedDataset): + saved_datasets.append(obj) # ============================================================ # Job 1: DataSources + Entities → FeatureViews @@ -1126,6 +1328,101 @@ def emit_apply( ) results.append(result) + # ============================================================ + # SavedDatasets: FeatureService/FeatureView → SavedDataset + # ============================================================ + for sd in saved_datasets: + from feast.openlineage.facets import FeastSavedDatasetFacet + + sd_inputs = [] + if sd.feature_service_name: + sd_inputs.append( + InputDataset( + namespace=namespace, + name=sd.feature_service_name, + ) + ) + + # FeatureView inputs from feature refs ("view_name:feature_name") + if sd.features: + from feast.utils import _parse_feature_ref + + seen_views: set = set() + for feat_ref in sd.features: + try: + view_name, _, _ = _parse_feature_ref(feat_ref) + except ValueError: + continue + if view_name and view_name not in seen_views: + seen_views.add(view_name) + sd_inputs.append( + InputDataset( + namespace=namespace, + name=view_name, + ) + ) + + # DataSource inputs matched by storage location + matched_ds = self._match_storage_to_datasources(sd, data_sources) + for ds_name in matched_ds: + sd_inputs.append( + InputDataset( + namespace=namespace, + name=ds_name, + ) + ) + + sd_facets: Dict[str, Any] = { + "feast_savedDataset": FeastSavedDatasetFacet( + name=sd.name, + features=list(sd.features) if sd.features else [], + join_keys=list(sd.join_keys) if sd.join_keys else [], + feature_service_name=sd.feature_service_name or "", + full_feature_names=sd.full_feature_names, + description=sd.description + if hasattr(sd, "description") and sd.description + else "", + tags=sd.tags if sd.tags else {}, + ) + } + + if sd.features: + sd_facets["schema"] = schema_dataset.SchemaDatasetFacet( + fields=[ + schema_dataset.SchemaDatasetFacetFields( + name=f, type="UNKNOWN" + ) + for f in sd.features + ] + ) + + sd_output = OutputDataset( + namespace=namespace, + name=sd.name, + facets=sd_facets, + ) + + from feast.openlineage.identity import ( + FeastJobKind, + ) + + sd_job_name = f"saved_dataset_{sd.name}" + sd_job_facets = { + "feast_project": FeastProjectFacet(project_name=project), + **self._job_kind_facets(FeastJobKind.DEFINITION, project), + } + + result = self._client.emit_run_event( + job_name=sd_job_name, + run_id=str(uuid.uuid4()), + event_type=RunState.COMPLETE, + inputs=sd_inputs, + outputs=[sd_output], + job_facets=sd_job_facets, + namespace=namespace, + ) + results.append(result) + return results except Exception as e: diff --git a/sdk/python/feast/openlineage/facets.py b/sdk/python/feast/openlineage/facets.py index 6720e253fd7..22842cf5677 100644 --- a/sdk/python/feast/openlineage/facets.py +++ b/sdk/python/feast/openlineage/facets.py @@ -305,6 +305,37 @@ def _get_schema() -> str: return f"{FEAST_FACET_SCHEMA_BASE}/FeastProjectFacet.json" +@attr.define(kw_only=True) +class FeastSavedDatasetFacet(DatasetFacet): + """ + Custom facet for Feast Saved Dataset metadata. + + A SavedDataset is a materialized snapshot of features retrieved via a + FeatureService, typically used for training or validation. + + Attributes: + name: Saved dataset name + features: List of feature names in the saved dataset + join_keys: List of join key column names + feature_service_name: Name of the FeatureService that produced this dataset + full_feature_names: Whether full feature names were used + description: Human-readable description + tags: Key-value tags + """ + + name: str = attr.field() + features: List[str] = attr.field(factory=list) + join_keys: List[str] = attr.field(factory=list) + feature_service_name: Optional[str] = attr.field(default=None) + full_feature_names: bool = attr.field(default=False) + description: str = attr.field(default="") + tags: Dict[str, str] = attr.field(factory=dict) + + @staticmethod + def _get_schema() -> str: + return f"{FEAST_FACET_SCHEMA_BASE}/FeastSavedDatasetFacet.json" + + @attr.define(kw_only=True) class FeastJobKindFacet(JobFacet): """ diff --git a/sdk/python/feast/openlineage/processor.py b/sdk/python/feast/openlineage/processor.py index 02e7227ab1d..592ecdfcabf 100644 --- a/sdk/python/feast/openlineage/processor.py +++ b/sdk/python/feast/openlineage/processor.py @@ -404,6 +404,7 @@ def _resolve_feast_mapping( ("feast_featureService", "featureService"), ("feast_entity", "entity"), ("feast_dataSource", "dataSource"), + ("feast_savedDataset", "savedDataset"), ) for facet_key, obj_type in facet_type_map: if facet_key in facets: diff --git a/sdk/python/feast/openlineage/store.py b/sdk/python/feast/openlineage/store.py index 8b9a7f86b0b..e21b7f69b45 100644 --- a/sdk/python/feast/openlineage/store.py +++ b/sdk/python/feast/openlineage/store.py @@ -60,17 +60,31 @@ def engine(self) -> Engine: def store_event(self, event_id: str, event_data: Dict[str, Any]): now = int(time.time() * 1000) + + event_type = _classify_event_type(event_data) + job = event_data.get("job", {}) run = event_data.get("run", {}) + dataset = event_data.get("dataset", {}) + + if job: + ns = job.get("namespace", "") + name = job.get("name", "") + elif dataset: + ns = dataset.get("namespace", "") + name = dataset.get("name", "") + else: + ns = "" + name = "" row = { "event_id": event_id, - "event_type": event_data.get("eventType", "UNKNOWN"), + "event_type": event_type, "event_time": _parse_timestamp(event_data.get("eventTime", "")), "producer": event_data.get("producer"), - "job_namespace": job.get("namespace", ""), - "job_name": job.get("name", ""), - "run_id": run.get("runId"), + "job_namespace": ns, + "job_name": name, + "run_id": run.get("runId") if run else None, "event_json": json.dumps(event_data), "created_at": now, } @@ -786,6 +800,26 @@ def get_run_detail(self, run_id: str) -> Optional[Dict[str, Any]]: run["facets"] = _safe_parse_json(run.pop("facets_json", None)) return run + def get_all_namespaces(self) -> List[str]: + """Return all distinct namespaces present across jobs and datasets.""" + tbl_jobs = OL_TABLES["jobs"] + tbl_ds = OL_TABLES["datasets"] + with self._engine.connect() as conn: + job_ns = conn.execute( + select(tbl_jobs.c.job_namespace).distinct() + ).fetchall() + ds_ns = conn.execute( + select(tbl_ds.c.dataset_namespace).distinct() + ).fetchall() + namespaces: set = set() + for row in job_ns: + if row[0]: + namespaces.add(row[0]) + for row in ds_ns: + if row[0]: + namespaces.add(row[0]) + return sorted(namespaces) + def get_all_lineage_edges( self, namespaces: Optional[List[str]] = None ) -> List[Dict[str, Any]]: @@ -801,6 +835,21 @@ def get_all_lineage_edges( return [dict(row._mapping) for row in rows] +def _classify_event_type(event_data: Dict[str, Any]) -> str: + """Determine the OL event type for storage. + + RunEvent has ``eventType`` (START/COMPLETE/FAIL/…). + DatasetEvent and JobEvent lack ``eventType``; classify by structure. + """ + if "eventType" in event_data: + return event_data["eventType"] + if "dataset" in event_data and "run" not in event_data and "job" not in event_data: + return "DATASET" + if "job" in event_data and "run" not in event_data: + return "JOB" + return "UNKNOWN" + + def _safe_parse_json(val: Optional[str]) -> Optional[Any]: if not val: return None diff --git a/ui/src/components/LineageEventsList.tsx b/ui/src/components/LineageEventsList.tsx index 1c029b9894d..d997e15a667 100644 --- a/ui/src/components/LineageEventsList.tsx +++ b/ui/src/components/LineageEventsList.tsx @@ -8,6 +8,8 @@ import { EuiEmptyPrompt, EuiBadge, EuiFieldSearch, + EuiSelect, + EuiFormRow, EuiFlexGroup, EuiFlexItem, EuiFlyout, @@ -17,7 +19,10 @@ import { EuiButtonEmpty, } from "@elastic/eui"; import type { OpenLineageEvent } from "../queries/useLoadOpenLineageGraph"; -import { useLoadOpenLineageEvents } from "../queries/useLoadOpenLineageGraph"; +import { + useLoadOpenLineageEvents, + useLoadNamespaces, +} from "../queries/useLoadOpenLineageGraph"; const eventTypeColor = (eventType: string) => { switch (eventType) { @@ -29,6 +34,10 @@ const eventTypeColor = (eventType: string) => { return "danger"; case "ABORT": return "warning"; + case "DATASET": + return "accent"; + case "JOB": + return "hollow"; default: return "default"; } @@ -47,6 +56,9 @@ const LineageEventsList: React.FC = () => { null, ); + const { data: nsData } = useLoadNamespaces(); + const namespaces = nsData?.namespaces || []; + const { data, isLoading, isError } = useLoadOpenLineageEvents( namespace || undefined, jobFilter || undefined, @@ -75,7 +87,7 @@ const LineageEventsList: React.FC = () => { }, { field: "job_name", - name: "Job", + name: "Job / Dataset", width: "250px", }, { @@ -142,12 +154,17 @@ const LineageEventsList: React.FC = () => { - setNamespace(e.target.value)} - aria-label="Filter by namespace" - /> + + ({ value: ns, text: ns })), + ]} + value={namespace} + onChange={(e) => setNamespace(e.target.value)} + aria-label="Filter by namespace" + /> + { return "#cc0000"; // Red case FEAST_FCO_TYPES.labelView: return "#e6570e"; // Deep orange for label views + case FEAST_FCO_TYPES.savedDataset: + return "#8B5CF6"; // Purple for saved datasets case FEAST_FCO_TYPES.mlflowRun: return "#0194e2"; // MLflow brand blue case FEAST_FCO_TYPES.mlflowModel: @@ -107,6 +109,8 @@ const getLightNodeColor = (type: FEAST_FCO_TYPES) => { return "#ffe6e6"; // Light red case FEAST_FCO_TYPES.labelView: return "#fde8dc"; // Light deep orange + case FEAST_FCO_TYPES.savedDataset: + return "#EDE9FE"; // Light purple case FEAST_FCO_TYPES.mlflowRun: return "#e6f6fd"; // Light MLflow blue case FEAST_FCO_TYPES.mlflowModel: @@ -132,6 +136,8 @@ const getNodeIcon = (type: FEAST_FCO_TYPES) => { return "◆"; // Diamond for data source case FEAST_FCO_TYPES.labelView: return "◉"; // Bullseye for label view + case FEAST_FCO_TYPES.savedDataset: + return "⬟"; // Pentagon for saved dataset case FEAST_FCO_TYPES.mlflowRun: return "⬡"; // Hexagon for MLflow run case FEAST_FCO_TYPES.mlflowModel: @@ -181,6 +187,9 @@ const CustomNode = ({ data }: { data: NodeData }) => { case FEAST_FCO_TYPES.labelView: path = `/p/${projectName}/label-view/${data.label}`; break; + case FEAST_FCO_TYPES.savedDataset: + path = `/p/${projectName}/data-set/${data.label}`; + break; default: return; } @@ -444,6 +453,7 @@ const getLayoutedElements = ( [FEAST_FCO_TYPES.featureView]: [], [FEAST_FCO_TYPES.featureService]: [], [FEAST_FCO_TYPES.labelView]: [], + [FEAST_FCO_TYPES.savedDataset]: [], [FEAST_FCO_TYPES.mlflowRun]: [], [FEAST_FCO_TYPES.mlflowModel]: [], [FEAST_FCO_TYPES.openlineageJob]: [], @@ -505,6 +515,7 @@ const Legend = () => { { type: FEAST_FCO_TYPES.labelView, label: "Label View" }, { type: FEAST_FCO_TYPES.entity, label: "Entity" }, { type: FEAST_FCO_TYPES.dataSource, label: "Data Source" }, + { type: FEAST_FCO_TYPES.savedDataset, label: "Saved Dataset" }, { type: FEAST_FCO_TYPES.mlflowRun, label: "MLflow Run" }, { type: FEAST_FCO_TYPES.mlflowModel, label: "Registered Model" }, ]; @@ -752,6 +763,27 @@ const registryToFlow = ( }); }); + (objects as any).savedDatasets?.forEach((sd: any) => { + const sdName = sd.spec?.name; + nodes.push({ + id: `sd-${sdName}`, + type: "custom", + data: { + label: sdName, + type: FEAST_FCO_TYPES.savedDataset, + metadata: sd, + permissions: permissions + ? getEntityPermissions( + permissions, + FEAST_FCO_TYPES.savedDataset, + sdName, + ) + : [], + }, + position: { x: 0, y: 0 }, + }); + }); + const dataSources = new Set(); objects.featureViews?.forEach((fv) => { @@ -938,6 +970,8 @@ const getNodePrefix = (type: FEAST_FCO_TYPES) => { return "ds"; case FEAST_FCO_TYPES.labelView: return "lv"; + case FEAST_FCO_TYPES.savedDataset: + return "sd"; case FEAST_FCO_TYPES.mlflowRun: return "mlflow"; case FEAST_FCO_TYPES.mlflowModel: diff --git a/ui/src/hooks/useFCOExploreSuggestions.ts b/ui/src/hooks/useFCOExploreSuggestions.ts index e9ab456f72b..838d6738265 100644 --- a/ui/src/hooks/useFCOExploreSuggestions.ts +++ b/ui/src/hooks/useFCOExploreSuggestions.ts @@ -23,6 +23,7 @@ const FCO_TO_URL_NAME_MAP: Record = { featureView: "/feature-view", featureService: "/feature-service", labelView: "/label-view", + savedDataset: "/data-set", mlflowRun: "/mlflow-run", mlflowModel: "/mlflow-model", openlineageJob: "/lineage", diff --git a/ui/src/pages/lineage/Index.tsx b/ui/src/pages/lineage/Index.tsx index 20867a6ca59..4487b98f9b3 100644 --- a/ui/src/pages/lineage/Index.tsx +++ b/ui/src/pages/lineage/Index.tsx @@ -6,6 +6,10 @@ import { EuiSkeletonText, EuiEmptyPrompt, EuiButtonGroup, + EuiFlexGroup, + EuiFlexItem, + EuiSelect, + EuiFormRow, } from "@elastic/eui"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; @@ -15,7 +19,10 @@ import RegistryVisualizationTab from "../../components/RegistryVisualizationTab" import { LineageGraph } from "../../components/OpenLineageGraph"; import LineageEventsList from "../../components/LineageEventsList"; import LineageJobsList from "../../components/LineageJobsList"; -import { useLoadOpenLineageGraph } from "../../queries/useLoadOpenLineageGraph"; +import { + useLoadOpenLineageGraph, + useLoadNamespaces, +} from "../../queries/useLoadOpenLineageGraph"; import { useParams } from "react-router-dom"; type ActiveTab = "lineage" | "jobs" | "events"; @@ -37,8 +44,14 @@ const LineagePage = () => { const [activeTab, setActiveTab] = useState("lineage"); const [registryOnly, setRegistryOnly] = useState(false); + const [selectedNamespace, setSelectedNamespace] = useState(""); + + const { data: nsData } = useLoadNamespaces(); + const namespaces = nsData?.namespaces || []; - const olGraphQuery = useLoadOpenLineageGraph(); + const olGraphQuery = useLoadOpenLineageGraph({ + namespace: selectedNamespace || undefined, + }); const olConsumerAvailable = !olGraphQuery.isError && olGraphQuery.data !== undefined; @@ -105,14 +118,42 @@ const LineagePage = () => { <> {olConsumerAvailable ? ( <> - setActiveTab(id as ActiveTab)} - buttonSize="m" - isFullWidth={false} - /> + + + setActiveTab(id as ActiveTab)} + buttonSize="m" + isFullWidth={false} + /> + + {namespaces.length > 1 && ( + + + ({ + value: ns, + text: ns, + })), + ]} + value={selectedNamespace} + onChange={(e) => + setSelectedNamespace(e.target.value) + } + aria-label="Filter by namespace" + /> + + + )} + {activeTab === "lineage" && ( diff --git a/ui/src/parsers/parseEntityRelationships.ts b/ui/src/parsers/parseEntityRelationships.ts index b1a014044a0..9d23ade2812 100644 --- a/ui/src/parsers/parseEntityRelationships.ts +++ b/ui/src/parsers/parseEntityRelationships.ts @@ -11,6 +11,62 @@ interface EntityRelation { target: EntityReference; } +/** + * Extract physical location identifiers (URIs, tables, paths) from a + * SavedDatasetStorage JSON object (protobuf-JSON camelCase format). + */ +const extractStorageIdentifiers = (storage: any): Set => { + const ids = new Set(); + if (!storage) return ids; + if (storage.fileStorage?.uri) ids.add(storage.fileStorage.uri); + if (storage.bigqueryStorage?.table) ids.add(storage.bigqueryStorage.table); + if (storage.redshiftStorage?.table) ids.add(storage.redshiftStorage.table); + if (storage.snowflakeStorage?.table) ids.add(storage.snowflakeStorage.table); + if (storage.sparkStorage?.path) ids.add(storage.sparkStorage.path); + if (storage.sparkStorage?.table) ids.add(storage.sparkStorage.table); + if (storage.trinoStorage?.table) ids.add(storage.trinoStorage.table); + if (storage.athenaStorage?.table) ids.add(storage.athenaStorage.table); + return ids; +}; + +/** + * Extract physical location identifiers from a DataSource JSON object. + */ +const extractDataSourceIdentifiers = (ds: any): Set => { + const ids = new Set(); + if (!ds) return ids; + if (ds.fileOptions?.uri) ids.add(ds.fileOptions.uri); + if (ds.bigqueryOptions?.table) ids.add(ds.bigqueryOptions.table); + if (ds.redshiftOptions?.table) ids.add(ds.redshiftOptions.table); + if (ds.snowflakeOptions?.table) ids.add(ds.snowflakeOptions.table); + if (ds.sparkOptions?.path) ids.add(ds.sparkOptions.path); + if (ds.sparkOptions?.table) ids.add(ds.sparkOptions.table); + if (ds.trinoOptions?.table) ids.add(ds.trinoOptions.table); + if (ds.athenaOptions?.table) ids.add(ds.athenaOptions.table); + // Embedded batch source + if (ds.batchSource) { + extractDataSourceIdentifiers(ds.batchSource).forEach((id) => ids.add(id)); + } + return ids; +}; + +/** + * Build a reverse index from physical location identifier → DataSource name. + */ +const buildDataSourceLocationIndex = ( + dataSources: any[], +): Map => { + const index = new Map(); + dataSources?.forEach((ds: any) => { + const name = ds.spec?.name || ds.name; + if (!name) return; + extractDataSourceIdentifiers(ds.spec || ds).forEach((id) => { + index.set(id, name); + }); + }); + return index; +}; + const parseEntityRelationships = (objects: feast.core.Registry) => { const links: EntityRelation[] = []; @@ -195,6 +251,73 @@ const parseEntityRelationships = (objects: feast.core.Registry) => { } }); + // Build data source location index for storage-based matching + const allDataSources = [ + ...((objects as any).dataSources || []), + ...(objects.featureViews || []) + .map((fv: any) => fv.spec?.batchSource) + .filter(Boolean), + ...(objects.streamFeatureViews || []) + .flatMap((sfv: any) => [sfv.spec?.batchSource, sfv.spec?.streamSource]) + .filter(Boolean), + ]; + const dsLocationIndex = buildDataSourceLocationIndex(allDataSources); + + (objects as any).savedDatasets?.forEach((sd: any) => { + if (sd.spec?.featureServiceName) { + links.push({ + source: { + type: FEAST_FCO_TYPES["featureService"], + name: sd.spec.featureServiceName, + }, + target: { + type: FEAST_FCO_TYPES["savedDataset"], + name: sd.spec?.name!, + }, + }); + } + + // FeatureView -> SavedDataset (derived from feature refs "view:feat") + const seenViews = new Set(); + sd.spec?.features?.forEach((featRef: string) => { + const parts = featRef.split(":"); + const viewName = parts.length >= 2 ? parts[0] : featRef; + if (viewName && !seenViews.has(viewName)) { + seenViews.add(viewName); + links.push({ + source: { + type: FEAST_FCO_TYPES["featureView"], + name: viewName, + }, + target: { + type: FEAST_FCO_TYPES["savedDataset"], + name: sd.spec?.name!, + }, + }); + } + }); + + // DataSource -> SavedDataset (matched by storage location) + const storageIds = extractStorageIdentifiers(sd.spec?.storage); + const matchedDsNames = new Set(); + storageIds.forEach((locId) => { + const dsName = dsLocationIndex.get(locId); + if (dsName && !matchedDsNames.has(dsName)) { + matchedDsNames.add(dsName); + links.push({ + source: { + type: FEAST_FCO_TYPES["dataSource"], + name: dsName, + }, + target: { + type: FEAST_FCO_TYPES["savedDataset"], + name: sd.spec?.name!, + }, + }); + } + }); + }); + return links; }; diff --git a/ui/src/parsers/types.ts b/ui/src/parsers/types.ts index fc9f88d045f..de6a6be9e99 100644 --- a/ui/src/parsers/types.ts +++ b/ui/src/parsers/types.ts @@ -4,6 +4,7 @@ enum FEAST_FCO_TYPES { featureView = "featureView", featureService = "featureService", labelView = "labelView", + savedDataset = "savedDataset", mlflowRun = "mlflowRun", mlflowModel = "mlflowModel", openlineageJob = "openlineageJob", diff --git a/ui/src/queries/useLoadOpenLineageGraph.ts b/ui/src/queries/useLoadOpenLineageGraph.ts index f8154a405a6..ec6350e8763 100644 --- a/ui/src/queries/useLoadOpenLineageGraph.ts +++ b/ui/src/queries/useLoadOpenLineageGraph.ts @@ -42,6 +42,7 @@ export interface OpenLineageGraphData { nodes: OpenLineageNode[]; edges: OpenLineageEdge[]; symlinks?: OpenLineageSymlink[]; + total_nodes?: number; } export interface OpenLineageEvent { @@ -68,16 +69,40 @@ export interface RegistryLineageData { indirect_relationships: RegistryRelationship[]; } -const useLoadOpenLineageGraph = () => { +const useLoadOpenLineageGraph = (options?: { + namespace?: string; + limit?: number; + offset?: number; +}) => { const registryUrl = useContext(RegistryPathContext); const { fetchOptions } = useDataMode(); + const params = new URLSearchParams(); + if (options?.namespace) params.set("namespace", options.namespace); + if (options?.limit) params.set("limit", options.limit.toString()); + if (options?.offset) params.set("offset", options.offset.toString()); + const qs = params.toString(); + const path = qs + ? `/lineage/openlineage/graph?${qs}` + : "/lineage/openlineage/graph"; + return useQuery( - ["openlineage-graph"], + ["openlineage-graph", options?.namespace, options?.limit, options?.offset], + () => restFetch(registryUrl, path, fetchOptions), + { enabled: !!registryUrl }, + ); +}; + +const useLoadNamespaces = () => { + const registryUrl = useContext(RegistryPathContext); + const { fetchOptions } = useDataMode(); + + return useQuery<{ namespaces: string[] }>( + ["openlineage-namespaces"], () => - restFetch( + restFetch<{ namespaces: string[] }>( registryUrl, - "/lineage/openlineage/graph", + "/lineage/openlineage/namespaces", fetchOptions, ), { enabled: !!registryUrl }, @@ -157,4 +182,5 @@ export { useLoadOpenLineageEvents, useLoadOpenLineageJobs, useLoadRegistryLineage, + useLoadNamespaces, }; From 3ad789504cc5199219b7ecb7bda4aba486d33a44 Mon Sep 17 00:00:00 2001 From: ntkathole Date: Mon, 10 Aug 2026 10:06:13 +0530 Subject: [PATCH 3/3] fix: Fixed OpenLineage actions via apis and apply Signed-off-by: ntkathole --- docs/reference/openlineage.md | 663 ++++++++++++------ .../feast/api/registry/rest/__init__.py | 19 + sdk/python/feast/feature_store.py | 11 +- sdk/python/feast/lineage/registry_lineage.py | 79 +-- sdk/python/feast/openlineage/config.py | 20 +- sdk/python/feast/openlineage/consumer.py | 22 +- sdk/python/feast/openlineage/emitter.py | 38 +- sdk/python/feast/openlineage/facets.py | 6 + sdk/python/feast/openlineage/mappers.py | 9 + sdk/python/feast/openlineage/processor.py | 31 +- sdk/python/feast/openlineage/store.py | 98 ++- sdk/python/feast/registry_server.py | 108 ++- sdk/python/feast/repo_config.py | 4 +- .../tests/unit/openlineage/test_consumer.py | 28 +- ui/src/components/OpenLineageGraph.tsx | 511 ++++++++++++-- ui/src/components/RegistryVisualization.tsx | 95 ++- .../components/RegistryVisualizationTab.tsx | 3 + ui/src/pages/lineage/Index.tsx | 20 +- ui/src/parsers/parseEntityRelationships.ts | 13 +- 19 files changed, 1418 insertions(+), 360 deletions(-) diff --git a/docs/reference/openlineage.md b/docs/reference/openlineage.md index bf9e18750ed..6d28ee26bfa 100644 --- a/docs/reference/openlineage.md +++ b/docs/reference/openlineage.md @@ -1,54 +1,75 @@ # OpenLineage Integration -This module provides **native integration** between Feast and [OpenLineage](https://openlineage.io/), enabling automatic data lineage tracking for ML feature engineering workflows. +Feast provides **native integration** with [OpenLineage](https://openlineage.io/), enabling automatic data lineage tracking for ML feature engineering workflows. Feast can act as both a **producer** (emitting lineage events) and a **consumer** (receiving and displaying lineage from any OpenLineage-compatible system). -## Overview - -When enabled, the integration **automatically** emits OpenLineage events for: +## Quick Start -- **Registry changes** - Events when feature views, feature services, and entities are applied -- **Feature materialization** - START, COMPLETE, and FAIL events when features are materialized +### 1. Install -**No code changes required** - just enable OpenLineage in your `feature_store.yaml`! +```bash +pip install feast[openlineage] +# or: pip install openlineage-python +``` -## Installation +### 2. Configure -OpenLineage is an optional dependency. Install it with: +```yaml +# feature_store.yaml +project: my_project +registry: + registry_type: sql + path: sqlite:///data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db -```bash -pip install openlineage-python +openlineage: + enabled: true + transport_type: console # or http, file, kafka + namespace: my_project + consumer: + enabled: true + store_type: sql ``` -Or install Feast with the OpenLineage extra: +### 3. Apply and View ```bash -pip install feast[openlineage] +feast apply # emits lineage events automatically +feast ui # starts the UI with lineage visualization ``` -## Configuration +Open http://localhost:8888 and navigate to the **Lineage** tab. You will see the full lineage graph — both the Feast registry view and the OpenLineage view. + +## Overview + +When enabled, the integration **automatically** emits OpenLineage events for: + +- **Registry changes** — events when feature views, on-demand feature views, feature services, entities, data sources, and saved datasets are applied +- **Feature materialization** — START, COMPLETE, and FAIL events when features are materialized + +**No code changes required** — just enable OpenLineage in your `feature_store.yaml`. + +## Prerequisites + +- **SQL registry required for consumer**: The OpenLineage consumer stores lineage data in SQL tables. If you enable the consumer, your Feast registry must use `registry_type: sql` (SQLite, PostgreSQL, MySQL). File-based registries are not supported for the consumer. The producer works with any registry type. + +## Producer Configuration Add the `openlineage` section to your `feature_store.yaml`: ```yaml -project: my_project -registry: data/registry.db -provider: local -online_store: - type: sqlite - path: data/online_store.db - openlineage: enabled: true transport_type: http transport_url: http://localhost:5000 transport_endpoint: api/v1/lineage - namespace: feast + namespace: my_project emit_on_apply: true emit_on_materialize: true ``` -Once configured, all Feast operations will automatically emit lineage events. - ### Environment Variables You can also configure via environment variables: @@ -58,9 +79,29 @@ export FEAST_OPENLINEAGE_ENABLED=true export FEAST_OPENLINEAGE_TRANSPORT_TYPE=http export FEAST_OPENLINEAGE_URL=http://localhost:5000 export FEAST_OPENLINEAGE_ENDPOINT=api/v1/lineage -export FEAST_OPENLINEAGE_NAMESPACE=feast +export FEAST_OPENLINEAGE_NAMESPACE=my_project ``` +### Configuration Options + +| Option | Default | Description | +|--------|---------|-------------| +| `enabled` | `false` | Enable/disable OpenLineage integration | +| `transport_type` | `None` | Transport type: `http`, `console`, `file`, `kafka`. When unset, defers to OpenLineage SDK defaults | +| `transport_url` | — | Base URL for HTTP transport (required when `transport_type` is `http`) | +| `transport_endpoint` | `api/v1/lineage` | API endpoint appended to `transport_url` for HTTP transport | +| `api_key` | — | Optional API key for authentication with the lineage server | +| `namespace` | `feast` | Namespace for lineage events. When set to `feast` (default), the Feast project name is used | +| `producer` | `feast` | Producer identifier included in every OpenLineage event | +| `emit_on_apply` | `true` | Emit lineage events when `feast apply` is called | +| `emit_on_materialize` | `true` | Emit lineage events during materialization | +| `additional_config` | `{}` | Extra transport-specific settings (e.g., `log_file_path` for file transport, `bootstrap_servers` for Kafka) | + +### Namespace Behavior + +- If `namespace` is `"feast"` (default): uses the project name as the namespace (e.g., `my_project`) +- If `namespace` is set to a custom value: uses `{namespace}/{project}` (e.g., `custom/my_project`) + ## Usage Once configured, lineage is tracked automatically: @@ -69,7 +110,6 @@ Once configured, lineage is tracked automatically: from feast import FeatureStore from datetime import datetime, timedelta -# Create FeatureStore - OpenLineage is initialized automatically if configured fs = FeatureStore(repo_path="feature_repo") # Apply operations emit lineage events automatically @@ -80,44 +120,188 @@ fs.materialize( start_date=datetime.now() - timedelta(days=1), end_date=datetime.now() ) - ``` -## Configuration Options - -| Option | Default | Description | -|--------|---------|-------------| -| `enabled` | `false` | Enable/disable OpenLineage integration | -| `transport_type` | `None` | Transport type: `http`, `console`, `file`, `kafka`. When unset, defers to OpenLineage SDK defaults. | -| `transport_url` | - | URL for HTTP transport (required) | -| `transport_endpoint` | `api/v1/lineage` | API endpoint for HTTP transport | -| `api_key` | - | Optional API key for authentication | -| `namespace` | `feast` | Namespace for lineage events (uses project name if set to "feast") | -| `producer` | `feast` | Producer identifier | -| `emit_on_apply` | `true` | Emit events on `feast apply` | -| `emit_on_materialize` | `true` | Emit events on materialization | - ## Lineage Graph Structure -When you run `feast apply`, Feast creates a lineage graph that matches the Feast UI: +When you run `feast apply`, Feast creates lineage events reflecting the full dependency graph: ``` -DataSources ──┐ - ├──→ feast_feature_views_{project} ──→ FeatureViews -Entities ─────┘ │ - │ - ▼ - feature_service_{name} ──→ FeatureService +DataSource ──────────┐ + ├──→ FeatureView ──────────────┐ +Entity ──────────────┘ │ │ + │ ├──→ FeatureService + ▼ │ +RequestSource ──→ OnDemandFeatureView ──────────────┘ + │ +FeatureView ─────────────────┘ (as input source) + +FeatureService ──→ SavedDataset +DataSource ──────→ SavedDataset (via storage matching) ``` -**Jobs created:** -- `feast_feature_views_{project}`: Shows DataSources + Entities → FeatureViews -- `feature_service_{name}`: Shows specific FeatureViews → FeatureService (one per service) +**Jobs created per `feast apply`:** + +| Job | Inputs | Outputs | +|-----|--------|---------| +| `feast_apply_entities` | — | Entity datasets | +| `feast_apply_data_sources` | — | DataSource datasets | +| `feast_apply_feature_view_{name}` | DataSource + Entity | FeatureView dataset | +| `feast_apply_odfv_{name}` | FeatureView + RequestSource | OnDemandFeatureView dataset | +| `feast_apply_feature_service_{name}` | FeatureView(s) + ODFV(s) | FeatureService dataset | +| `feast_apply_saved_dataset_{name}` | FeatureService + DataSource | SavedDataset dataset | **Datasets include:** -- Schema with feature names, types, descriptions, and tags -- Feast-specific facets with metadata (TTL, entities, owner, etc.) -- Documentation facets with descriptions + +- OpenLineage `SchemaDatasetFacet` with feature names, types, and descriptions +- Feast-specific facets with rich metadata (TTL, entities, owner, tags, etc.) + +## Feast to OpenLineage Mapping + +| Feast Concept | OpenLineage Concept | Facet | +|---------------|---------------------|-------| +| DataSource | InputDataset | `FeastDataSourceFacet` | +| Entity | InputDataset | `FeastEntityFacet` | +| FeatureView | OutputDataset (of FV job) / InputDataset (of FS or ODFV job) | `FeastFeatureViewFacet` | +| OnDemandFeatureView | OutputDataset | `FeastFeatureViewFacet` (with `mode: ON_DEMAND`) | +| StreamFeatureView | OutputDataset | `FeastFeatureViewFacet` (with `mode: STREAM`) | +| FeatureService | OutputDataset | `FeastFeatureServiceFacet` | +| SavedDataset | OutputDataset | `FeastSavedDatasetFacet` | +| Feature | Schema field in `SchemaDatasetFacet` | — | +| Materialization | RunEvent (START/COMPLETE/FAIL) | `FeastMaterializationFacet` | +| Online Store (per FV) | OutputDataset (materialization target) | `FeastOnlineStoreFacet` | + +## Custom Feast Facets + +The integration includes custom OpenLineage facets that carry Feast-specific metadata: + +### FeastFeatureViewFacet + +Captures metadata about feature views (regular, on-demand, and stream): + +| Field | Description | +|-------|-------------| +| `name` | Feature view name | +| `ttl_seconds` | Time-to-live in seconds (0 = no TTL) | +| `entities` | List of entity names | +| `features` | List of feature names | +| `online_enabled` / `offline_enabled` | Store configuration | +| `mode` | Transformation mode: `ON_DEMAND`, `STREAM`, `PYTHON`, `PANDAS`, etc. | +| `description` | Human-readable description | +| `owner` | Owner identifier | +| `tags` | Key-value tags | + +### FeastFeatureServiceFacet + +Captures metadata about feature services: + +| Field | Description | +|-------|-------------| +| `name` | Feature service name | +| `feature_views` | List of feature view names | +| `feature_count` | Total number of features | +| `description` | Description | +| `owner` | Owner identifier | +| `tags` | Key-value tags | +| `logging_enabled` | Whether feature logging is enabled | + +### FeastDataSourceFacet + +Captures metadata about data sources: + +| Field | Description | +|-------|-------------| +| `name` | Data source name | +| `source_type` | Type: `FileSource`, `BigQuerySource`, `SnowflakeSource`, `RequestSource`, etc. | +| `timestamp_field` | Event timestamp column name | +| `created_timestamp_field` | Created timestamp column name | +| `field_mapping` | Source-to-feature field mapping | +| `description` | Description | +| `tags` | Key-value tags | + +### FeastEntityFacet + +Captures metadata about entities (join keys for feature lookups): + +| Field | Description | +|-------|-------------| +| `name` | Entity name | +| `join_keys` | List of join key column names | +| `value_type` | Data type (INT64, STRING, etc.) | +| `description` | Description | +| `owner` | Owner identifier | +| `tags` | Key-value tags | + +### FeastSavedDatasetFacet + +Captures metadata about saved datasets (materialized feature snapshots): + +| Field | Description | +|-------|-------------| +| `name` | Saved dataset name | +| `features` | List of feature names | +| `join_keys` | List of join key column names | +| `feature_service_name` | Name of the FeatureService that produced this dataset | +| `full_feature_names` | Whether full feature names were used | +| `description` | Description | +| `tags` | Key-value tags | + +### FeastMaterializationFacet + +Captures materialization run metadata (attached to RunEvents): + +| Field | Description | +|-------|-------------| +| `feature_views` | Feature views being materialized | +| `start_date` / `end_date` | Materialization time window | +| `project` | Feast project name | +| `rows_written` | Number of rows written | +| `online_store_type` | Online store backend type | +| `offline_store_type` | Offline store backend type | + +### FeastOnlineStoreFacet + +Identifies the online store sink during materialization: + +| Field | Description | +|-------|-------------| +| `feature_view` | Feature view whose features are stored | +| `store_type` | Online store backend (redis, sqlite, dynamodb, etc.) | +| `description` | Description | + +### FeastProjectFacet + +Captures Feast project context on job events: + +| Field | Description | +|-------|-------------| +| `project_name` | Feast project name | +| `provider` | Infrastructure provider (local, gcp, aws) | +| `online_store_type` | Online store type | +| `offline_store_type` | Offline store type | +| `registry_type` | Registry type (file, sql) | + +### FeastJobKindFacet + +Distinguishes Feast jobs by semantic role: + +| Field | Description | +|-------|-------------| +| `kind` | `definition` (registry/apply events) or `transform` (runtime materialize/compute) | +| `feast_project` | Feast project name | + +### FeastRetrievalFacet + +Captures feature retrieval metadata: + +| Field | Description | +|-------|-------------| +| `retrieval_type` | `online` or `historical` | +| `feature_service` | Feature service name (if used) | +| `feature_views` | Feature views queried | +| `features` | Features retrieved | +| `entity_count` | Number of entities queried | +| `full_feature_names` | Whether full feature names were used | ## Transport Types @@ -129,9 +313,19 @@ openlineage: transport_type: http transport_url: http://marquez:5000 transport_endpoint: api/v1/lineage - api_key: your-api-key # Optional + api_key: your-api-key ``` +### Console Transport (Development) + +```yaml +openlineage: + enabled: true + transport_type: console +``` + +Events are printed to stdout — useful for debugging. + ### File Transport ```yaml @@ -153,81 +347,36 @@ openlineage: topic: openlineage.events ``` -## Custom Feast Facets - -The integration includes custom Feast-specific facets in lineage events: - -### FeastFeatureViewFacet - -Captures metadata about feature views: -- `name`: Feature view name -- `ttl_seconds`: Time-to-live in seconds -- `entities`: List of entity names -- `features`: List of feature names -- `online_enabled` / `offline_enabled`: Store configuration -- `description`: Feature view description -- `tags`: Key-value tags - -### FeastFeatureServiceFacet - -Captures metadata about feature services: -- `name`: Feature service name -- `feature_views`: List of feature view names -- `feature_count`: Total number of features -- `description`: Feature service description -- `tags`: Key-value tags - -### FeastMaterializationFacet - -Captures materialization run metadata: -- `feature_views`: Feature views being materialized -- `start_date` / `end_date`: Materialization window -- `rows_written`: Number of rows written - ## Lineage Visualization -### Option 1: Feast UI (Built-in) +### Option 1: Feast UI (Built-in Consumer) -Feast includes a built-in OpenLineage consumer that can receive, store, and visualize lineage from **all** OpenLineage producers (Airflow, Spark, dbt, Feast itself, etc.) directly in the Feast UI. See the [OpenLineage Consumer](#openlineage-consumer) section below. +Feast includes a built-in OpenLineage consumer that receives, stores, and visualizes lineage from **all** OpenLineage producers directly in the Feast UI. See the [OpenLineage Consumer](#openlineage-consumer) section below. ### Option 2: Marquez Use [Marquez](https://marquezproject.ai/) to visualize your Feast lineage: ```bash -# Start Marquez docker run -p 5000:5000 -p 3000:3000 marquezproject/marquez - -# Configure Feast to emit to Marquez (in feature_store.yaml) -# openlineage: -# enabled: true -# transport_type: http -# transport_url: http://localhost:5000 ``` -Then access the Marquez UI at http://localhost:3000 to see your feature lineage. - -## Namespace Behavior +Configure Feast to emit to Marquez: -- If `namespace` is set to `"feast"` (default): Uses project name as namespace (e.g., `my_project`) -- If `namespace` is set to a custom value: Uses `{namespace}/{project}` (e.g., `custom/my_project`) - -## Feast to OpenLineage Mapping +```yaml +openlineage: + enabled: true + transport_type: http + transport_url: http://localhost:5000 +``` -| Feast Concept | OpenLineage Concept | -|---------------|---------------------| -| DataSource | InputDataset | -| FeatureView | OutputDataset (of feature views job) / InputDataset (of feature service job) | -| Feature | Schema field | -| Entity | InputDataset | -| FeatureService | OutputDataset | -| Materialization | RunEvent (START/COMPLETE/FAIL) | +Access the Marquez UI at http://localhost:3000. --- ## OpenLineage Consumer -Feast can act as an **OpenLineage consumer**, receiving lineage events from any OpenLineage-compatible producer and displaying them in the Feast UI. This eliminates the need for a separate Marquez deployment when you want to visualize cross-system data lineage alongside your feature store. +Feast can act as an **OpenLineage consumer**, receiving lineage events from any OpenLineage-compatible producer and displaying them in the Feast UI. This eliminates the need for a separate Marquez deployment. ### Consumer Architecture @@ -251,7 +400,7 @@ Producers (Airflow, Spark, dbt, Feast, Flink, …) └──────────────────────────┘ ``` -When the consumer is **not** enabled, the Feast UI shows only the original registry-based lineage view — no tabs are added. +When the consumer is **not** enabled, the Feast UI shows only the original registry-based lineage view. ### Enabling the Consumer @@ -260,8 +409,8 @@ Add the `consumer` section under `openlineage` in your `feature_store.yaml`: ```yaml project: my_project registry: - registry_type: sql - path: postgresql://user:****@host:5432/feast # pragma: allowlist secret + registry_type: sql # Required for consumer + path: postgresql://user:****@host:5432/feast # pragma: allowlist secret openlineage: enabled: true @@ -272,10 +421,10 @@ openlineage: # Optional: separate database for lineage storage. # If omitted, the SQL registry database is reused. # connection_string: postgresql://user:****@host:5432/feast_lineage - api_key: "change-me" # pragma: allowlist secret + api_key: "change-me" # pragma: allowlist secret namespace_mapping: - airflow_ns: my_project - spark_ns: my_project + "spark://ml-team": "my_project" + "airflow://prod-cluster": "my_project" ``` Or via environment variables: @@ -283,9 +432,11 @@ Or via environment variables: ```bash export FEAST_OPENLINEAGE_CONSUMER_ENABLED=true export FEAST_OPENLINEAGE_CONSUMER_STORE_TYPE=sql -export FEAST_OPENLINEAGE_CONSUMER_API_KEY=change-me # pragma: allowlist secret +export FEAST_OPENLINEAGE_CONSUMER_API_KEY=change-me # pragma: allowlist secret # Optional separate DB: # export FEAST_OPENLINEAGE_CONSUMER_CONNECTION_STRING=postgresql://... +# Namespace mapping (JSON format): +export FEAST_OPENLINEAGE_CONSUMER_NAMESPACE_MAPPING='{"spark://ml-team": "my_project", "airflow://prod-cluster": "my_project"}' ``` ### Consumer Configuration Options @@ -294,69 +445,95 @@ export FEAST_OPENLINEAGE_CONSUMER_API_KEY=change-me # pragma: allowlist secret |--------|---------|-------------| | `consumer.enabled` | `false` | Enable the OpenLineage consumer | | `consumer.store_type` | `sql` | Storage backend type. Currently only `sql` is supported | -| `consumer.connection_string` | - | Optional separate database connection string. If omitted, reuses the SQL registry database | -| `consumer.api_key` | - | API key that producers must provide when sending events | -| `consumer.namespace_mapping` | `{}` | Maps OpenLineage namespaces to Feast projects for RBAC scoping | +| `consumer.connection_string` | — | Optional separate database connection string. If omitted, reuses the SQL registry database | +| `consumer.api_key` | — | API key that producers must provide when sending events | +| `consumer.namespace_mapping` | `{}` | Maps external OpenLineage namespaces to Feast project names for RBAC scoping (see [Namespace Mapping](#namespace-mapping)) | + +### Running the Server + +The `feast ui` command starts a single server that handles everything: + +- Serves the React UI with lineage visualization +- Exposes the OpenLineage consumer endpoints (both ingestion and query) +- Reads from the Feast registry + +```bash +feast ui --port 8888 +``` + +When both producer and consumer are enabled, Feast's own events (from `feast apply`, materialization) are **automatically ingested** into the local consumer store via an in-process wiring — no HTTP transport configuration is needed for self-reporting. + +```yaml +# Minimal config for producer + consumer (self-contained) +openlineage: + enabled: true + transport_type: console # still prints to stdout for debugging + namespace: my_project + consumer: + enabled: true +``` ### Consumer API Endpoints -When the consumer is enabled, the following endpoints are available on the Feast REST registry server: +When the consumer is enabled, the following endpoints are available. All paths shown are relative to the server mount point (e.g., `/api/v1` on the UI server). -#### Event Receiver (Producer-facing) +#### Event Ingestion (Producer-facing) | Endpoint | Method | Description | |----------|--------|-------------| -| `/api/v1/lineage` | `POST` | Receive a single OpenLineage event (or array of events) | -| `/api/v1/lineage/batch` | `POST` | Receive a batch of OpenLineage events | +| `/api/v1/lineage` | `POST` | Receive a single OpenLineage event (or array of events). Returns `201` for single events, `200` for batch | +| `/api/v1/lineage/batch` | `POST` | Receive a batch of OpenLineage events. Returns `204` on full success | -Both endpoints require the `X-API-Key` header (or `Authorization: Bearer `) if `consumer.api_key` is configured. +Both endpoints accept the `X-API-Key` header (or `Authorization: Bearer `) when `consumer.api_key` is configured. -#### Admin Endpoints +#### Lineage Query Endpoints (UI-facing) | Endpoint | Method | Description | |----------|--------|-------------| -| `/lineage/openlineage/reset` | `DELETE` | Purge all OpenLineage data. Accepts optional `?namespace=X` to delete only a specific namespace. Requires API key. | +| `/api/v1/lineage/openlineage/graph` | `GET` | Full lineage graph with nodes, edges, and symlinks. Supports `?namespace=X`, `?limit=N`, `?offset=N` | +| `/api/v1/lineage/openlineage/graph/{node_type}/{namespace}/{name}` | `GET` | Lineage subgraph centered on a specific node. Supports `?depth=N`, `?direction=both|upstream|downstream` | +| `/api/v1/lineage/openlineage/namespaces` | `GET` | List all distinct namespaces | +| `/api/v1/lineage/openlineage/events` | `GET` | Browse events with `?namespace=X`, `?job_name=Y`, `?limit=N`, `?offset=N` | +| `/api/v1/lineage/openlineage/jobs` | `GET` | List all known jobs | +| `/api/v1/lineage/openlineage/datasets` | `GET` | List all known datasets | +| `/api/v1/lineage/openlineage/runs` | `GET` | List runs with `?job_namespace=X&job_name=Y`, `?limit=N`, `?offset=N` | +| `/api/v1/lineage/openlineage/runs/{run_id}` | `GET` | Single run detail with input/output datasets | -#### OpenLineage Query Endpoints (UI-facing) +#### Registry Lineage Endpoints | Endpoint | Method | Description | |----------|--------|-------------| -| `/lineage/openlineage/graph` | `GET` | Full lineage graph with all nodes, edges, and symlinks | -| `/lineage/openlineage/graph/{node_type}/{namespace}/{name}` | `GET` | Lineage graph centered on a specific node | -| `/lineage/openlineage/events` | `GET` | Browse stored events with filtering | -| `/lineage/openlineage/jobs` | `GET` | List all known OpenLineage jobs | -| `/lineage/openlineage/datasets` | `GET` | List all known OpenLineage datasets | -| `/lineage/openlineage/runs` | `GET` | List runs with optional `?job_namespace=X&job_name=Y` filtering | -| `/lineage/openlineage/runs/{run_id}` | `GET` | Single run detail with input/output datasets | +| `/api/v1/lineage/registry` | `GET` | Feast registry lineage with `?project=X` | +| `/api/v1/lineage/registry/all` | `GET` | Registry lineage for all projects | +| `/api/v1/lineage/objects/{object_type}/{object_name}` | `GET` | Detail for a specific registry object | +| `/api/v1/lineage/complete` | `GET` | Complete registry lineage with full object metadata | +| `/api/v1/lineage/complete/all` | `GET` | Complete registry lineage for all projects | -#### Registry Query Endpoints +#### Admin Endpoints | Endpoint | Method | Description | |----------|--------|-------------| -| `/lineage/registry` | `GET` | Feast registry lineage (entities, feature views, services) | -| `/lineage/registry/all` | `GET` | All registry objects with full metadata | -| `/lineage/objects/{object_type}/{object_name}` | `GET` | Detail for a specific registry object | -| `/lineage/complete` | `GET` | Complete registry lineage with relationships | -| `/lineage/complete/all` | `GET` | Complete registry lineage for all objects | +| `/api/v1/lineage/openlineage/reset` | `DELETE` | Purge all OpenLineage data. Accepts `?namespace=X` to delete a specific namespace only. Requires API key | -### Configuring Producers to Send Events to Feast +### Configuring External Producers -Configure any OpenLineage producer to send events to your Feast instance: +Configure any OpenLineage producer to send events to Feast. The ingestion endpoint is `POST /api/v1/lineage`. #### Airflow ```python # In airflow.cfg or environment -OPENLINEAGE_URL = "http://feast-registry:8080/api" -OPENLINEAGE_API_KEY = "change-me" # pragma: allowlist secret +OPENLINEAGE_URL = "http://feast-server:8888" +OPENLINEAGE_ENDPOINT = "api/v1/lineage" +OPENLINEAGE_API_KEY = "change-me" # pragma: allowlist secret ``` #### Spark ```properties spark.openlineage.transport.type=http -spark.openlineage.transport.url=http://feast-registry:8080/api -spark.openlineage.transport.endpoint=/v1/lineage +spark.openlineage.transport.url=http://feast-server:8888 +spark.openlineage.transport.endpoint=api/v1/lineage spark.openlineage.transport.auth.type=api_key spark.openlineage.transport.auth.apiKey=change-me ``` @@ -365,114 +542,208 @@ spark.openlineage.transport.auth.apiKey=change-me ```yaml # In profiles.yml or environment -OPENLINEAGE_URL: "http://feast-registry:8080/api" -OPENLINEAGE_API_KEY: "change-me" # pragma: allowlist secret +OPENLINEAGE_URL: "http://feast-server:8888" +OPENLINEAGE_ENDPOINT: "api/v1/lineage" +OPENLINEAGE_API_KEY: "change-me" # pragma: allowlist secret ``` #### Feast (Self-reporting) -When both the OpenLineage producer and consumer are enabled, Feast's own events (from `feast apply`, materialization, etc.) are automatically ingested into the local consumer store — no HTTP transport is needed. +When both the OpenLineage producer and consumer are enabled in the same `feature_store.yaml`, Feast's own events (from `feast apply`, materialization) are automatically ingested into the local consumer store via an in-process wiring — no HTTP transport is needed. ```yaml -# In feature_store.yaml openlineage: enabled: true namespace: my_project consumer: enabled: true - api_key: change-me # pragma: allowlist secret + api_key: change-me # pragma: allowlist secret ``` ### Feast UI Lineage Views -When the consumer is enabled, the lineage page in the Feast UI shows two tabs: +When the consumer is enabled, the lineage page in the Feast UI provides two views: -**Lineage tab** +**OpenLineage Graph** (default when events exist) -- **OpenLineage Graph** (default) — shows lineage from all OpenLineage producers with cross-producer connectivity. Nodes are color-coded by producer (colors generated dynamically). The graph supports filtering by type, producer, and object name. Clicking a node opens a **detail panel** showing description, schema, tags, features, entities, data quality metrics, data source info, other facets, and **run history** (for job nodes — see [Per-Run Lineage](#per-run-lineage-run-history)). -- **Feast Only Lineage** (checkbox) — switches to the original Feast registry view (DataSource → FeatureView → FeatureService) powered entirely by the Feast registry. +- Shows lineage from all OpenLineage producers in a unified graph +- Nodes are color-coded by Feast object type (DataSource, Entity, FeatureView, FeatureService, etc.) +- Clicking a node opens a detail panel with description, schema, tags, features, entities, facets, and run history (for job nodes) +- Supports filtering by namespace -**Events tab** +**Feast Only Lineage** (checkbox toggle) -- Browse individual OpenLineage events with filtering by event type, job name, and run ID. Expand any event to inspect the full JSON payload. +- Shows the registry-based lineage view: DataSource → FeatureView → FeatureService, Entity → FeatureView, and OnDemandFeatureView relationships +- Powered entirely by the Feast registry — works independently of OpenLineage configuration +- When the consumer is enabled but has no events yet, this view is shown by default with the toggle visible to switch between views ### Cross-Producer Lineage Connectivity -The consumer automatically links datasets across different producers when they refer to the same physical data. Linking mechanisms: +The consumer automatically links datasets across different producers when they refer to the same physical data: -1. **Shared namespace + name** — If Airflow writes to `s3://bucket/path` and Spark reads from the same `s3://bucket/path`, the graph connects them automatically. -2. **SymlinksDatasetFacet** — Producers can declare aliases. For example, Feast can declare that its internal `driver_hourly_stats` is a symlink to the Spark output at `s3://bucket/features/driver_hourly_stats/`. -3. **dataSource URI matching** — Datasets with matching `dataSource.uri` facets are linked even if their namespace or name differ. - -Compatible producers include Airflow, Spark, dbt, Flink, Feast, and Dagster. +1. **Shared namespace + name** — if Airflow writes to `s3://bucket/path` and Spark reads from the same `s3://bucket/path`, the graph connects them +2. **SymlinksDatasetFacet** — producers can declare aliases (e.g., Feast declaring its `driver_hourly_stats` is a symlink to `s3://bucket/features/driver_hourly_stats/`) +3. **dataSource URI matching** — datasets with matching `dataSource.uri` facets are linked even if their namespace or name differ ### RBAC for Lineage -The OpenLineage consumer integrates with Feast's existing RBAC: +The OpenLineage consumer integrates with Feast's existing RBAC — no new permissions or +`AuthzedAction` values are introduced: -- **Write access** (producers sending events): Authenticated via API key in the `X-API-Key` header -- **Read access** (UI viewing lineage): Namespace-based filtering maps OpenLineage namespaces to Feast projects. Users see only lineage data for namespaces they have access to via the `namespace_mapping` configuration +- **Write access** (producers sending events): authenticated via API key in the `X-API-Key` header. Any authenticated producer can send events. +- **Read access** (UI viewing lineage): based on existing Feast project permissions. Users who can `DESCRIBE` a Feast project see lineage from that project's namespace **plus** any external namespaces mapped to it via `namespace_mapping`. -### Lineage Cleanup / Reset +### Namespace Mapping -Over time the OpenLineage store accumulates historical data. Two mechanisms are provided for cleanup: +The `consumer.namespace_mapping` configuration is a read-side RBAC bridge that maps +external OpenLineage namespaces to Feast project names. It controls **who can see what** +in the lineage UI and API — it does **not** rewrite, reroute, or alter ingested events. -#### Admin Reset Endpoint +Events are always stored exactly as the producer sent them, with their original namespace +intact. -Use the `DELETE /lineage/openlineage/reset` endpoint to purge lineage data. The endpoint requires the same API key used for event ingestion. +#### How It Works -```bash -# Purge ALL OpenLineage data -curl -X DELETE -H "X-API-Key: your-key" \ - http://localhost:8080/api/v1/lineage/openlineage/reset +When RBAC is enabled, each API query determines which namespaces the current user may +see: -# Purge only a specific namespace -curl -X DELETE -H "X-API-Key: your-key" \ - "http://localhost:8080/api/v1/lineage/openlineage/reset?namespace=airflow://prod-cluster" -``` +1. List all Feast projects the user can `DESCRIBE` (existing Feast RBAC). +2. For each allowed project, resolve its OpenLineage namespace. +3. Scan `namespace_mapping` — for each entry whose **value** (Feast project name) + matches an allowed project, add that entry's **key** (external namespace) to the + allowed set. +4. Filter all query results to only include data from the allowed namespaces. -A full purge deletes data from all seven `openlineage_*` tables. A namespace-scoped purge deletes jobs, datasets, runs, events, edges, and symlinks associated with that namespace, leaving other namespaces intact. +#### Configuration -#### Feast Teardown Hook +In `feature_store.yaml`: -When you run `feast teardown`, Feast automatically cleans up OpenLineage data for the project's namespace (if the consumer is configured). This ensures that tearing down a Feast project doesn't leave orphaned lineage data behind. +```yaml +openlineage: + enabled: true + namespace: feast + consumer: + enabled: true + api_key: "change-me" # pragma: allowlist secret + namespace_mapping: + "spark://ml-team": "ml_team" + "airflow://prod-cluster": "ml_team" + "ray://ml-team": "ml_team" +``` + +Or via environment variable (JSON format): ```bash -# Tears down the Feast project AND its OpenLineage lineage -feast teardown +export FEAST_OPENLINEAGE_CONSUMER_NAMESPACE_MAPPING='{"spark://ml-team": "ml_team", "airflow://prod-cluster": "ml_team"}' ``` -### Per-Run Lineage (Run History) +#### Cross-Producer Example + +``` + Spark Airflow Ray Feast + namespace: namespace: namespace: namespace: + spark://ml-team airflow://prod ray://ml-team ml_team + + │ │ │ │ + └──────────────────┼─────────────────────┘ │ + ▼ │ + POST /api/v1/lineage │ + │ │ + ▼ │ + ┌──────────────────────┐ (local wire) │ + │ Feast OL Consumer │ ◄─────────────────────────┘ + │ namespace_mapping: │ + │ spark://ml-team │ + │ → ml_team │ + │ airflow://prod │ + │ → ml_team │ + │ ray://ml-team │ + │ → ml_team │ + └──────────────────────┘ + │ + ▼ + Feast UI / API + (unified lineage view + filtered by RBAC) +``` + +A user who can `DESCRIBE` the `ml_team` Feast project sees lineage from **all four +producers** in a single unified graph. + +#### Namespace Resolution for Feast Object Mapping + +Beyond RBAC, `namespace_mapping` helps the event processor map incoming datasets +to Feast registry objects during ingest. When a dataset arrives with namespace +`spark://ml-team`, the processor resolves it to Feast project `ml_team` and can +match the dataset against known Feast objects in that project. + +Resolution priority: -The consumer tracks individual pipeline runs in the `openlineage_runs` table. When you click on a **job node** in the OpenLineage Graph, the detail panel shows a **Run History** section with: +1. **Exact match** — `namespace_mapping["spark://ml-team"]` → `"ml_team"` +2. **Authority/path match** — for `scheme://authority/path` namespaces, try the + authority+path portion +3. **Fallback** — use the last path segment as the project name -- A table of past runs: truncated run ID, status badge (COMPLETE, FAIL, RUNNING, ABORT), start time, and duration -- Click any run to expand its **inputs and outputs** — the specific datasets that run consumed and produced +#### When Namespace Mapping Is Not Needed -#### Run History API +- **Single-project setups**: if you only have one Feast project and no external + producers, the default behavior (namespace = project name) works without mapping. +- **Feast-only lineage**: the Feast Only Lineage view operates purely on registry + data and does not use `namespace_mapping`. +- **No RBAC**: when Feast RBAC is disabled, all namespaces are visible to all users. + Mapping is still used for Feast object resolution during ingest. + +### Per-Run Lineage (Run History) + +The consumer tracks individual pipeline runs. When you click on a **job node** in the OpenLineage Graph, the detail panel shows a **Run History** section with: + +- A table of past runs: run ID, status badge (COMPLETE, FAIL, RUNNING, ABORT), start time, and duration +- Click any run to see its specific **inputs and outputs** — the datasets that run consumed and produced ```bash # List runs for a specific job -curl "http://localhost:8080/api/v1/lineage/openlineage/runs?job_namespace=spark://emr-cluster&job_name=feature_engineering" +curl "http://localhost:8888/api/v1/lineage/openlineage/runs?job_namespace=spark://ml-team&job_name=feature_engineering" # Get a single run with its I/O datasets -curl "http://localhost:8080/api/v1/lineage/openlineage/runs/{run_id}" +curl "http://localhost:8888/api/v1/lineage/openlineage/runs/{run_id}" ``` -The run detail response includes `inputs` and `outputs` arrays, each containing the dataset namespace, name, and any I/O facets recorded by the producer. +### Lineage Cleanup / Reset + +#### Admin Reset Endpoint + +Use `DELETE /api/v1/lineage/openlineage/reset` to purge lineage data: + +```bash +# Purge ALL OpenLineage data +curl -X DELETE -H "X-API-Key: your-key" \ + http://localhost:8888/api/v1/lineage/openlineage/reset + +# Purge only a specific namespace +curl -X DELETE -H "X-API-Key: your-key" \ + "http://localhost:8888/api/v1/lineage/openlineage/reset?namespace=airflow://prod-cluster" +``` + +#### Feast Teardown Hook + +When you run `feast teardown`, Feast automatically cleans up OpenLineage data for the project's namespace (if the consumer is configured). + +```bash +feast teardown +``` ### Database Schema -The consumer creates the following tables (automatically on first startup): +The consumer creates the following tables automatically on first startup: | Table | Purpose | |-------|---------| | `openlineage_events` | Raw event storage with JSON payloads | | `openlineage_jobs` | Deduplicated job records with producer, description, and facets | -| `openlineage_datasets` | Deduplicated dataset records with schema, facets, and Feast mapping | +| `openlineage_datasets` | Deduplicated dataset records with schema, facets, and Feast object mapping | | `openlineage_runs` | Run lifecycle tracking (START/COMPLETE/FAIL) | | `openlineage_run_io` | Input/output relationships between runs and datasets | | `openlineage_lineage_edges` | Materialized lineage graph edges for efficient traversal | | `openlineage_dataset_symlinks` | Cross-producer dataset linking via `SymlinksDatasetFacet` and `dataSource` URI matching | -By default these tables are created in the **same database** as the SQL registry (hybrid storage). Set `consumer.connection_string` to store them in a separate database instead. +By default these tables are created in the **same database** as the SQL registry. Set `consumer.connection_string` to use a separate database. diff --git a/sdk/python/feast/api/registry/rest/__init__.py b/sdk/python/feast/api/registry/rest/__init__.py index 4aaf59635cd..ea0b2610292 100644 --- a/sdk/python/feast/api/registry/rest/__init__.py +++ b/sdk/python/feast/api/registry/rest/__init__.py @@ -159,6 +159,14 @@ def _build_allowed_namespaces_fn(fs): Uses ``permitted_resources`` with ``DESCRIBE`` on all known projects. The projects the current user may describe become the allowed OL namespaces (mapped through ``resolve_namespace``). + + External producer namespaces (e.g. ``spark://ml-team``, + ``airflow://prod-cluster``) are included when + ``consumer.namespace_mapping`` maps them to a Feast project the + user is allowed to DESCRIBE. This is the only purpose of + ``namespace_mapping`` — it is a **read-side RBAC bridge**, not a + routing or rewrite mechanism. Ingest stores events as-is; + producers own their namespace. """ def _get_allowed(): @@ -183,9 +191,20 @@ def _get_allowed(): ) ol_ns_config = getattr(ol_config, "namespace", "feast") + allowed_project_names = {p.name for p in allowed_projects} + namespaces = set() for p in allowed_projects: namespaces.add(resolve_namespace(ol_ns_config, p.name)) + + # Include external namespaces whose namespace_mapping + # target is a project the user can DESCRIBE. + consumer_cfg = getattr(ol_config, "consumer", None) + ns_map = getattr(consumer_cfg, "namespace_mapping", None) or {} + for ext_ns, mapped_project in ns_map.items(): + if mapped_project in allowed_project_names: + namespaces.add(ext_ns) + return list(namespaces) if namespaces else None except Exception: return None diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 4607f84d150..aab260e774b 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1907,9 +1907,18 @@ def _mlflow_log_apply( _logger.debug("MLflow apply logging failed: %s", e) def _emit_openlineage_apply(self, objects: List[Any]): - """Emit OpenLineage events for applied objects.""" + """Emit OpenLineage events for applied objects. + + Skips when using a remote registry — the RegistryServer already + emits OL events in its Apply* handlers, so emitting here would + double-count every object. + """ if self.openlineage_emitter is None: return + from feast.infra.registry.remote import RemoteRegistry + + if isinstance(self._registry, RemoteRegistry): + return try: self.openlineage_emitter.emit_apply(objects, self.project) except Exception as e: diff --git a/sdk/python/feast/lineage/registry_lineage.py b/sdk/python/feast/lineage/registry_lineage.py index c58fbde65a3..0f4c2ac8c83 100644 --- a/sdk/python/feast/lineage/registry_lineage.py +++ b/sdk/python/feast/lineage/registry_lineage.py @@ -314,59 +314,38 @@ def _parse_direct_relationships(self, registry: Registry) -> List[EntityRelation source_items = [(k, v) for k, v in enumerate(odfv.spec.sources)] for source_name, source in source_items: - if ( - hasattr(source, "request_data_source") - and source.request_data_source - ): - if hasattr(source.request_data_source, "name"): - relationships.append( - EntityRelation( - source=EntityReference( - FeastObjectType.DATA_SOURCE, - source.request_data_source.name, - ), - target=EntityReference( - FeastObjectType.FEATURE_VIEW, odfv.spec.name - ), - ) + has_req = hasattr(source, "HasField") and source.HasField( + "request_data_source" + ) + has_fvp = hasattr(source, "HasField") and source.HasField( + "feature_view_projection" + ) + + if has_req and source.request_data_source.name: + relationships.append( + EntityRelation( + source=EntityReference( + FeastObjectType.DATA_SOURCE, + source.request_data_source.name, + ), + target=EntityReference( + FeastObjectType.FEATURE_VIEW, odfv.spec.name + ), ) - elif ( - hasattr(source, "feature_view_projection") - and source.feature_view_projection - ): - # Find the source feature view's batch source - if hasattr(source.feature_view_projection, "feature_view_name"): - source_fv = next( - ( - fv - for fv in registry.feature_views - if hasattr(fv, "spec") - and fv.spec - and hasattr(fv.spec, "name") - and fv.spec.name - == source.feature_view_projection.feature_view_name + ) + elif has_fvp and source.feature_view_projection.feature_view_name: + relationships.append( + EntityRelation( + source=EntityReference( + FeastObjectType.FEATURE_VIEW, + source.feature_view_projection.feature_view_name, + ), + target=EntityReference( + FeastObjectType.FEATURE_VIEW, + odfv.spec.name, ), - None, ) - if ( - source_fv - and hasattr(source_fv, "spec") - and source_fv.spec - and hasattr(source_fv.spec, "batch_source") - and source_fv.spec.batch_source - and hasattr(source_fv.spec.batch_source, "name") - ): - relationships.append( - EntityRelation( - source=EntityReference( - FeastObjectType.DATA_SOURCE, - source_fv.spec.batch_source.name, - ), - target=EntityReference( - FeastObjectType.FEATURE_VIEW, odfv.spec.name - ), - ) - ) + ) # Stream FeatureView relationships for sfv in registry.stream_feature_views: diff --git a/sdk/python/feast/openlineage/config.py b/sdk/python/feast/openlineage/config.py index 9380df3b8cf..4bfaa67be67 100644 --- a/sdk/python/feast/openlineage/config.py +++ b/sdk/python/feast/openlineage/config.py @@ -16,6 +16,7 @@ Configuration classes for Feast OpenLineage integration. """ +import json import os from dataclasses import dataclass, field from typing import Any, Dict, Optional @@ -31,7 +32,12 @@ class OpenLineageConsumerConfig: store_type: Storage backend type ('sql' uses the SQL registry DB) connection_string: Optional separate DB connection string api_key: API key for authenticating producers sending events - namespace_mapping: Map of OL namespace -> Feast project for RBAC scoping + namespace_mapping: Read-side RBAC bridge mapping external OpenLineage + namespaces to Feast project names. When a user can DESCRIBE a Feast + project, they also see lineage from any external namespace mapped to + that project. Also used during ingest to resolve incoming datasets to + Feast registry objects. Example: + {"spark://ml-team": "ml_team", "airflow://prod-cluster": "ml_team"} """ enabled: bool = False @@ -139,16 +145,28 @@ def from_env(cls) -> "OpenLineageConfig": FEAST_OPENLINEAGE_API_KEY: API key for authentication FEAST_OPENLINEAGE_NAMESPACE: Default namespace (default: feast) FEAST_OPENLINEAGE_PRODUCER: Producer identifier + FEAST_OPENLINEAGE_CONSUMER_NAMESPACE_MAPPING: JSON object mapping external + OL namespaces to Feast project names for RBAC scoping. + Example: '{"spark://ml-team": "ml_team", "airflow://prod-cluster": "prod"}' Returns: OpenLineageConfig instance """ + ns_mapping_raw = os.getenv("FEAST_OPENLINEAGE_CONSUMER_NAMESPACE_MAPPING", "") + ns_mapping: Dict[str, str] = {} + if ns_mapping_raw: + try: + ns_mapping = json.loads(ns_mapping_raw) + except json.JSONDecodeError: + pass + consumer = OpenLineageConsumerConfig( enabled=os.getenv("FEAST_OPENLINEAGE_CONSUMER_ENABLED", "false").lower() == "true", store_type=os.getenv("FEAST_OPENLINEAGE_CONSUMER_STORE_TYPE", "sql"), connection_string=os.getenv("FEAST_OPENLINEAGE_CONSUMER_CONNECTION_STRING"), api_key=os.getenv("FEAST_OPENLINEAGE_CONSUMER_API_KEY"), + namespace_mapping=ns_mapping, ) return cls( diff --git a/sdk/python/feast/openlineage/consumer.py b/sdk/python/feast/openlineage/consumer.py index 6dfa7a0bf90..fc80c78a7ae 100644 --- a/sdk/python/feast/openlineage/consumer.py +++ b/sdk/python/feast/openlineage/consumer.py @@ -86,7 +86,7 @@ def get_consumer_router( # ── Producer-facing: receive events ── - @router.post("/v1/lineage") + @router.post("/lineage") async def receive_lineage_event( request: Request, x_api_key: Optional[str] = Header(None, alias="X-API-Key"), @@ -97,6 +97,13 @@ async def receive_lineage_event( Compatible with the standard OpenLineage API endpoint. Accepts RunEvent, DatasetEvent, or JobEvent. + + The router defines POST /lineage; the full path depends on + the server mount: + - UI server (mounted at /api/v1): POST /api/v1/lineage + - REST server (root_path=/api/v1): POST /lineage + When behind a reverse proxy, the proxy strips /api/v1, + so external clients use POST /api/v1/lineage in both cases. """ api_key = getattr(config, "consumer_api_key", None) if not _verify_api_key(api_key, x_api_key, authorization): @@ -130,7 +137,7 @@ async def receive_lineage_event( logger.error(f"Failed to process event: {e}") raise HTTPException(status_code=500, detail=str(e)) - @router.post("/v1/lineage/batch") + @router.post("/lineage/batch") async def receive_lineage_batch( request: Request, x_api_key: Optional[str] = Header(None, alias="X-API-Key"), @@ -202,6 +209,8 @@ def list_events( ): """List stored OpenLineage events with optional filtering.""" ns_filter = _get_namespace_filter(get_allowed_namespaces) + if namespace and ns_filter is not None and namespace not in ns_filter: + return {"events": [], "total": 0} events = store.get_events( namespace=namespace, job_name=job_name, @@ -362,11 +371,15 @@ def list_runs( offset: int = Query(0, ge=0), ): """List runs, optionally filtered by job namespace and name.""" + ns_filter = _get_namespace_filter(get_allowed_namespaces) + if job_namespace and ns_filter is not None and job_namespace not in ns_filter: + return {"runs": [], "total": 0} runs = store.get_runs( job_namespace=job_namespace, job_name=job_name, limit=limit, offset=offset, + namespaces=ns_filter if not job_namespace else None, ) return {"runs": runs, "total": len(runs)} @@ -376,6 +389,11 @@ def get_run_detail(run_id: str): run = store.get_run_detail(run_id) if not run: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") + ns_filter = _get_namespace_filter(get_allowed_namespaces) + if ns_filter is not None: + run_ns = run.get("job_namespace") + if run_ns and run_ns not in ns_filter: + raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return run def _get_namespace_filter(ns_callable) -> Optional[List[str]]: diff --git a/sdk/python/feast/openlineage/emitter.py b/sdk/python/feast/openlineage/emitter.py index b7a2c8c844c..a93eca7f23d 100644 --- a/sdk/python/feast/openlineage/emitter.py +++ b/sdk/python/feast/openlineage/emitter.py @@ -445,15 +445,29 @@ def emit_on_demand_feature_view_lineage( ) for source_name, req_source in odfv.source_request_sources.items(): + req_name = getattr(req_source, "name", source_name) inputs.append( InputDataset( namespace=namespace, - name=f"request_source_{source_name}", + name=req_name, ) ) - # Build output - output_facets = {} + # Build output with feast_featureView facet on the dataset + output_facets: Dict[str, Any] = { + "feast_featureView": FeastFeatureViewFacet( + name=odfv.name, + ttl_seconds=0, + entities=[], + features=[f.name for f in odfv.features] if odfv.features else [], + online_enabled=True, + offline_enabled=True, + mode="ON_DEMAND", + description=odfv.description if odfv.description else "", + owner=odfv.owner if hasattr(odfv, "owner") and odfv.owner else "", + tags=odfv.tags if odfv.tags else {}, + ), + } if odfv.features: output_facets["schema"] = schema_dataset.SchemaDatasetFacet( fields=[feast_field_to_schema_field(f) for f in odfv.features] @@ -467,20 +481,12 @@ def emit_on_demand_feature_view_lineage( ) ] - # Build job facets + from feast.openlineage.facets import FeastProjectFacet + from feast.openlineage.identity import FeastJobKind + job_facets = { - "feast_featureView": FeastFeatureViewFacet( - name=odfv.name, - ttl_seconds=0, - entities=[], - features=[f.name for f in odfv.features] if odfv.features else [], - online_enabled=True, - offline_enabled=True, - mode="ON_DEMAND", - description=odfv.description if odfv.description else "", - owner=odfv.owner if hasattr(odfv, "owner") and odfv.owner else "", - tags=odfv.tags if odfv.tags else {}, - ) + "feast_project": FeastProjectFacet(project_name=project), + **self._job_kind_facets(FeastJobKind.DEFINITION, project), } # Emit a RunEvent with COMPLETE state to create lineage connection diff --git a/sdk/python/feast/openlineage/facets.py b/sdk/python/feast/openlineage/facets.py index 22842cf5677..14ee0240fbe 100644 --- a/sdk/python/feast/openlineage/facets.py +++ b/sdk/python/feast/openlineage/facets.py @@ -147,6 +147,9 @@ class FeastDataSourceFacet(DatasetFacet): timestamp_field: Name of the timestamp field created_timestamp_field: Name of the created timestamp field field_mapping: Mapping from source fields to feature names + path: File path (for file-based sources) + table: Table name (for database sources) + query: SQL query (for query-based sources) description: Human-readable description tags: Key-value tags """ @@ -156,6 +159,9 @@ class FeastDataSourceFacet(DatasetFacet): timestamp_field: Optional[str] = attr.field(default=None) created_timestamp_field: Optional[str] = attr.field(default=None) field_mapping: Dict[str, str] = attr.field(factory=dict) + path: Optional[str] = attr.field(default=None) + table: Optional[str] = attr.field(default=None) + query: Optional[str] = attr.field(default=None) description: str = attr.field(default="") tags: Dict[str, str] = attr.field(factory=dict) diff --git a/sdk/python/feast/openlineage/mappers.py b/sdk/python/feast/openlineage/mappers.py index 38243b2f579..23cda027bca 100644 --- a/sdk/python/feast/openlineage/mappers.py +++ b/sdk/python/feast/openlineage/mappers.py @@ -139,6 +139,15 @@ def data_source_to_dataset( field_mapping=data_source.field_mapping if hasattr(data_source, "field_mapping") else {}, + path=data_source.path + if hasattr(data_source, "path") and data_source.path + else None, + table=data_source.table + if hasattr(data_source, "table") and data_source.table + else None, + query=data_source.query + if hasattr(data_source, "query") and data_source.query + else None, description=data_source.description if hasattr(data_source, "description") else "", diff --git a/sdk/python/feast/openlineage/processor.py b/sdk/python/feast/openlineage/processor.py index 592ecdfcabf..359e0a0cc00 100644 --- a/sdk/python/feast/openlineage/processor.py +++ b/sdk/python/feast/openlineage/processor.py @@ -387,16 +387,37 @@ def _resolve_feast_mapping( """ facets = facets or {} - # Namespace may be project or prefix/project; mapping keys are usually - # the logical OL namespace (e.g. customer_churn). + # Resolve OL namespace to a Feast project name. + # + # Priority: + # 1. Exact match in namespace_mapping (e.g. "spark://ml-team") + # 2. Authority/path match for scheme-prefixed namespaces + # (e.g. "ml-team" from "spark://ml-team") + # 3. Last path segment for path-style namespaces + # (e.g. "customer_churn" from "org/customer_churn") + # 4. Fallback: use the namespace as-is or its last segment feast_project = self._namespace_mapping.get(namespace) - if feast_project is None and "/" in namespace: + if feast_project is None and "://" in namespace: + authority_path = namespace.split("://", 1)[1] + for candidate in (authority_path, authority_path.split("/")[-1]): + if candidate in self._namespace_mapping: + feast_project = self._namespace_mapping[candidate] + break + if feast_project is None: + feast_project = ( + authority_path.split("/")[-1] + if "/" in authority_path + else authority_path + ) + elif feast_project is None and "/" in namespace: for part in (namespace.split("/")[-1], namespace.split("/")[0]): if part in self._namespace_mapping: feast_project = self._namespace_mapping[part] break - if feast_project is None: - feast_project = namespace.split("/")[-1] if "/" in namespace else namespace + if feast_project is None: + feast_project = namespace.split("/")[-1] + elif feast_project is None: + feast_project = namespace facet_type_map = ( ("feast_onlineStore", "onlineStore"), diff --git a/sdk/python/feast/openlineage/store.py b/sdk/python/feast/openlineage/store.py index e21b7f69b45..465d01ceb47 100644 --- a/sdk/python/feast/openlineage/store.py +++ b/sdk/python/feast/openlineage/store.py @@ -691,6 +691,99 @@ def get_all_symlinks(self) -> List[Dict[str, Any]]: # ── Cleanup methods ── + def delete_dataset(self, namespace: str, name: str): + """Delete a specific dataset and its related edges, runs, and jobs.""" + with self._engine.begin() as conn: + tbl_edges = OL_TABLES["lineage_edges"] + conn.execute( + tbl_edges.delete().where( + ( + (tbl_edges.c.source_namespace == namespace) + & (tbl_edges.c.source_name == name) + ) + | ( + (tbl_edges.c.target_namespace == namespace) + & (tbl_edges.c.target_name == name) + ) + ) + ) + + tbl_sym = OL_TABLES["dataset_symlinks"] + conn.execute( + tbl_sym.delete().where( + ( + (tbl_sym.c.dataset_namespace == namespace) + & (tbl_sym.c.dataset_name == name) + ) + | ( + (tbl_sym.c.linked_namespace == namespace) + & (tbl_sym.c.linked_name == name) + ) + ) + ) + + tbl_rio = OL_TABLES["run_io"] + conn.execute( + tbl_rio.delete().where( + (tbl_rio.c.dataset_namespace == namespace) + & (tbl_rio.c.dataset_name == name) + ) + ) + + tbl_ds = OL_TABLES["datasets"] + conn.execute( + tbl_ds.delete().where( + (tbl_ds.c.dataset_namespace == namespace) + & (tbl_ds.c.dataset_name == name) + ) + ) + + logger.info(f"Deleted OL dataset: {namespace}/{name}") + + def delete_job(self, namespace: str, name: str): + """Delete a specific job and its related runs, events, and edges.""" + with self._engine.begin() as conn: + tbl_runs = OL_TABLES["runs"] + run_ids_q = select(tbl_runs.c.run_id).where( + (tbl_runs.c.job_namespace == namespace) & (tbl_runs.c.job_name == name) + ) + run_ids = [r[0] for r in conn.execute(run_ids_q).fetchall()] + if run_ids: + tbl_rio = OL_TABLES["run_io"] + conn.execute(tbl_rio.delete().where(tbl_rio.c.run_id.in_(run_ids))) + conn.execute(tbl_runs.delete().where(tbl_runs.c.run_id.in_(run_ids))) + + tbl_ev = OL_TABLES["events"] + conn.execute( + tbl_ev.delete().where( + (tbl_ev.c.job_namespace == namespace) & (tbl_ev.c.job_name == name) + ) + ) + + tbl_edges = OL_TABLES["lineage_edges"] + conn.execute( + tbl_edges.delete().where( + ( + (tbl_edges.c.source_namespace == namespace) + & (tbl_edges.c.source_name == name) + ) + | ( + (tbl_edges.c.target_namespace == namespace) + & (tbl_edges.c.target_name == name) + ) + ) + ) + + tbl_jobs = OL_TABLES["jobs"] + conn.execute( + tbl_jobs.delete().where( + (tbl_jobs.c.job_namespace == namespace) + & (tbl_jobs.c.job_name == name) + ) + ) + + logger.info(f"Deleted OL job: {namespace}/{name}") + def purge_all(self): """Delete all data from all OpenLineage tables.""" table_order = [ @@ -755,8 +848,9 @@ def get_runs( job_name: Optional[str] = None, limit: int = 50, offset: int = 0, + namespaces: Optional[List[str]] = None, ) -> List[Dict[str, Any]]: - """Get runs, optionally filtered by job.""" + """Get runs, optionally filtered by job and/or RBAC-allowed namespaces.""" tbl = OL_TABLES["runs"] query = ( select(tbl).order_by(tbl.c.updated_at.desc()).limit(limit).offset(offset) @@ -765,6 +859,8 @@ def get_runs( query = query.where(tbl.c.job_namespace == job_namespace) if job_name: query = query.where(tbl.c.job_name == job_name) + if namespaces is not None: + query = query.where(tbl.c.job_namespace.in_(namespaces)) with self._engine.connect() as conn: rows = conn.execute(query).fetchall() return [dict(row._mapping) for row in rows] diff --git a/sdk/python/feast/registry_server.py b/sdk/python/feast/registry_server.py index 46c822a88cc..eb49e1392a9 100644 --- a/sdk/python/feast/registry_server.py +++ b/sdk/python/feast/registry_server.py @@ -182,6 +182,82 @@ def __init__(self, registry: BaseRegistry, store=None) -> None: self.proxied_registry = registry self.store = store + _JOB_NAME_TEMPLATES: dict = { + "featureView": ["feast_apply_feature_view_{name}"], + "featureService": [ + "feature_service_{name}", + "feast_apply_feature_service_{name}", + ], + "savedDataset": ["saved_dataset_{name}"], + "onDemandFeatureView": ["feast_apply_odfv_{name}"], + } + + @property + def _openlineage_enabled(self) -> bool: + """Fast cached check: is OpenLineage configured and enabled? + + Evaluated once per RegistryServer lifetime so Apply/Delete handlers + pay zero cost when OL is disabled. + """ + cached = getattr(self, "_ol_enabled_cache", None) + if cached is not None: + return cached + enabled = False + try: + if self.store: + ol_cfg = getattr(self.store.config, "openlineage", None) + if ol_cfg is not None and getattr(ol_cfg, "enabled", False): + enabled = True + except Exception: + pass + self._ol_enabled_cache = enabled + return enabled + + def _emit_openlineage_for_objects(self, objects: list, project: str): + """Emit OpenLineage events for objects modified via API/gRPC. + + Skips entirely when OL is disabled — no imports, no emitter access. + All errors are caught so registry operations never fail due to OL. + """ + if not self._openlineage_enabled or not objects: + return + try: + emitter = self.store.openlineage_emitter + if emitter is not None: + emitter.emit_apply(objects, project) + except Exception as e: + logger.warning(f"Failed to emit OpenLineage events for API apply: {e}") + + def _delete_openlineage_for_object(self, name: str, project: str, object_type: str): + """Remove OpenLineage data for a deleted Feast object. + + Skips entirely when OL consumer is not active. + All errors are caught so registry deletes never fail due to OL. + """ + if not self._openlineage_enabled: + return + try: + import feast.api.registry.rest as rest_module + + ol_store = getattr(rest_module, "_ol_store_instance", None) + if ol_store is None: + return + + ol_config = getattr(rest_module, "_ol_config", None) + namespace = ( + ol_config.namespace if ol_config and ol_config.namespace else project + ) + + ol_store.delete_dataset(namespace, name) + + for tpl in self._JOB_NAME_TEMPLATES.get(object_type, []): + ol_store.delete_job(namespace, tpl.format(name=name)) + + except Exception as e: + logger.warning( + f"Failed to delete OpenLineage data for {object_type}/{name}: {e}" + ) + def Proto(self, request: Empty, context) -> RegistryProto: """Build a RegistryProto from individually RBAC-filtered list calls. @@ -288,6 +364,7 @@ def ApplyEntity(self, request: RegistryServer_pb2.ApplyEntityRequest, context): project=request.project, commit=request.commit, ) + self._emit_openlineage_for_objects([entity], request.project) return Empty() @@ -334,6 +411,7 @@ def DeleteEntity(self, request: RegistryServer_pb2.DeleteEntityRequest, context) self.proxied_registry.delete_entity( name=request.name, project=request.project, commit=request.commit ) + self._delete_openlineage_for_object(request.name, request.project, "entity") return Empty() def ApplyDataSource( @@ -352,6 +430,7 @@ def ApplyDataSource( project=request.project, commit=request.commit, ) + self._emit_openlineage_for_objects([data_source], request.project) return Empty() @@ -405,6 +484,7 @@ def DeleteDataSource( self.proxied_registry.delete_data_source( name=request.name, project=request.project, commit=request.commit ) + self._delete_openlineage_for_object(request.name, request.project, "dataSource") return Empty() def GetFeatureView( @@ -497,13 +577,12 @@ def ApplyFeatureView( else: raise ValueError(f"Unexpected feature view type: {feature_view_type}") - ( - self.proxied_registry.apply_feature_view( - feature_view=feature_view, - project=request.project, - commit=request.commit, - ), + self.proxied_registry.apply_feature_view( + feature_view=feature_view, + project=request.project, + commit=request.commit, ) + self._emit_openlineage_for_objects([feature_view], request.project) return Empty() @@ -689,6 +768,14 @@ def DeleteFeatureView( name=request.name, project=request.project, allow_cache=False ) + from feast.on_demand_feature_view import OnDemandFeatureView + + fv_type = ( + "onDemandFeatureView" + if isinstance(feature_view, OnDemandFeatureView) + else "featureView" + ) + assert_permissions( resource=cast(FeastObject, feature_view), actions=[AuthzedAction.DELETE], @@ -696,6 +783,7 @@ def DeleteFeatureView( self.proxied_registry.delete_feature_view( name=request.name, project=request.project, commit=request.commit ) + self._delete_openlineage_for_object(request.name, request.project, fv_type) return Empty() def GetStreamFeatureView( @@ -832,6 +920,7 @@ def ApplyFeatureService( project=request.project, commit=request.commit, ) + self._emit_openlineage_for_objects([feature_service], request.project) return Empty() @@ -904,6 +993,9 @@ def DeleteFeatureService( self.proxied_registry.delete_feature_service( name=request.name, project=request.project, commit=request.commit ) + self._delete_openlineage_for_object( + request.name, request.project, "featureService" + ) return Empty() def ApplySavedDataset( @@ -922,6 +1014,7 @@ def ApplySavedDataset( project=request.project, commit=request.commit, ) + self._emit_openlineage_for_objects([saved_dataset], request.project) return Empty() @@ -980,6 +1073,9 @@ def DeleteSavedDataset( self.proxied_registry.delete_saved_dataset( name=request.name, project=request.project, commit=request.commit ) + self._delete_openlineage_for_object( + request.name, request.project, "savedDataset" + ) return Empty() diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 775a62aab57..a3ac01e6c1c 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -272,7 +272,9 @@ class OpenLineageConsumerConfig(FeastBaseModel): """ str: API key for authenticating producers sending events. """ namespace_mapping: Optional[Dict[str, str]] = None - """ dict: Map of OL namespace -> Feast project for RBAC scoping. """ + """ dict: Read-side RBAC bridge mapping external OL namespaces to Feast project + names. Users who can DESCRIBE a project also see lineage from mapped namespaces. + Example: {"spark://ml-team": "ml_team", "airflow://prod-cluster": "ml_team"} """ class OpenLineageConfig(FeastBaseModel): diff --git a/sdk/python/tests/unit/openlineage/test_consumer.py b/sdk/python/tests/unit/openlineage/test_consumer.py index 1666063dcbf..f111917a493 100644 --- a/sdk/python/tests/unit/openlineage/test_consumer.py +++ b/sdk/python/tests/unit/openlineage/test_consumer.py @@ -111,7 +111,7 @@ class TestEventIngestion: def test_single_event_returns_201(self, app_no_key): app, _ = app_no_key client = TestClient(app) - resp = client.post("/api/v1/v1/lineage", json=_make_run_event()) + resp = client.post("/api/v1/lineage", json=_make_run_event()) assert resp.status_code == 201 assert "event_id" in resp.json() @@ -119,7 +119,7 @@ def test_array_of_events_returns_200(self, app_no_key): app, _ = app_no_key client = TestClient(app) events = [_make_run_event(run_id=f"r{i}") for i in range(3)] - resp = client.post("/api/v1/v1/lineage", json=events) + resp = client.post("/api/v1/lineage", json=events) assert resp.status_code == 200 assert resp.json()["summary"]["received"] == 3 assert resp.json()["summary"]["successful"] == 3 @@ -128,26 +128,26 @@ def test_batch_endpoint(self, app_no_key): app, _ = app_no_key client = TestClient(app) events = [_make_run_event(run_id=f"r{i}") for i in range(2)] - resp = client.post("/api/v1/v1/lineage/batch", json=events) + resp = client.post("/api/v1/lineage/batch", json=events) assert resp.status_code == 204 def test_batch_rejects_non_array(self, app_no_key): app, _ = app_no_key client = TestClient(app) - resp = client.post("/api/v1/v1/lineage/batch", json={"not": "array"}) + resp = client.post("/api/v1/lineage/batch", json={"not": "array"}) assert resp.status_code == 400 def test_api_key_required(self, app_with_key): app, _ = app_with_key client = TestClient(app) - resp = client.post("/api/v1/v1/lineage", json=_make_run_event()) + resp = client.post("/api/v1/lineage", json=_make_run_event()) assert resp.status_code == 401 def test_api_key_via_header(self, app_with_key): app, _ = app_with_key client = TestClient(app) resp = client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(), headers={"X-API-Key": "secret-key"}, ) @@ -157,7 +157,7 @@ def test_api_key_via_bearer(self, app_with_key): app, _ = app_with_key client = TestClient(app) resp = client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(), headers={"Authorization": "Bearer secret-key"}, ) @@ -167,7 +167,7 @@ def test_wrong_api_key_rejected(self, app_with_key): app, _ = app_with_key client = TestClient(app) resp = client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(), headers={"X-API-Key": "wrong-key"}, ) @@ -180,7 +180,7 @@ def test_wrong_api_key_rejected(self, app_with_key): class TestQueryEndpoints: def _ingest(self, client, events): for evt in events: - client.post("/api/v1/v1/lineage", json=evt) + client.post("/api/v1/lineage", json=evt) def test_get_events(self, app_no_key): app, _ = app_no_key @@ -292,7 +292,7 @@ def test_reset_full_purge(self, app_with_key): app, store = app_with_key client = TestClient(app) client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(), headers={"X-API-Key": "secret-key"}, ) @@ -308,12 +308,12 @@ def test_reset_by_namespace(self, app_with_key): app, store = app_with_key client = TestClient(app) client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(job_ns="ns-keep", run_id="r1"), headers={"X-API-Key": "secret-key"}, ) client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event(job_ns="ns-delete", run_id="r2"), headers={"X-API-Key": "secret-key"}, ) @@ -338,7 +338,7 @@ def test_reset_no_key_when_not_required(self, app_no_key): class TestRunsEndpoints: def _seed(self, client): client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event( job_ns="ns-a", job_name="j1", @@ -348,7 +348,7 @@ def _seed(self, client): ), ) client.post( - "/api/v1/v1/lineage", + "/api/v1/lineage", json=_make_run_event( job_ns="ns-b", job_name="j2", diff --git a/ui/src/components/OpenLineageGraph.tsx b/ui/src/components/OpenLineageGraph.tsx index 2636e3e66ad..343540c40e2 100644 --- a/ui/src/components/OpenLineageGraph.tsx +++ b/ui/src/components/OpenLineageGraph.tsx @@ -1,4 +1,10 @@ -import React, { useEffect, useMemo, useState } from "react"; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { ReactFlow, Node, @@ -285,6 +291,17 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => { const producerLabel = displayProducer(data.producer); const isJob = data.type === "job"; + const nodeRef = data.nodeRef; + const feastObjType = nodeRef?.feast_object_type; + const feastDsFacet = nodeRef?.facets?.feast_dataSource; + const subtitle = isJob + ? "click for runs" + : feastObjType === "dataSource" && feastDsFacet?.source_type + ? feastDsFacet.source_type + : feastObjType && feastObjType !== "unknown" + ? feastObjType + : undefined; + const handleClick = () => { if (data.onNodeClick && data.nodeRef) { data.onNodeClick(data.nodeRef); @@ -386,9 +403,9 @@ const LineageCustomNode = ({ data }: { data: LineageNodeData }) => { > {data.label}
- {isJob && ( + {subtitle && (
- click for runs + {subtitle}
)} @@ -406,6 +423,40 @@ const lineageNodeTypes = { lineageCustom: LineageCustomNode }; // ── Dagre layout ── +/** + * Derive a rank key from a ReactFlow node's data so that nodes of the + * same semantic type are placed on the same horizontal column (LR) or + * vertical row (TB). + * + * Priority: feast_object_type > OL type (job / dataset). + * The returned key is used to group nodes into dagre rank-groups. + */ +const rankKeyForNode = (node: Node): string => { + const ref = (node.data as LineageNodeData | undefined)?.nodeRef; + const feastType = ref?.feast_object_type; + if (feastType && feastType !== "unknown") return `feast:${feastType}`; + const olType = (node.data as LineageNodeData | undefined)?.type; + return olType === "job" ? "ol:job" : "ol:dataset"; +}; + +/** + * Ordered rank tiers – earlier entries are placed further left (LR) or + * further up (TB). Keys not listed here are appended automatically so + * external / unknown types still get a stable position. + */ +const RANK_ORDER: string[] = [ + "feast:dataSource", + "feast:entity", + "ol:dataset", + "feast:featureView", + "feast:onDemandFeatureView", + "feast:streamFeatureView", + "ol:job", + "feast:featureService", + "feast:savedDataset", + "feast:onlineStore", +]; + const layoutGraph = (nodes: Node[], edges: Edge[], direction = "LR") => { const dagreGraph = new dagre.graphlib.Graph(); dagreGraph.setDefaultEdgeLabel(() => ({})); @@ -417,7 +468,12 @@ const layoutGraph = (nodes: Node[], edges: Edge[], direction = "LR") => { marginy: 50, }); + // Group nodes by rank key so we can assign same-rank constraints. + const rankGroups = new Map(); nodes.forEach((node) => { + const key = rankKeyForNode(node); + if (!rankGroups.has(key)) rankGroups.set(key, []); + rankGroups.get(key)!.push(node.id); dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight }); }); @@ -427,8 +483,58 @@ const layoutGraph = (nodes: Node[], edges: Edge[], direction = "LR") => { } }); + // Assign explicit rank to each group via invisible chain edges so dagre + // aligns every node of the same type on the same column. + // We pick one representative node per group and chain them in the + // canonical order, then mark all other nodes in a group as same-rank + // by adding zero-weight edges to the representative. + const orderedKeys = [...RANK_ORDER]; + Array.from(rankGroups.keys()).forEach((key) => { + if (!orderedKeys.includes(key)) orderedKeys.push(key); + }); + const activeKeys = orderedKeys.filter((k) => rankGroups.has(k)); + + // Chain representatives to enforce ordering between groups + for (let i = 0; i < activeKeys.length - 1; i++) { + const repA = rankGroups.get(activeKeys[i])![0]; + const repB = rankGroups.get(activeKeys[i + 1])![0]; + dagreGraph.setEdge(repA, repB, { + minlen: 1, + weight: 0, + style: "invis", + _rank_edge: true, + }); + } + + // Pull same-group nodes to the same rank as their representative + Array.from(rankGroups.values()).forEach((ids) => { + if (ids.length <= 1) return; + const rep = ids[0]; + for (let i = 1; i < ids.length; i++) { + dagreGraph.setEdge(rep, ids[i], { + minlen: 0, + weight: 2, + style: "invis", + _rank_edge: true, + }); + dagreGraph.setEdge(ids[i], rep, { + minlen: 0, + weight: 2, + style: "invis", + _rank_edge: true, + }); + } + }); + dagre.layout(dagreGraph); + // Collect the invisible rank-edge ids so we can strip them from output + const rankEdgeSet = new Set(); + dagreGraph.edges().forEach((e) => { + const label = dagreGraph.edge(e); + if (label?._rank_edge) rankEdgeSet.add(`${e.v}->${e.w}`); + }); + return { nodes: nodes.map((node) => { const pos = dagreGraph.node(node.id); @@ -704,6 +810,16 @@ const NodeDetailPanel: React.FC<{ const fields = schema?.fields || []; const facets = node.facets || {}; + const featureViews: string[] = + facets.feast_featureService?.feature_views || []; + const dqMetrics = facets.dataQualityMetrics; + const sqlFacet = facets.sql; + const dataSource = facets.dataSource; + const feastDataSource = facets.feast_dataSource; + const ownership = facets.ownership; + const onlineStore = facets.feast_onlineStore; + const savedDataset = facets.feast_savedDataset; + const tags: string[] = []; if (facets.feast_featureView?.tags) { Object.entries(facets.feast_featureView.tags).forEach(([k, v]) => @@ -715,23 +831,28 @@ const NodeDetailPanel: React.FC<{ tags.push(`${k}:${v}`), ); } + if (feastDataSource?.tags) { + Object.entries(feastDataSource.tags).forEach(([k, v]) => + tags.push(`${k}:${v}`), + ); + } + if (savedDataset?.tags) { + Object.entries(savedDataset.tags).forEach(([k, v]) => + tags.push(`${k}:${v}`), + ); + } - const features: string[] = facets.feast_featureView?.features || []; + const features: string[] = + facets.feast_featureView?.features || savedDataset?.features || []; const entities: string[] = facets.feast_featureView?.entities || []; const fvDescription = node.description || facets.documentation?.description || facets.feast_featureView?.description || facets.feast_featureService?.description || - facets.feast_entity?.description; - - const featureViews: string[] = - facets.feast_featureService?.feature_views || []; - const dqMetrics = facets.dataQualityMetrics; - const sqlFacet = facets.sql; - const dataSource = facets.dataSource; - const ownership = facets.ownership; - const onlineStore = facets.feast_onlineStore; + facets.feast_entity?.description || + feastDataSource?.description || + savedDataset?.description; const knownFacetKeys = new Set([ "schema", @@ -740,6 +861,7 @@ const NodeDetailPanel: React.FC<{ "feast_featureService", "feast_entity", "feast_dataSource", + "feast_savedDataset", "feast_onlineStore", "dataQualityMetrics", "sql", @@ -965,23 +1087,159 @@ const NodeDetailPanel: React.FC<{ )} - {dataSource && ( + {(dataSource || feastDataSource) && (
Data Source
- {dataSource.name && ( + {(feastDataSource?.name || dataSource?.name) && (
- Name: {dataSource.name} + Name:{" "} + {feastDataSource?.name || dataSource?.name}
)} - {dataSource.uri && ( -
- {dataSource.uri} + {feastDataSource?.source_type && ( +
+ Type:{" "} + {feastDataSource.source_type} +
+ )} + {feastDataSource?.timestamp_field && ( +
+ Timestamp Field:{" "} + {feastDataSource.timestamp_field} +
+ )} + {feastDataSource?.created_timestamp_field && ( +
+ Created Timestamp:{" "} + {feastDataSource.created_timestamp_field} +
+ )} + {feastDataSource?.path && ( +
+ Path:{" "} + + {feastDataSource.path} + +
+ )} + {feastDataSource?.table && ( +
+ Table:{" "} + {feastDataSource.table} +
+ )} + {feastDataSource?.query && ( +
+ Query:{" "} +
+                {feastDataSource.query}
+              
+
+ )} + {dataSource?.uri && ( +
+ URI:{" "} + + {dataSource.uri} + +
+ )} + {feastDataSource?.field_mapping && + Object.keys(feastDataSource.field_mapping).length > 0 && ( +
+ Field Mapping: + + + + + + + + + {Object.entries(feastDataSource.field_mapping).map( + ([src, dst]) => ( + + + + + ), + )} + +
+ Source + + Feature +
+ {src} + + {String(dst)} +
+
+ )} +
+ )} + + {savedDataset && ( +
+
Saved Dataset
+ {savedDataset.feature_service_name && ( +
+ Feature Service:{" "} + {savedDataset.feature_service_name} +
+ )} + {savedDataset.join_keys?.length > 0 && ( +
+ Join Keys:{" "} + {savedDataset.join_keys.join(", ")}
)}
@@ -1110,11 +1368,52 @@ const LineageGraph: React.FC = ({ const [edges, setEdges, onEdgesChange] = useEdgesState([]); const [filterType, setFilterType] = useState(""); + const [filterFeastType, setFilterFeastType] = useState(""); const [filterProducer, setFilterProducer] = useState(""); const [filterObject, setFilterObject] = useState(""); const [selectedNode, setSelectedNode] = useState( null, ); + const [hoveredNodeId, setHoveredNodeId] = useState(null); + const edgesRef = useRef([]); + + const connectedIds = useMemo(() => { + if (!hoveredNodeId) return null; + const ids = new Set([hoveredNodeId]); + const allEdges = edgesRef.current; + + // Walk upstream (target → source) + const upQueue = [hoveredNodeId]; + while (upQueue.length > 0) { + const cur = upQueue.shift()!; + for (const e of allEdges) { + if (e.target === cur && !ids.has(e.source)) { + ids.add(e.source); + upQueue.push(e.source); + } + } + } + + // Walk downstream (source → target) + const downQueue = [hoveredNodeId]; + while (downQueue.length > 0) { + const cur = downQueue.shift()!; + for (const e of allEdges) { + if (e.source === cur && !ids.has(e.target)) { + ids.add(e.target); + downQueue.push(e.target); + } + } + } + + return ids; + }, [hoveredNodeId]); + + const onNodeMouseEnter = useCallback( + (_: React.MouseEvent, node: Node) => setHoveredNodeId(node.id), + [], + ); + const onNodeMouseLeave = useCallback(() => setHoveredNodeId(null), []); const objectsOnly = viewMode === "objects"; const showTypeFilter = viewMode !== "objects"; @@ -1147,10 +1446,33 @@ const LineageGraph: React.FC = ({ return Array.from(set).sort(); }, [baseNodes]); + const feastObjectTypes = useMemo(() => { + const set = new Set(); + for (const n of baseNodes) { + if (n.feast_object_type && n.feast_object_type !== "unknown") { + set.add(n.feast_object_type); + } + } + return Array.from(set).sort(); + }, [baseNodes]); + + const feastTypeLabels: Record = { + dataSource: "Data Source", + featureView: "Feature View", + featureService: "Feature Service", + entity: "Entity", + onDemandFeatureView: "On-Demand Feature View", + streamFeatureView: "Stream Feature View", + savedDataset: "Saved Dataset", + onlineStore: "Online Store", + }; + const objectOptions = useMemo(() => { return baseNodes .filter((n) => { if (filterType && n.type !== filterType) return false; + if (filterFeastType && n.feast_object_type !== filterFeastType) + return false; if (filterProducer && displayProducer(n.producer) !== filterProducer) return false; return true; @@ -1158,11 +1480,11 @@ const LineageGraph: React.FC = ({ .map((n) => n.name) .filter((v, i, a) => a.indexOf(v) === i) .sort(); - }, [baseNodes, filterType, filterProducer]); + }, [baseNodes, filterType, filterFeastType, filterProducer]); useEffect(() => { setFilterObject(""); - }, [filterType, filterProducer]); + }, [filterType, filterFeastType, filterProducer]); useEffect(() => { if (objectsOnly && filterType === "job") { @@ -1177,6 +1499,11 @@ const LineageGraph: React.FC = ({ if (filterType) { filteredNodes = filteredNodes.filter((n) => n.type === filterType); } + if (filterFeastType) { + filteredNodes = filteredNodes.filter( + (n) => n.feast_object_type === filterFeastType, + ); + } if (filterProducer) { filteredNodes = filteredNodes.filter( (n) => displayProducer(n.producer) === filterProducer, @@ -1193,17 +1520,52 @@ const LineageGraph: React.FC = ({ .map((n) => makeId(n.type, n.namespace, n.name)), ); - const connectedIds = new Set(); - for (const e of baseEdges) { - const srcId = makeId(e.source_type, e.source_namespace, e.source_name); - const tgtId = makeId(e.target_type, e.target_namespace, e.target_name); - if (focusIds.has(srcId)) connectedIds.add(tgtId); - if (focusIds.has(tgtId)) connectedIds.add(srcId); + const visibleIds = new Set(focusIds); + + // Walk upstream: follow edges where target matches current → add source + const upQueue = Array.from(focusIds); + while (upQueue.length > 0) { + const current = upQueue.shift()!; + for (const e of baseEdges) { + const srcId = makeId( + e.source_type, + e.source_namespace, + e.source_name, + ); + const tgtId = makeId( + e.target_type, + e.target_namespace, + e.target_name, + ); + if (tgtId === current && !visibleIds.has(srcId)) { + visibleIds.add(srcId); + upQueue.push(srcId); + } + } + } + + // Walk downstream: follow edges where source matches current → add target + const downQueue = Array.from(focusIds); + while (downQueue.length > 0) { + const current = downQueue.shift()!; + for (const e of baseEdges) { + const srcId = makeId( + e.source_type, + e.source_namespace, + e.source_name, + ); + const tgtId = makeId( + e.target_type, + e.target_namespace, + e.target_name, + ); + if (srcId === current && !visibleIds.has(tgtId)) { + visibleIds.add(tgtId); + downQueue.push(tgtId); + } + } } - const visibleIds = new Set( - Array.from(focusIds).concat(Array.from(connectedIds)), - ); filteredNodes = baseNodes.filter((n) => visibleIds.has(makeId(n.type, n.namespace, n.name)), ); @@ -1276,6 +1638,7 @@ const LineageGraph: React.FC = ({ }); const { nodes: ln, edges: le } = layoutGraph(flowNodes, flowEdges); + edgesRef.current = le; setNodes(ln); setEdges(le); }, [ @@ -1283,12 +1646,39 @@ const LineageGraph: React.FC = ({ baseNodes, baseEdges, filterType, + filterFeastType, filterProducer, filterObject, setNodes, setEdges, ]); + // Apply hover-dim: fade unrelated nodes and edges + const styledNodes = useMemo(() => { + if (!connectedIds) return nodes; + return nodes.map((n) => ({ + ...n, + style: { + ...n.style, + opacity: connectedIds.has(n.id) ? 1 : 0.15, + transition: "opacity 0.2s", + }, + })); + }, [nodes, connectedIds]); + + const styledEdges = useMemo(() => { + if (!connectedIds) return edges; + return edges.map((e) => ({ + ...e, + style: { + ...e.style, + opacity: + connectedIds.has(e.source) && connectedIds.has(e.target) ? 1 : 0.08, + transition: "opacity 0.2s", + }, + })); + }, [edges, connectedIds]); + if (olLoading) { return ( @@ -1320,6 +1710,17 @@ const LineageGraph: React.FC = ({ if (baseNodes.length === 0) { return ( + {feastOnlyCheckbox && ( +
+ {feastOnlyCheckbox} +
+ )} = ({

{objectsOnly ? "No dataset lineage edges have been recorded yet. Materialize features or emit OpenLineage events that include datasets." - : "No OpenLineage events yet. Apply features and run materialization so datasets, jobs, and runs appear here."} + : "No OpenLineage events yet. Apply features and run materialization so datasets, jobs, and runs appear here. Toggle 'Feast Only Lineage' above to view registry-based lineage."}

} /> @@ -1380,10 +1781,10 @@ const LineageGraph: React.FC = ({ )}
- + {showTypeFilter && ( - - + + = ({ ]} value={filterType} onChange={(e) => setFilterType(e.target.value)} - aria-label="Filter by type" + aria-label="Filter by OL type" + /> + + + )} + {feastObjectTypes.length > 0 && ( + + + ({ + value: t, + text: feastTypeLabels[t] || t, + })), + ]} + value={filterFeastType} + onChange={(e) => setFilterFeastType(e.target.value)} + aria-label="Filter by Feast type" /> )} - + = ({ /> - + = ({
= ({ fitView minZoom={0.1} maxZoom={8} + onNodeMouseEnter={onNodeMouseEnter} + onNodeMouseLeave={onNodeMouseLeave} onPaneClick={() => setSelectedNode(null)} > diff --git a/ui/src/components/RegistryVisualization.tsx b/ui/src/components/RegistryVisualization.tsx index 45d32dcacd9..34e93539b9c 100644 --- a/ui/src/components/RegistryVisualization.tsx +++ b/ui/src/components/RegistryVisualization.tsx @@ -1,4 +1,10 @@ -import React, { useEffect, useState } from "react"; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { useNavigate, useParams } from "react-router-dom"; import { ReactFlow, @@ -681,7 +687,7 @@ const registryToFlow = ( objects.onDemandFeatureViews?.forEach((odfv) => { const odfvName = odfv.spec?.name; nodes.push({ - id: `odfv-${odfvName}`, + id: `fv-${odfvName}`, type: "custom", data: { label: odfvName, @@ -704,7 +710,7 @@ const registryToFlow = ( objects.streamFeatureViews?.forEach((sfv) => { const sfvName = sfv.spec?.name; nodes.push({ - id: `sfv-${sfvName}`, + id: `fv-${sfvName}`, type: "custom", data: { label: sfvName, @@ -801,6 +807,16 @@ const registryToFlow = ( } }); + objects.onDemandFeatureViews?.forEach((odfv: any) => { + if (odfv.spec?.sources) { + Object.values(odfv.spec.sources).forEach((input: any) => { + if (input.requestDataSource?.name) { + dataSources.add(input.requestDataSource.name); + } + }); + } + }); + (objects as any).labelViews?.forEach((lv: any) => { if (lv.spec?.source?.name) { dataSources.add(lv.spec.source.name); @@ -1014,6 +1030,47 @@ const RegistryVisualization: React.FC = ({ const [showIsolatedNodes, setShowIsolatedNodes] = useState(false); const direction = "LR"; + const [hoveredNodeId, setHoveredNodeId] = useState(null); + const edgesRef = useRef([]); + + const connectedIds = useMemo(() => { + if (!hoveredNodeId) return null; + const ids = new Set([hoveredNodeId]); + const allEdges = edgesRef.current; + + // Walk upstream (target → source) + const upQueue = [hoveredNodeId]; + while (upQueue.length > 0) { + const cur = upQueue.shift()!; + for (const e of allEdges) { + if (e.target === cur && !ids.has(e.source)) { + ids.add(e.source); + upQueue.push(e.source); + } + } + } + + // Walk downstream (source → target) + const downQueue = [hoveredNodeId]; + while (downQueue.length > 0) { + const cur = downQueue.shift()!; + for (const e of allEdges) { + if (e.source === cur && !ids.has(e.target)) { + ids.add(e.target); + downQueue.push(e.target); + } + } + } + + return ids; + }, [hoveredNodeId]); + + const onNodeMouseEnter = useCallback( + (_: React.MouseEvent, node: Node) => setHoveredNodeId(node.id), + [], + ); + const onNodeMouseLeave = useCallback(() => setHoveredNodeId(null), []); + useEffect(() => { if (registryData && relationships) { setLoading(true); @@ -1092,6 +1149,7 @@ const RegistryVisualization: React.FC = ({ showIsolatedNodes, ); + edgesRef.current = layoutedEdges; setNodes(layoutedNodes); setEdges(layoutedEdges); setLoading(false); @@ -1109,6 +1167,31 @@ const RegistryVisualization: React.FC = ({ setEdges, ]); + const styledNodes = useMemo(() => { + if (!connectedIds) return nodes; + return nodes.map((n) => ({ + ...n, + style: { + ...n.style, + opacity: connectedIds.has(n.id) ? 1 : 0.15, + transition: "opacity 0.2s", + }, + })); + }, [nodes, connectedIds]); + + const styledEdges = useMemo(() => { + if (!connectedIds) return edges; + return edges.map((e) => ({ + ...e, + style: { + ...e.style, + opacity: + connectedIds.has(e.source) && connectedIds.has(e.target) ? 1 : 0.08, + transition: "opacity 0.2s", + }, + })); + }, [edges, connectedIds]); + return ( @@ -1159,8 +1242,8 @@ const RegistryVisualization: React.FC = ({ ) : (
= ({ fitView minZoom={0.1} maxZoom={8} + onNodeMouseEnter={onNodeMouseEnter} + onNodeMouseLeave={onNodeMouseLeave} > diff --git a/ui/src/components/RegistryVisualizationTab.tsx b/ui/src/components/RegistryVisualizationTab.tsx index 4fed3c4f856..f710746d6c2 100644 --- a/ui/src/components/RegistryVisualizationTab.tsx +++ b/ui/src/components/RegistryVisualizationTab.tsx @@ -71,6 +71,8 @@ const RegistryVisualizationTab: React.FC = ({ return objects.labelViews?.map((lv: any) => lv.spec?.name) || []; case "featureService": return objects.featureServices?.map((fs: any) => fs.spec?.name) || []; + case "savedDataset": + return objects.savedDatasets?.map((sd: any) => sd.spec?.name) || []; default: return []; } @@ -133,6 +135,7 @@ const RegistryVisualizationTab: React.FC = ({ { value: "featureView", text: "Feature View" }, { value: "labelView", text: "Label View" }, { value: "featureService", text: "Feature Service" }, + { value: "savedDataset", text: "Saved Dataset" }, ...(mlflowData?.runs?.length ? [{ value: "mlflowRun", text: "MLflow Run" }] : []), diff --git a/ui/src/pages/lineage/Index.tsx b/ui/src/pages/lineage/Index.tsx index 4487b98f9b3..fc9f53f2d8d 100644 --- a/ui/src/pages/lineage/Index.tsx +++ b/ui/src/pages/lineage/Index.tsx @@ -43,7 +43,7 @@ const LineagePage = () => { ); const [activeTab, setActiveTab] = useState("lineage"); - const [registryOnly, setRegistryOnly] = useState(false); + const [registryOnly, setRegistryOnly] = useState(null); const [selectedNamespace, setSelectedNamespace] = useState(""); const { data: nsData } = useLoadNamespaces(); @@ -56,6 +56,14 @@ const LineagePage = () => { const olConsumerAvailable = !olGraphQuery.isError && olGraphQuery.data !== undefined; + const olHasData = + olConsumerAvailable && + olGraphQuery.data != null && + (olGraphQuery.data.nodes?.length ?? 0) > 0; + + const effectiveRegistryOnly = + registryOnly !== null ? registryOnly : !olHasData; + if (projectName === "all") { return ( @@ -145,9 +153,7 @@ const LineagePage = () => { })), ]} value={selectedNamespace} - onChange={(e) => - setSelectedNamespace(e.target.value) - } + onChange={(e) => setSelectedNamespace(e.target.value)} aria-label="Filter by namespace" /> @@ -158,13 +164,13 @@ const LineagePage = () => { {activeTab === "lineage" && ( <> - {registryOnly ? ( + {effectiveRegistryOnly ? ( setRegistryOnly(e.target.checked) } @@ -183,7 +189,7 @@ const LineagePage = () => {