From e2e0b9177fd9e5e50a73d4bc3bc87a76c87202ea Mon Sep 17 00:00:00 2001 From: Jacob Bush Date: Thu, 4 Jun 2026 14:46:05 -0400 Subject: [PATCH] feat: Add mTLS support to remote registry gRPC client Add client certificate authentication (mutual TLS) to RemoteRegistry so it can connect to gRPC servers that require client certificates. New config fields on RemoteRegistryConfig: - client_cert: path to the client certificate (PEM) - client_key: path to the client private key (PEM) - authority: override the gRPC :authority header for TLS hostname verification (required when connecting through a tunnel or proxy) The existing cert field continues to serve as the CA certificate for server verification. Signed-off-by: Jacob Bush Co-authored-by: AI (Pi/Claude Opus 4.6 [250k]) --- .../starting-feast-servers-tls-mode.md | 46 +++ docs/reference/registries/remote.md | 22 +- sdk/python/feast/infra/registry/remote.py | 59 ++- .../registry/test_remote_registry_mtls.py | 344 ++++++++++++++++++ .../tests/utils/ssl_certifcates_util.py | 91 +++++ 5 files changed, 552 insertions(+), 10 deletions(-) create mode 100644 sdk/python/tests/unit/infra/registry/test_remote_registry_mtls.py diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md index ffc7e5d9e90..c3696a35532 100644 --- a/docs/how-to-guides/starting-feast-servers-tls-mode.md +++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md @@ -128,6 +128,52 @@ auth: `cert` is an optional configuration to the public certificate path when the registry server starts in TLS(SSL) mode. Typically, this file ends with `*.crt`, `*.cer`, or `*.pem`. +### Feast client connecting to remote registry server with mTLS + +If the Registry Server requires mutual TLS (mTLS), the client must present a certificate and private key in addition to trusting the server's CA certificate. Add `client_cert` and `client_key` to the registry configuration: + +```yaml +project: feast-project +registry: + registry_type: remote + path: feature-registry.example.com:443 + cert: /path/to/ca.crt + client_cert: /path/to/tls.crt + client_key: /path/to/tls.key +provider: local +online_store: + path: http://localhost:6566 + type: remote +entity_key_serialization_version: 3 +auth: + type: no_auth +``` + +* `cert` — CA certificate used to verify the server (or use the `SSL_CERT_FILE` / `REQUESTS_CA_BUNDLE` environment variable). +* `client_cert` — Client certificate presented to the server. Must be paired with `client_key`. +* `client_key` — Private key for the client certificate. + +#### Connecting through a tunnel or proxy + +When connecting through a tunnel (e.g. `gcloud compute start-iap-tunnel`) the client connects to `localhost`, but the server certificate is issued for the real service hostname. Set the `authority` field so that gRPC's TLS hostname verification passes: + +```shell +# In one terminal — start the tunnel: +gcloud compute start-iap-tunnel feature-registry.example.com 443 --local-host-port=localhost:8443 +``` + +```yaml +registry: + registry_type: remote + path: localhost:8443 + cert: /path/to/ca.crt + client_cert: /path/to/tls.crt + client_key: /path/to/tls.key + authority: feature-registry.example.com +``` + +Without `authority`, the gRPC client would check the server certificate against `localhost`, which would fail because the certificate's Subject Alternative Name (SAN) is `feature-registry.example.com`. + ## Starting feast offline server in TLS mode To start the offline server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments with the `feast serve_offline` command. diff --git a/docs/reference/registries/remote.md b/docs/reference/registries/remote.md index a03e30ac85f..64055304283 100644 --- a/docs/reference/registries/remote.md +++ b/docs/reference/registries/remote.md @@ -18,7 +18,27 @@ registry: {% endcode %} The optional `cert` parameter can be configured as well, it should point to the public certificate path when the Registry Server starts in SSL mode. This may be needed if the Registry Server is started with a self-signed certificate, typically this file ends with *.crt, *.cer, or *.pem. -More info about the `cert` parameter can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#feast-client-connecting-to-remote-registry-sever-started-in-tls-mode) + +For **mutual TLS (mTLS)**, you can also configure: +* `client_cert` — Path to the client certificate presented to the server. Must be paired with `client_key`. Typically ends with `*.crt` or `*.pem`. +* `client_key` — Path to the client private key. Must be paired with `client_cert`. Typically ends with `*.key` or `*.pem`. + +When connecting through a tunnel or proxy where the connection address differs from the server hostname, set: +* `authority` — Overrides the gRPC `:authority` header so the server certificate is validated against the correct hostname. + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_type: remote + path: localhost:8443 + cert: /path/to/ca.crt + client_cert: /path/to/tls.crt + client_key: /path/to/tls.key + authority: feature-registry.example.com +``` +{% endcode %} + +More info about TLS configuration can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#feast-client-connecting-to-remote-registry-sever-started-in-tls-mode) ## How to configure the server diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index ac5961cd677..469a958eadf 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -64,10 +64,25 @@ class RemoteRegistryConfig(RegistryConfig): is_tls: bool = False """ bool: Set to `True` if you plan to connect to a registry server running in TLS (SSL) mode. - If you intend to add the public certificate to the trust store instead of passing it via the `cert` parameter, this field must be set to `True`. - If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `true` is mandatory. + If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `True` is mandatory. """ + client_cert: StrictStr = "" + """ str: Path to the client certificate for mTLS (mutual TLS) authentication. + Required when connecting to a server that enforces mutual TLS. + Must be provided together with `client_key`. + Typically this file ends with `*.crt` or `*.pem`. """ + + client_key: StrictStr = "" + """ str: Path to the client private key for mTLS (mutual TLS) authentication. + Must be provided together with `client_cert`. + Typically this file ends with `*.key` or `*.pem`. """ + + authority: StrictStr = "" + """ str: Override the gRPC :authority header for TLS connections. + Required when the connection address differs from the service hostname, + e.g. when connecting through a tunnel or proxy for local development. """ + class RemoteRegistry(BaseRegistry): def __init__( @@ -89,19 +104,45 @@ def __init__( def _create_grpc_channel(self, registry_config): assert isinstance(registry_config, RemoteRegistryConfig) if registry_config.cert or registry_config.is_tls: - cafile = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE") - if not cafile and not registry_config.cert: + cafile = ( + registry_config.cert + or os.getenv("SSL_CERT_FILE") + or os.getenv("REQUESTS_CA_BUNDLE") + ) + if not cafile: raise EnvironmentError( "SSL_CERT_FILE or REQUESTS_CA_BUNDLE environment variable must be set to use secure TLS or set the cert parameter in feature_Store.yaml file under remote registry configuration." ) - with open( - registry_config.cert if registry_config.cert else cafile, "rb" - ) as cert_file: + if (registry_config.client_cert and not registry_config.client_key) or ( + not registry_config.client_cert and registry_config.client_key + ): + raise ValueError( + "Both client_cert and client_key must be provided for mTLS. " + "Only one was set in the remote registry configuration." + ) + + with open(cafile, "rb") as cert_file: trusted_certs = cert_file.read() + private_key: Optional[bytes] = None + certificate_chain: Optional[bytes] = None + if registry_config.client_cert and registry_config.client_key: + with open(registry_config.client_key, "rb") as key_file: + private_key = key_file.read() + with open(registry_config.client_cert, "rb") as cert_file: + certificate_chain = cert_file.read() tls_credentials = grpc.ssl_channel_credentials( - root_certificates=trusted_certs + root_certificates=trusted_certs, + private_key=private_key, + certificate_chain=certificate_chain, + ) + + options = [] + if registry_config.authority: + options.append(("grpc.default_authority", registry_config.authority)) + + return grpc.secure_channel( + registry_config.path, tls_credentials, options=options ) - return grpc.secure_channel(registry_config.path, tls_credentials) else: # Create an insecure gRPC channel return grpc.insecure_channel(registry_config.path) diff --git a/sdk/python/tests/unit/infra/registry/test_remote_registry_mtls.py b/sdk/python/tests/unit/infra/registry/test_remote_registry_mtls.py new file mode 100644 index 00000000000..e63b3d637df --- /dev/null +++ b/sdk/python/tests/unit/infra/registry/test_remote_registry_mtls.py @@ -0,0 +1,344 @@ +""" +Tests for mTLS (mutual TLS) support on RemoteRegistry. + +Covers two deployment patterns: + +1. **Direct connection** — client connects straight to the service. The + server cert's SAN includes the connection target so ``authority`` is + optional (gRPC defaults it to the target host). + +2. **IAP-tunnel / proxy** — ``gcloud compute start-iap-tunnel`` forwards + traffic through ``localhost``. The server cert's SAN is the real service + hostname (not ``localhost``), so the client *must* set ``authority`` to + match — otherwise TLS hostname verification fails. +""" + +import tempfile +from collections.abc import Generator +from concurrent import futures + +import grpc +import pytest +from google.protobuf.empty_pb2 import Empty + +from feast.protos.feast.core import Project_pb2, Registry_pb2 +from feast.protos.feast.registry import RegistryServer_pb2, RegistryServer_pb2_grpc +from tests.utils.ssl_certifcates_util import generate_mtls_certs + +# The hostname the server cert is issued for. +# Deliberately *not* localhost — the whole point of the authority override. +SERVICE_HOSTNAME = "feature-registry.example.com" + + +# --------------------------------------------------------------------------- +# Minimal gRPC servicer — just enough to prove the channel works +# --------------------------------------------------------------------------- + + +class _StubRegistryServicer(RegistryServer_pb2_grpc.RegistryServerServicer): + """Returns a single dummy project so list_projects() has something to assert on.""" + + def ListProjects( + self, + request: RegistryServer_pb2.ListProjectsRequest, + context: grpc.ServicerContext, + ) -> RegistryServer_pb2.ListProjectsResponse: + spec = Project_pb2.ProjectSpec( + name="test_project", + description="created by mTLS test", + ) + project = Project_pb2.Project(spec=spec) + return RegistryServer_pb2.ListProjectsResponse(projects=[project]) + + def Proto( + self, + request: Empty, + context: grpc.ServicerContext, + ) -> Registry_pb2.Registry: + return Registry_pb2.Registry() + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _generate_certs_and_start_server( + tmpdir: str, server_san: str +) -> tuple[dict[str, str], grpc.Server, int]: + """Helper: generate mTLS certs and start a gRPC server requiring client auth.""" + certs = generate_mtls_certs(tmpdir, server_san=server_san) + + with open(certs["ca_cert"], "rb") as f: + ca_cert = f.read() + with open(certs["server_key"], "rb") as f: + server_key = f.read() + with open(certs["server_cert"], "rb") as f: + server_cert = f.read() + + server_creds = grpc.ssl_server_credentials( + [(server_key, server_cert)], + root_certificates=ca_cert, + require_client_auth=True, + ) + + server = grpc.server(futures.ThreadPoolExecutor(max_workers=2)) + RegistryServer_pb2_grpc.add_RegistryServerServicer_to_server( + _StubRegistryServicer(), server + ) + port = server.add_secure_port("localhost:0", server_creds) + server.start() + return certs, server, port + + +# -- Fixtures for IAP-tunnel tests (SAN ≠ localhost) -- + + +@pytest.fixture(scope="module") +def mtls_certs() -> Generator[tuple[dict[str, str], int], None, None]: + """Certs where the server SAN is SERVICE_HOSTNAME (not localhost).""" + with tempfile.TemporaryDirectory() as tmpdir: + certs, server, port = _generate_certs_and_start_server(tmpdir, SERVICE_HOSTNAME) + yield certs, port + server.stop(grace=0) + + +@pytest.fixture(scope="module") +def mtls_server(mtls_certs: tuple[dict[str, str], int]) -> int: + return mtls_certs[1] + + +@pytest.fixture(scope="module") +def mtls_certs_only(mtls_certs: tuple[dict[str, str], int]) -> dict[str, str]: + return mtls_certs[0] + + +# -- Fixtures for direct-connection tests (SAN = localhost) -- + + +@pytest.fixture(scope="module") +def direct_mtls() -> Generator[tuple[dict[str, str], int], None, None]: + """Certs where the server SAN includes localhost (direct connection).""" + with tempfile.TemporaryDirectory() as tmpdir: + certs, server, port = _generate_certs_and_start_server(tmpdir, "localhost") + yield certs, port + server.stop(grace=0) + + +@pytest.fixture(scope="module") +def direct_server(direct_mtls: tuple[dict[str, str], int]) -> int: + return direct_mtls[1] + + +@pytest.fixture(scope="module") +def direct_certs(direct_mtls: tuple[dict[str, str], int]) -> dict[str, str]: + return direct_mtls[0] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestMtlsDirectConnection: + """ + Direct connection to the registry — no proxy or tunnel. + + The server cert's SAN includes ``localhost`` so gRPC's default authority + (derived from the connection target) already matches. The ``authority`` + field should be optional. + + Config equivalent: + registry: + registry_type: remote + path: feature-registry.example.com:443 + cert: /etc/tls/internal-client/ca.crt + client_cert: /etc/tls/internal-client/tls.crt + client_key: /etc/tls/internal-client/tls.key + """ + + def test_mtls_without_authority( + self, direct_server: int, direct_certs: dict[str, str] + ) -> None: + """authority is not set — gRPC defaults it to 'localhost', which matches the SAN.""" + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{direct_server}", + cert=direct_certs["ca_cert"], + client_cert=direct_certs["client_cert"], + client_key=direct_certs["client_key"], + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + projects = registry.list_projects() + assert len(projects) == 1 + assert projects[0].name == "test_project" + finally: + registry.close() + + def test_mtls_with_explicit_authority_matching_san( + self, direct_server: int, direct_certs: dict[str, str] + ) -> None: + """authority is explicitly set to the same host — should also work.""" + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{direct_server}", + cert=direct_certs["ca_cert"], + client_cert=direct_certs["client_cert"], + client_key=direct_certs["client_key"], + authority="localhost", + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + projects = registry.list_projects() + assert len(projects) == 1 + assert projects[0].name == "test_project" + finally: + registry.close() + + +class TestMtlsIapTunnel: + """ + Simulates ``gcloud compute start-iap-tunnel`` to a remote registry. + + The server cert's SAN is ``feature-registry.example.com`` — it does NOT + include ``localhost``. The ``authority`` field is mandatory. + + Config equivalent: + registry: + registry_type: remote + path: localhost:8443 + cert: /etc/tls/internal-client/ca.crt + client_cert: /etc/tls/internal-client/tls.crt + client_key: /etc/tls/internal-client/tls.key + authority: feature-registry.example.com + """ + + def test_list_projects_via_mtls_with_authority( + self, mtls_server: int, mtls_certs_only: dict[str, str] + ) -> None: + """The primary IAP-tunnel scenario: all four fields set.""" + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{mtls_server}", + cert=mtls_certs_only["ca_cert"], + client_cert=mtls_certs_only["client_cert"], + client_key=mtls_certs_only["client_key"], + authority=SERVICE_HOSTNAME, + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + projects = registry.list_projects() + assert len(projects) == 1 + assert projects[0].name == "test_project" + assert projects[0].description == "created by mTLS test" + finally: + registry.close() + + def test_proto_via_mtls_with_authority( + self, mtls_server: int, mtls_certs_only: dict[str, str] + ) -> None: + """Proto() is the simplest RPC — verify the channel works for it too.""" + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{mtls_server}", + cert=mtls_certs_only["ca_cert"], + client_cert=mtls_certs_only["client_cert"], + client_key=mtls_certs_only["client_key"], + authority=SERVICE_HOSTNAME, + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + proto = registry.proto() + assert proto is not None + finally: + registry.close() + + +class TestMtlsRejections: + """Verify mTLS enforcement — connections fail without proper credentials.""" + + def test_rejected_without_client_cert( + self, mtls_server: int, mtls_certs_only: dict[str, str] + ) -> None: + """Server requires mTLS; connecting with only server-TLS must fail.""" + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{mtls_server}", + cert=mtls_certs_only["ca_cert"], + is_tls=True, + authority=SERVICE_HOSTNAME, + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + with pytest.raises(grpc.RpcError): + registry.list_projects() + finally: + registry.close() + + def test_fails_without_authority_when_san_mismatch( + self, mtls_server: int, mtls_certs_only: dict[str, str] + ) -> None: + """ + Without authority override, the gRPC client checks the server cert SAN + against 'localhost' — which doesn't match — so the handshake fails. + This proves the authority field is necessary for the IAP-tunnel case. + """ + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{mtls_server}", + cert=mtls_certs_only["ca_cert"], + client_cert=mtls_certs_only["client_cert"], + client_key=mtls_certs_only["client_key"], + # No authority — should fail because SAN is SERVICE_HOSTNAME, not localhost + ) + registry = RemoteRegistry(config, project="test", repo_path=None) + try: + with pytest.raises(grpc.RpcError): + registry.list_projects() + finally: + registry.close() + + +class TestMtlsConfigValidation: + """Config validation catches mismatched cert/key before any I/O.""" + + def test_client_cert_without_client_key_raises(self) -> None: + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path="localhost:0", + cert="/dev/null", + client_cert="/dev/null", + ) + with pytest.raises( + ValueError, match="Both client_cert and client_key must be provided" + ): + RemoteRegistry(config, project="test", repo_path=None) + + def test_client_key_without_client_cert_raises(self) -> None: + from feast.infra.registry.remote import RemoteRegistry, RemoteRegistryConfig + + config = RemoteRegistryConfig( + registry_type="remote", + path="localhost:0", + cert="/dev/null", + client_key="/dev/null", + ) + with pytest.raises( + ValueError, match="Both client_cert and client_key must be provided" + ): + RemoteRegistry(config, project="test", repo_path=None) diff --git a/sdk/python/tests/utils/ssl_certifcates_util.py b/sdk/python/tests/utils/ssl_certifcates_util.py index 53b9df3973c..4525e0249c9 100644 --- a/sdk/python/tests/utils/ssl_certifcates_util.py +++ b/sdk/python/tests/utils/ssl_certifcates_util.py @@ -15,6 +15,97 @@ logger = logging.getLogger(__name__) +def generate_mtls_certs( + output_dir: str, + server_san: str = "feature-registry.example.com", +) -> dict[str, str]: + """ + Generate a CA, server, and client certificate chain for mTLS testing. + + The server cert's SAN is set to ``server_san`` (no ``localhost``), so a + gRPC client connecting to ``localhost`` must set the ``authority`` channel + option to ``server_san`` — exactly the IAP-tunnel pattern. + + Returns a dict with keys: ca_cert, server_key, server_cert, + client_key, client_cert. + """ + os.makedirs(output_dir, exist_ok=True) + paths = { + "ca_cert": os.path.join(output_dir, "ca.crt"), + "server_key": os.path.join(output_dir, "server.key"), + "server_cert": os.path.join(output_dir, "server.crt"), + "client_key": os.path.join(output_dir, "client.key"), + "client_cert": os.path.join(output_dir, "client.crt"), + } + + # --- CA (self-signed) --- + ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "TestCA")]) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after(datetime.utcnow() + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(ca_key, hashes.SHA256()) + ) + with open(paths["ca_cert"], "wb") as f: + f.write(ca_cert.public_bytes(serialization.Encoding.PEM)) + + def _issue_cert( + cn: str, + key_path: str, + cert_path: str, + san: x509.SubjectAlternativeName | None = None, + ): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]) + builder = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(ca_cert.subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after(datetime.utcnow() + timedelta(days=1)) + ) + if san is not None: + builder = builder.add_extension(san, critical=False) + cert = builder.sign(ca_key, hashes.SHA256()) + + with open(key_path, "wb") as f: + f.write( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + with open(cert_path, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + # --- Server cert --- + _issue_cert( + cn=server_san, + key_path=paths["server_key"], + cert_path=paths["server_cert"], + san=x509.SubjectAlternativeName([x509.DNSName(server_san)]), + ) + + # --- Client cert (no SAN needed) --- + _issue_cert( + cn="TestClient", + key_path=paths["client_key"], + cert_path=paths["client_cert"], + ) + + logger.info(f"mTLS certificates generated in {output_dir}") + return paths + + def generate_self_signed_cert( cert_path="cert.pem", key_path="key.pem", common_name="localhost" ):