From b090e8643d10261a3e93a52c869a069b5444c1e6 Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Wed, 6 Nov 2024 17:33:54 -0500
Subject: [PATCH 1/6] * Adding TLS support for offline server. * Added test
 cases for the TLS offline server by creating
 RemoteOfflineTlsStoreDataSourceCreator

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 sdk/python/feast/cli.py                       | 26 ++++++-
 sdk/python/feast/feature_store.py             | 10 ++-
 .../feast/infra/offline_stores/remote.py      | 27 ++++++--
 sdk/python/feast/offline_server.py            | 51 ++++++++++++--
 .../feature_repos/repo_configuration.py       |  3 +-
 .../universal/data_sources/file.py            | 67 +++++++++++++++++--
 .../test_universal_historical_retrieval.py    |  3 +-
 7 files changed, 167 insertions(+), 20 deletions(-)

diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py
index 06db93d6803..52c9b5d8198 100644
--- a/sdk/python/feast/cli.py
+++ b/sdk/python/feast/cli.py
@@ -1114,16 +1114,40 @@ def serve_registry_command(
     default=DEFAULT_OFFLINE_SERVER_PORT,
     help="Specify a port for the server",
 )
+@click.option(
+    "--key",
+    "-k",
+    "tls_key_path",
+    type=click.STRING,
+    default="",
+    show_default=False,
+    help="path to TLS certificate private key. You need to pass --cert as well to start server in TLS mode",
+)
+@click.option(
+    "--cert",
+    "-c",
+    "tls_cert_path",
+    type=click.STRING,
+    default="",
+    show_default=False,
+    help="path to TLS certificate public key. You need to pass --key as well to start server in TLS mode",
+)
 @click.pass_context
 def serve_offline_command(
     ctx: click.Context,
     host: str,
     port: int,
+    tls_key_path: str,
+    tls_cert_path: str,
 ):
     """Start a remote server locally on a given host, port."""
+    if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path):
+        raise click.BadParameter(
+            "Please pass --cert and --key args to start the offline server in TLS mode."
+        )
     store = create_feature_store(ctx)
 
-    store.serve_offline(host, port)
+    store.serve_offline(host, port, tls_key_path, tls_cert_path)
 
 
 @cli.command("validate")
diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py
index e6cdf90b4a3..429b653b5ab 100644
--- a/sdk/python/feast/feature_store.py
+++ b/sdk/python/feast/feature_store.py
@@ -1963,11 +1963,17 @@ def serve_registry(
             self, port=port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path
         )
 
-    def serve_offline(self, host: str, port: int) -> None:
+    def serve_offline(
+        self,
+        host: str,
+        port: int,
+        tls_key_path: str = "",
+        tls_cert_path: str = "",
+    ) -> None:
         """Start offline server locally on a given port."""
         from feast import offline_server
 
-        offline_server.start_server(self, host, port)
+        offline_server.start_server(self, host, port, tls_key_path, tls_cert_path)
 
     def serve_transformations(self, port: int) -> None:
         """Start the feature transformation server locally on a given port."""
diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py
index 7ee018ac6d9..691c24a7e9f 100644
--- a/sdk/python/feast/infra/offline_stores/remote.py
+++ b/sdk/python/feast/infra/offline_stores/remote.py
@@ -70,22 +70,38 @@ def list_actions(self, options: FlightCallOptions = None):
         return super().list_actions(options)
 
 
-def build_arrow_flight_client(host: str, port, auth_config: AuthConfig):
+def build_arrow_flight_client(scheme: str, host: str, port, auth_config: AuthConfig):
+    arrow_scheme = "grpc+tcp"
+    if scheme == "https":
+        logger.info(
+            "Scheme is https so going to connect offline server in SSL(TLS) mode."
+        )
+        arrow_scheme = "grpc+tls"
+
     if auth_config.type != AuthType.NONE.value:
         middlewares = [FlightAuthInterceptorFactory(auth_config)]
-        return FeastFlightClient(f"grpc://{host}:{port}", middleware=middlewares)
+        return FeastFlightClient(
+            f"{arrow_scheme}://{host}:{port}", middleware=middlewares
+        )
 
-    return FeastFlightClient(f"grpc://{host}:{port}")
+    return FeastFlightClient(f"{arrow_scheme}://{host}:{port}")
 
 
 class RemoteOfflineStoreConfig(FeastConfigBaseModel):
     type: Literal["remote"] = "remote"
+
+    scheme: Literal["http", "https"] = "http"
+
     host: StrictStr
     """ str: remote offline store server port, e.g. the host URL for offline store  of arrow flight server. """
 
     port: Optional[StrictInt] = None
     """ str: remote offline store server port."""
 
+    cert: StrictStr = ""
+    """ str: Path to the public certificate when the offline server starts in TLS(SSL) mode. This may be needed if the offline server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`.
+    If type is 'remote', then this configuration is needed to connect to remote offline server in TLS mode. """
+
 
 class RemoteRetrievalJob(RetrievalJob):
     def __init__(
@@ -178,7 +194,10 @@ def get_historical_features(
         assert isinstance(config.offline_store, RemoteOfflineStoreConfig)
 
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            scheme=config.offline_store.scheme,
+            host=config.offline_store.host,
+            port=config.offline_store.port,
+            auth_config=config.auth_config,
         )
 
         feature_view_names = [fv.name for fv in feature_views]
diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py
index cec043129e7..13fab17bb77 100644
--- a/sdk/python/feast/offline_server.py
+++ b/sdk/python/feast/offline_server.py
@@ -39,12 +39,22 @@
 
 
 class OfflineServer(fl.FlightServerBase):
-    def __init__(self, store: FeatureStore, location: str, **kwargs):
+    def __init__(
+        self,
+        store: FeatureStore,
+        location: str,
+        host: str = "localhost",
+        tls_certificates: [] = None,
+        verify_client=False,
+        **kwargs,
+    ):
         super(OfflineServer, self).__init__(
-            location,
+            location=location,
             middleware=self.arrow_flight_auth_middleware(
                 str_to_auth_manager_type(store.config.auth_config.type)
             ),
+            tls_certificates=tls_certificates,
+            verify_client=verify_client,
             **kwargs,
         )
         self._location = location
@@ -52,6 +62,8 @@ def __init__(self, store: FeatureStore, location: str, **kwargs):
         self.flights: Dict[str, Any] = {}
         self.store = store
         self.offline_store = get_offline_store_from_config(store.config.offline_store)
+        self.host = host
+        self.tls_certificates = tls_certificates
 
     def arrow_flight_auth_middleware(
         self,
@@ -81,8 +93,13 @@ def descriptor_to_key(self, descriptor: fl.FlightDescriptor):
         )
 
     def _make_flight_info(self, key: Any, descriptor: fl.FlightDescriptor):
-        endpoints = [fl.FlightEndpoint(repr(key), [self._location])]
-        # TODO calculate actual schema from the given features
+        if len(self.tls_certificates) != 0:
+            location = fl.Location.for_grpc_tls(self.host, self.port)
+        else:
+            location = fl.Location.for_grpc_tcp(self.host, self.port)
+        endpoints = [
+            fl.FlightEndpoint(repr(key), [location]),
+        ]
         schema = pa.schema([])
 
         return fl.FlightInfo(schema, descriptor, endpoints, -1, -1)
@@ -549,11 +566,33 @@ def start_server(
     store: FeatureStore,
     host: str,
     port: int,
+    tls_key_path: str = "",
+    tls_cert_path: str = "",
 ):
     _init_auth_manager(store)
 
-    location = "grpc+tcp://{}:{}".format(host, port)
-    server = OfflineServer(store, location)
+    tls_certificates = []
+    scheme = "grpc+tcp"
+    if tls_key_path and tls_cert_path:
+        logger.info(
+            "Found SSL certificates in the args so going to start offline server in TLS(SSL) mode."
+        )
+        scheme = "grpc+tls"
+        with open(tls_cert_path, "rb") as cert_file:
+            tls_cert_chain = cert_file.read()
+        with open(tls_key_path, "rb") as key_file:
+            tls_private_key = key_file.read()
+        tls_certificates.append((tls_cert_chain, tls_private_key))
+
+    location = "{}://{}:{}".format(scheme, host, port)
+    server = OfflineServer(
+        store,
+        location=location,
+        host=host,
+        port=port,
+        tls_certificates=tls_certificates,
+        verify_client=True,
+    )
     try:
         logger.info(f"Offline store server serving at: {location}")
         server.serve()
diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py
index c688a848362..a6008dec94a 100644
--- a/sdk/python/tests/integration/feature_repos/repo_configuration.py
+++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py
@@ -48,7 +48,7 @@
     DuckDBDeltaDataSourceCreator,
     FileDataSourceCreator,
     RemoteOfflineOidcAuthStoreDataSourceCreator,
-    RemoteOfflineStoreDataSourceCreator,
+    RemoteOfflineStoreDataSourceCreator, RemoteOfflineTlsStoreDataSourceCreator,
 )
 from tests.integration.feature_repos.universal.data_sources.redshift import (
     RedshiftDataSourceCreator,
@@ -131,6 +131,7 @@
     ("local", DuckDBDeltaDataSourceCreator),
     ("local", RemoteOfflineStoreDataSourceCreator),
     ("local", RemoteOfflineOidcAuthStoreDataSourceCreator),
+    ("local", RemoteOfflineTlsStoreDataSourceCreator),
 ]
 
 if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True":
diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
index 35325c2737e..fff1e89fca8 100644
--- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
+++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
@@ -34,6 +34,7 @@
     DataSourceCreator,
 )
 from tests.utils.auth_permissions_util import include_auth_config
+from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert
 from tests.utils.http_server import check_port_open, free_port  # noqa: E402
 
 logger = logging.getLogger(__name__)
@@ -410,11 +411,67 @@ def setup(self, registry: RegistryConfig):
         )
         return "grpc+tcp://{}:{}".format(host, self.server_port)
 
+class RemoteOfflineTlsStoreDataSourceCreator(FileDataSourceCreator):
+    def __init__(self, project_name: str, *args, **kwargs):
+            super().__init__(project_name)
+            self.server_port: int = 0
+            self.proc: Optional[Popen[bytes]] = None
+
+    def setup(self, registry: RegistryConfig):
+        parent_offline_config = super().create_offline_store_config()
+        config = RepoConfig(
+            project=self.project_name,
+            provider="local",
+            offline_store=parent_offline_config,
+            registry=registry.path,
+            entity_key_serialization_version=2,
+        )
+
+        certificates_path = tempfile.mkdtemp()
+        tls_key_path = os.path.join(certificates_path, "key.pem")
+        self.tls_cert_path = os.path.join(certificates_path, "cert.pem")
+        generate_self_signed_cert(cert_path=self.tls_cert_path, key_path=tls_key_path)
+
+
+        repo_path = Path(tempfile.mkdtemp())
+        with open(repo_path / "feature_store.yaml", "w") as outfile:
+            yaml.dump(config.model_dump(by_alias=True), outfile)
+        repo_path = repo_path.resolve()
+
+        self.server_port = free_port()
+        host = "0.0.0.0"
+        cmd = [
+            "feast",
+            "-c" + str(repo_path),
+            "serve_offline",
+            "--host",
+            host,
+            "--port",
+            str(self.server_port),
+            "--key",
+            str(tls_key_path),
+            "--cert",
+            str(self.tls_cert_path)
+            ]
+        self.proc = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
+        )
+
+        _time_out_sec: int = 60
+        # Wait for server to start
+        wait_retry_backoff(
+            lambda: (None, check_port_open(host, self.server_port)),
+            timeout_secs=_time_out_sec,
+            timeout_msg=f"Unable to start the feast remote offline server in {_time_out_sec} seconds at port={self.server_port}",
+        )
+        return "grpc+tls://{}:{}".format(host, self.server_port)
+
+
     def create_offline_store_config(self) -> FeastConfigBaseModel:
-        self.remote_offline_store_config = RemoteOfflineStoreConfig(
-            type="remote", host="0.0.0.0", port=self.server_port
+        remote_offline_store_config = RemoteOfflineStoreConfig(
+            type="remote", host="0.0.0.0", port=self.server_port, scheme="https", cert=self.tls_cert_path
         )
-        return self.remote_offline_store_config
+        return remote_offline_store_config
 
     def teardown(self):
         super().teardown()
@@ -499,10 +556,10 @@ def setup(self, registry: RegistryConfig):
         return "grpc+tcp://{}:{}".format(host, self.server_port)
 
     def create_offline_store_config(self) -> FeastConfigBaseModel:
-        self.remote_offline_store_config = RemoteOfflineStoreConfig(
+        remote_offline_store_config = RemoteOfflineStoreConfig(
             type="remote", host="0.0.0.0", port=self.server_port
         )
-        return self.remote_offline_store_config
+        return remote_offline_store_config
 
     def get_keycloak_url(self):
         return self.keycloak_url
diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
index 97ad54251fe..b3047746d24 100644
--- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
+++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
@@ -22,7 +22,7 @@
 )
 from tests.integration.feature_repos.universal.data_sources.file import (
     RemoteOfflineOidcAuthStoreDataSourceCreator,
-    RemoteOfflineStoreDataSourceCreator,
+    RemoteOfflineStoreDataSourceCreator, RemoteOfflineTlsStoreDataSourceCreator,
 )
 from tests.integration.feature_repos.universal.data_sources.snowflake import (
     SnowflakeDataSourceCreator,
@@ -166,6 +166,7 @@ def test_historical_features_main(
         environment.data_source_creator,
         (
             RemoteOfflineStoreDataSourceCreator,
+            RemoteOfflineTlsStoreDataSourceCreator,
             RemoteOfflineOidcAuthStoreDataSourceCreator,
         ),
     ):

From dd9dca076e5afbc97287b561adba06749a3a14c6 Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Thu, 7 Nov 2024 10:42:03 -0500
Subject: [PATCH 2/6] * Fixing the lint error and also integration tests.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 .../feast/infra/offline_stores/remote.py      | 30 +++++++++++++++----
 sdk/python/feast/offline_server.py            |  2 +-
 .../feature_repos/repo_configuration.py       |  3 +-
 .../universal/data_sources/file.py            | 19 +++++++-----
 .../test_universal_historical_retrieval.py    |  3 +-
 5 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py
index 691c24a7e9f..8e4f650d68c 100644
--- a/sdk/python/feast/infra/offline_stores/remote.py
+++ b/sdk/python/feast/infra/offline_stores/remote.py
@@ -233,7 +233,10 @@ def pull_all_from_table_or_query(
 
         # Initialize the client connection
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         api_parameters = {
@@ -266,7 +269,10 @@ def pull_latest_from_table_or_query(
 
         # Initialize the client connection
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         api_parameters = {
@@ -301,7 +307,10 @@ def write_logged_features(
 
         # Initialize the client connection
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         api_parameters = {
@@ -327,7 +336,10 @@ def offline_write_batch(
 
         # Initialize the client connection
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         feature_view_names = [feature_view.name]
@@ -355,7 +367,10 @@ def validate_data_source(
         assert isinstance(config.offline_store, RemoteOfflineStoreConfig)
 
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         api_parameters = {
@@ -376,7 +391,10 @@ def get_table_column_names_and_types_from_data_source(
         assert isinstance(config.offline_store, RemoteOfflineStoreConfig)
 
         client = build_arrow_flight_client(
-            config.offline_store.host, config.offline_store.port, config.auth_config
+            config.offline_store.scheme,
+            config.offline_store.host,
+            config.offline_store.port,
+            config.auth_config,
         )
 
         api_parameters = {
diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py
index 13fab17bb77..d94df0fffda 100644
--- a/sdk/python/feast/offline_server.py
+++ b/sdk/python/feast/offline_server.py
@@ -44,7 +44,7 @@ def __init__(
         store: FeatureStore,
         location: str,
         host: str = "localhost",
-        tls_certificates: [] = None,
+        tls_certificates: List = [],
         verify_client=False,
         **kwargs,
     ):
diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py
index a6008dec94a..bf464681600 100644
--- a/sdk/python/tests/integration/feature_repos/repo_configuration.py
+++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py
@@ -48,7 +48,8 @@
     DuckDBDeltaDataSourceCreator,
     FileDataSourceCreator,
     RemoteOfflineOidcAuthStoreDataSourceCreator,
-    RemoteOfflineStoreDataSourceCreator, RemoteOfflineTlsStoreDataSourceCreator,
+    RemoteOfflineStoreDataSourceCreator,
+    RemoteOfflineTlsStoreDataSourceCreator,
 )
 from tests.integration.feature_repos.universal.data_sources.redshift import (
     RedshiftDataSourceCreator,
diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
index fff1e89fca8..fbfb418278e 100644
--- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
+++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
@@ -411,11 +411,12 @@ def setup(self, registry: RegistryConfig):
         )
         return "grpc+tcp://{}:{}".format(host, self.server_port)
 
+
 class RemoteOfflineTlsStoreDataSourceCreator(FileDataSourceCreator):
     def __init__(self, project_name: str, *args, **kwargs):
-            super().__init__(project_name)
-            self.server_port: int = 0
-            self.proc: Optional[Popen[bytes]] = None
+        super().__init__(project_name)
+        self.server_port: int = 0
+        self.proc: Optional[Popen[bytes]] = None
 
     def setup(self, registry: RegistryConfig):
         parent_offline_config = super().create_offline_store_config()
@@ -432,7 +433,6 @@ def setup(self, registry: RegistryConfig):
         self.tls_cert_path = os.path.join(certificates_path, "cert.pem")
         generate_self_signed_cert(cert_path=self.tls_cert_path, key_path=tls_key_path)
 
-
         repo_path = Path(tempfile.mkdtemp())
         with open(repo_path / "feature_store.yaml", "w") as outfile:
             yaml.dump(config.model_dump(by_alias=True), outfile)
@@ -451,8 +451,8 @@ def setup(self, registry: RegistryConfig):
             "--key",
             str(tls_key_path),
             "--cert",
-            str(self.tls_cert_path)
-            ]
+            str(self.tls_cert_path),
+        ]
         self.proc = subprocess.Popen(
             cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
         )
@@ -466,10 +466,13 @@ def setup(self, registry: RegistryConfig):
         )
         return "grpc+tls://{}:{}".format(host, self.server_port)
 
-
     def create_offline_store_config(self) -> FeastConfigBaseModel:
         remote_offline_store_config = RemoteOfflineStoreConfig(
-            type="remote", host="0.0.0.0", port=self.server_port, scheme="https", cert=self.tls_cert_path
+            type="remote",
+            host="0.0.0.0",
+            port=self.server_port,
+            scheme="https",
+            cert=self.tls_cert_path,
         )
         return remote_offline_store_config
 
diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
index b3047746d24..3f28245f3c7 100644
--- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
+++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py
@@ -22,7 +22,8 @@
 )
 from tests.integration.feature_repos.universal.data_sources.file import (
     RemoteOfflineOidcAuthStoreDataSourceCreator,
-    RemoteOfflineStoreDataSourceCreator, RemoteOfflineTlsStoreDataSourceCreator,
+    RemoteOfflineStoreDataSourceCreator,
+    RemoteOfflineTlsStoreDataSourceCreator,
 )
 from tests.integration.feature_repos.universal.data_sources.snowflake import (
     SnowflakeDataSourceCreator,

From f92d26bbb439a2e77068b97e9dedbbed3bb9e832 Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Thu, 7 Nov 2024 11:24:09 -0500
Subject: [PATCH 3/6] * Added documentation for the offline server and moved to
 how to guide. * Fixing the issue with integration test.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 docs/SUMMARY.md                               |  1 +
 .../starting-feast-servers-tls-mode.md        | 76 +++++++++++++++++--
 sdk/python/feast/offline_server.py            |  1 -
 3 files changed, 69 insertions(+), 9 deletions(-)
 rename docs/{reference => how-to-guides}/starting-feast-servers-tls-mode.md (64%)

diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index 91ef61dac94..09d0f55a685 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -64,6 +64,7 @@
   * [Adding a new online store](how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md)
   * [Adding a custom provider](how-to-guides/customizing-feast/creating-a-custom-provider.md)
 * [Adding or reusing tests](how-to-guides/adding-or-reusing-tests.md)
+* [Starting Feast servers in TLS(SSL) Mode](how-to-guides/starting-feast-servers-tls-mode.md)
 
 ## Reference
 
diff --git a/docs/reference/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md
similarity index 64%
rename from docs/reference/starting-feast-servers-tls-mode.md
rename to docs/how-to-guides/starting-feast-servers-tls-mode.md
index 366cd79d564..8ae595cec35 100644
--- a/docs/reference/starting-feast-servers-tls-mode.md
+++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md
@@ -1,7 +1,9 @@
 # Starting feast servers in TLS (SSL) mode.
 TLS (Transport Layer Security) and SSL (Secure Sockets Layer) are both protocols encrypts communications between a client and server to provide enhanced security.TLS or SSL words used interchangeably. 
 This article is going to show the sample code to start all the feast servers such as online server, offline server, registry server and UI server in TLS mode. 
-Also show examples related to feast clients to communicate with the feast servers started in TLS mode.
+Also show examples related to feast clients to communicate with the feast servers started in TLS mode. 
+
+We assume you have basic understanding of feast terminology before going through this tutorial, if you are new to feast then we would recommend to go through existing [starter tutorials](./../../examples) of feast.
 
 ## Obtaining a self-signed TLS certificate and key
 In development mode we can generate a self-signed certificate for testing. In an actual production environment it is always recommended to get it from a trusted TLS certificate provider.
@@ -17,15 +19,32 @@ The above command will generate two files
 You can use the public or private keys generated from above command in the rest of the sections in this tutorial.
 
 ## Create the feast demo repo for the rest of the sections.
-create a feast repo using `feast init` command and use this repo as a demo for subsequent sections.
+Create a feast repo and initialize using `feast init` and `feast apply` command and use this repo as a demo for subsequent sections.
 
 ```shell
 feast init feast_repo_ssl_demo
-```
 
-Output is
-```
+#output will be something similar as below
 Creating a new Feast repository in /Documents/Src/feast/feast_repo_ssl_demo.
+
+cd feast_repo_ssl_demo/feature_repo
+feast apply
+
+#output will be something similar as below
+Applying changes for project feast_repo_ssl_demo
+
+Created project feast_repo_ssl_demo
+Created entity driver
+Created feature view driver_hourly_stats
+Created feature view driver_hourly_stats_fresh
+Created on demand feature view transformed_conv_rate
+Created on demand feature view transformed_conv_rate_fresh
+Created feature service driver_activity_v1
+Created feature service driver_activity_v3
+Created feature service driver_activity_v2
+
+Created sqlite table feast_repo_ssl_demo_driver_hourly_stats_fresh
+Created sqlite table feast_repo_ssl_demo_driver_hourly_stats
 ```
 
 You need to execute the feast cli commands from  `feast_repo_ssl_demo/feature_repo` directory created from the above `feast init` command.
@@ -68,7 +87,7 @@ entity_key_serialization_version: 2
 auth:
   type: no_auth
 ```
-{% endcode %}
+
 
 `cert` is an optional configuration to the public certificate path when the online server starts in TLS(SSL) mode. Typically, this file ends with `*.crt`, `*.cer`, or `*.pem`.
 
@@ -106,14 +125,55 @@ entity_key_serialization_version: 2
 auth:
   type: no_auth
 ```
-{% endcode %}
 
 `cert` is an optional configuration to the public certificate path when the registry server starts in TLS(SSL) mode. Typically, this file ends with `*.crt`, `*.cer`, or `*.pem`.
 
 ## Starting feast offline server in TLS mode
 
-TBD
+To start the offline server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments with the `feast serve_offline` command.
+
+```shell
+feast serve_offline --key /path/to/key.pem --cert /path/to/cert.pem
+```
+You will see the output something similar to as below. Note the server url starts in the `https` mode.
+
+```shell
+11/07/2024 11:10:01 AM feast.offline_server INFO: Found SSL certificates in the args so going to start offline server in TLS(SSL) mode.
+11/07/2024 11:10:01 AM feast.offline_server INFO: Offline store server serving at: grpc+tls://127.0.0.1:8815
+11/07/2024 11:10:01 AM feast.offline_server INFO: offline server starting with pid: [11606]
+```
+
+### Feast client connecting to remote registry sever started in TLS mode.
+
+Sometimes you may need to pass the self-signed public key to connect to the remote registry server started in SSL mode if you have not added the public key to the certificate store.
+You have to add `scheme` to `https`.
+
+feast client example:
+
+```yaml
+project: feast-project
+registry:
+  registry_type: remote
+  path: https://localhost:6570
+  cert: /path/to/cert.pem
+provider: local
+online_store:
+  path: http://localhost:6566
+  type: remote
+  cert: /path/to/cert.pem
+entity_key_serialization_version: 2
+offline_store:
+  type: remote
+  host: localhost
+  port: 8815
+  scheme: https
+  cert: /path/to/cert.pem
+auth:
+  type: no_auth
+```
 
+`cert` is an optional configuration to the public certificate path when the registry server starts in TLS(SSL) mode. Typically, this file ends with `*.crt`, `*.cer`, or `*.pem`.
+`scheme` should be `https`. By default, it will be `http` so you have to explicitly configure to `https` if you are planning to connect to remote offline server which is started in TLS mode.
 
 ## Starting feast UI server (react app) in TLS mode
 To start the feast UI server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments with the `feast ui` command.
diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py
index d94df0fffda..87d1ac3d91e 100644
--- a/sdk/python/feast/offline_server.py
+++ b/sdk/python/feast/offline_server.py
@@ -589,7 +589,6 @@ def start_server(
         store,
         location=location,
         host=host,
-        port=port,
         tls_certificates=tls_certificates,
         verify_client=True,
     )

From 033e0657601e2bf1d583422f02fc7a7e970b0320 Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Thu, 7 Nov 2024 11:24:09 -0500
Subject: [PATCH 4/6] * Added documentation for the offline server and moved to
 how to guide. * Fixing the issue with integration test.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 docs/how-to-guides/starting-feast-servers-tls-mode.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md
index 8ae595cec35..e1ddbc08be5 100644
--- a/docs/how-to-guides/starting-feast-servers-tls-mode.md
+++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md
@@ -143,7 +143,7 @@ You will see the output something similar to as below. Note the server url start
 11/07/2024 11:10:01 AM feast.offline_server INFO: offline server starting with pid: [11606]
 ```
 
-### Feast client connecting to remote registry sever started in TLS mode.
+### Feast client connecting to remote offline sever started in TLS mode.
 
 Sometimes you may need to pass the self-signed public key to connect to the remote registry server started in SSL mode if you have not added the public key to the certificate store.
 You have to add `scheme` to `https`.

From c60c07a52d65f370e2ab57869882f0af435be614 Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Thu, 7 Nov 2024 14:11:00 -0500
Subject: [PATCH 5/6] * fixing the integration test by adding extra flag
 verify_client

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 sdk/python/feast/cli.py                       | 12 ++++++++-
 sdk/python/feast/feature_store.py             |  5 +++-
 .../feast/infra/offline_stores/remote.py      | 25 ++++++++++++++-----
 sdk/python/feast/offline_server.py            |  3 ++-
 .../universal/data_sources/file.py            |  3 +++
 5 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py
index 52c9b5d8198..a02013b11f9 100644
--- a/sdk/python/feast/cli.py
+++ b/sdk/python/feast/cli.py
@@ -1132,6 +1132,15 @@ def serve_registry_command(
     show_default=False,
     help="path to TLS certificate public key. You need to pass --key as well to start server in TLS mode",
 )
+@click.option(
+    "--verify_client",
+    "-v",
+    "tls_verify_client",
+    type=click.BOOL,
+    default="True",
+    show_default=True,
+    help="Verify the client or not for the TLS client certificate.",
+)
 @click.pass_context
 def serve_offline_command(
     ctx: click.Context,
@@ -1139,6 +1148,7 @@ def serve_offline_command(
     port: int,
     tls_key_path: str,
     tls_cert_path: str,
+    tls_verify_client: bool,
 ):
     """Start a remote server locally on a given host, port."""
     if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path):
@@ -1147,7 +1157,7 @@ def serve_offline_command(
         )
     store = create_feature_store(ctx)
 
-    store.serve_offline(host, port, tls_key_path, tls_cert_path)
+    store.serve_offline(host, port, tls_key_path, tls_cert_path, tls_verify_client)
 
 
 @cli.command("validate")
diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py
index 429b653b5ab..54a8034b540 100644
--- a/sdk/python/feast/feature_store.py
+++ b/sdk/python/feast/feature_store.py
@@ -1969,11 +1969,14 @@ def serve_offline(
         port: int,
         tls_key_path: str = "",
         tls_cert_path: str = "",
+        tls_verify_client: bool = True,
     ) -> None:
         """Start offline server locally on a given port."""
         from feast import offline_server
 
-        offline_server.start_server(self, host, port, tls_key_path, tls_cert_path)
+        offline_server.start_server(
+            self, host, port, tls_key_path, tls_cert_path, tls_verify_client
+        )
 
     def serve_transformations(self, port: int) -> None:
         """Start the feature transformation server locally on a given port."""
diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py
index 8e4f650d68c..35b08e4174f 100644
--- a/sdk/python/feast/infra/offline_stores/remote.py
+++ b/sdk/python/feast/infra/offline_stores/remote.py
@@ -70,7 +70,9 @@ def list_actions(self, options: FlightCallOptions = None):
         return super().list_actions(options)
 
 
-def build_arrow_flight_client(scheme: str, host: str, port, auth_config: AuthConfig):
+def build_arrow_flight_client(
+    scheme: str, host: str, port, auth_config: AuthConfig, cert: str = ""
+):
     arrow_scheme = "grpc+tcp"
     if scheme == "https":
         logger.info(
@@ -83,8 +85,12 @@ def build_arrow_flight_client(scheme: str, host: str, port, auth_config: AuthCon
         return FeastFlightClient(
             f"{arrow_scheme}://{host}:{port}", middleware=middlewares
         )
+    kwargs = {}
+    if cert:
+        with open(cert, "rb") as root_certs:
+            kwargs["tls_root_certs"] = root_certs.read()
 
-    return FeastFlightClient(f"{arrow_scheme}://{host}:{port}")
+    return FeastFlightClient(f"{arrow_scheme}://{host}:{port}", **kwargs)
 
 
 class RemoteOfflineStoreConfig(FeastConfigBaseModel):
@@ -198,6 +204,7 @@ def get_historical_features(
             host=config.offline_store.host,
             port=config.offline_store.port,
             auth_config=config.auth_config,
+            cert=config.offline_store.cert,
         )
 
         feature_view_names = [fv.name for fv in feature_views]
@@ -233,10 +240,11 @@ def pull_all_from_table_or_query(
 
         # Initialize the client connection
         client = build_arrow_flight_client(
-            config.offline_store.scheme,
-            config.offline_store.host,
-            config.offline_store.port,
-            config.auth_config,
+            scheme=config.offline_store.scheme,
+            host=config.offline_store.host,
+            port=config.offline_store.port,
+            auth_config=config.auth_config,
+            cert=config.offline_store.cert,
         )
 
         api_parameters = {
@@ -273,6 +281,7 @@ def pull_latest_from_table_or_query(
             config.offline_store.host,
             config.offline_store.port,
             config.auth_config,
+            cert=config.offline_store.cert,
         )
 
         api_parameters = {
@@ -311,6 +320,7 @@ def write_logged_features(
             config.offline_store.host,
             config.offline_store.port,
             config.auth_config,
+            config.offline_store.cert,
         )
 
         api_parameters = {
@@ -340,6 +350,7 @@ def offline_write_batch(
             config.offline_store.host,
             config.offline_store.port,
             config.auth_config,
+            config.offline_store.cert,
         )
 
         feature_view_names = [feature_view.name]
@@ -371,6 +382,7 @@ def validate_data_source(
             config.offline_store.host,
             config.offline_store.port,
             config.auth_config,
+            config.offline_store.cert,
         )
 
         api_parameters = {
@@ -395,6 +407,7 @@ def get_table_column_names_and_types_from_data_source(
             config.offline_store.host,
             config.offline_store.port,
             config.auth_config,
+            config.offline_store.cert,
         )
 
         api_parameters = {
diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py
index 87d1ac3d91e..8774dea8aed 100644
--- a/sdk/python/feast/offline_server.py
+++ b/sdk/python/feast/offline_server.py
@@ -568,6 +568,7 @@ def start_server(
     port: int,
     tls_key_path: str = "",
     tls_cert_path: str = "",
+    tls_verify_client: bool = True,
 ):
     _init_auth_manager(store)
 
@@ -590,7 +591,7 @@ def start_server(
         location=location,
         host=host,
         tls_certificates=tls_certificates,
-        verify_client=True,
+        verify_client=tls_verify_client,
     )
     try:
         logger.info(f"Offline store server serving at: {location}")
diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
index fbfb418278e..dc716f45e1e 100644
--- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
+++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py
@@ -452,6 +452,9 @@ def setup(self, registry: RegistryConfig):
             str(tls_key_path),
             "--cert",
             str(self.tls_cert_path),
+            # This is needed for the self-signed certificate, disabled verify_client for integration tests.
+            "--verify_client",
+            str(False),
         ]
         self.proc = subprocess.Popen(
             cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL

From 0f9dc98c38066f7e72e905554447a84e1d5c6b4f Mon Sep 17 00:00:00 2001
From: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
Date: Thu, 7 Nov 2024 17:57:19 -0500
Subject: [PATCH 6/6] * Adding alias names for the host in self-signed
 certificate.

Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com>
---
 sdk/python/feast/infra/offline_stores/remote.py    | 13 +++++++------
 .../utils/generate_self_signed_certifcate_util.py  | 14 ++++++++++----
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py
index 35b08e4174f..6f26e06c6ba 100644
--- a/sdk/python/feast/infra/offline_stores/remote.py
+++ b/sdk/python/feast/infra/offline_stores/remote.py
@@ -74,22 +74,23 @@ def build_arrow_flight_client(
     scheme: str, host: str, port, auth_config: AuthConfig, cert: str = ""
 ):
     arrow_scheme = "grpc+tcp"
-    if scheme == "https":
+    if cert:
         logger.info(
             "Scheme is https so going to connect offline server in SSL(TLS) mode."
         )
         arrow_scheme = "grpc+tls"
 
-    if auth_config.type != AuthType.NONE.value:
-        middlewares = [FlightAuthInterceptorFactory(auth_config)]
-        return FeastFlightClient(
-            f"{arrow_scheme}://{host}:{port}", middleware=middlewares
-        )
     kwargs = {}
     if cert:
         with open(cert, "rb") as root_certs:
             kwargs["tls_root_certs"] = root_certs.read()
 
+    if auth_config.type != AuthType.NONE.value:
+        middlewares = [FlightAuthInterceptorFactory(auth_config)]
+        return FeastFlightClient(
+            f"{arrow_scheme}://{host}:{port}", middleware=middlewares, **kwargs
+        )
+
     return FeastFlightClient(f"{arrow_scheme}://{host}:{port}", **kwargs)
 
 
diff --git a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py b/sdk/python/tests/utils/generate_self_signed_certifcate_util.py
index 1b0b212818c..559ee18cde7 100644
--- a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py
+++ b/sdk/python/tests/utils/generate_self_signed_certifcate_util.py
@@ -1,3 +1,4 @@
+import ipaddress
 import logging
 from datetime import datetime, timedelta
 
@@ -36,6 +37,14 @@ def generate_self_signed_cert(
         ]
     )
 
+    # Define the certificate's Subject Alternative Names (SANs)
+    alt_names = [
+        x509.DNSName("localhost"),  # Hostname
+        x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")),  # Localhost IP
+        x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")),  # Bind-all IP (optional)
+    ]
+    san = x509.SubjectAlternativeName(alt_names)
+
     certificate = (
         x509.CertificateBuilder()
         .subject_name(subject)
@@ -47,10 +56,7 @@ def generate_self_signed_cert(
             # Certificate valid for 1 year
             datetime.utcnow() + timedelta(days=365)
         )
-        .add_extension(
-            x509.SubjectAlternativeName([x509.DNSName(common_name)]),
-            critical=False,
-        )
+        .add_extension(san, critical=False)
         .sign(key, hashes.SHA256(), default_backend())
     )