From e916f716521ea16d4087d2d909b3e4f1a896f05f Mon Sep 17 00:00:00 2001 From: Jiabin Hu Date: Sun, 8 Mar 2026 23:20:30 -0700 Subject: [PATCH 1/6] [QI-3367] Allow specifiying query tags as a dict upon connection creation (#749) * Allow specifiying query tags as a dict upon connection creation Signed-off-by: Jiabin Hu * fix comment Signed-off-by: Jiabin Hu --------- Signed-off-by: Jiabin Hu --- examples/query_tags_example.py | 15 ++++++--------- src/databricks/sql/client.py | 11 +++++++++++ tests/unit/test_session.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/examples/query_tags_example.py b/examples/query_tags_example.py index 687ce4140..977dc6ad5 100644 --- a/examples/query_tags_example.py +++ b/examples/query_tags_example.py @@ -8,7 +8,7 @@ in the system.query.history table for analytical purposes. There are two ways to set query tags: -1. Session-level: Set in session_configuration (applies to all queries in the session) +1. Connection-level: Pass query_tags parameter to sql.connect() (applies to all queries in the session) 2. Per-query level: Pass query_tags parameter to execute() or execute_async() (applies to specific query) Format: Dictionary with string keys and optional string values @@ -17,21 +17,18 @@ Special cases: - If a value is None, only the key is included (no colon or value) - Special characters (comma, colon and backslash) in values are automatically escaped -- Keys are not escaped (should be controlled identifiers) +- Backslashes in keys are automatically escaped; other special characters in keys are not allowed """ print("=== Query Tags Example ===\n") -# Example 1: Session-level query tags (old approach) -print("Example 1: Session-level query tags") +# Example 1: Connection-level query tags +print("Example 1: Connection-level query tags") with sql.connect( server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), http_path=os.getenv("DATABRICKS_HTTP_PATH"), access_token=os.getenv("DATABRICKS_TOKEN"), - session_configuration={ - 'QUERY_TAGS': 'team:engineering,test:query-tags', - 'ansi_mode': False - } + query_tags={"team": "engineering", "application": "etl"}, ) as connection: with connection.cursor() as cursor: @@ -41,7 +38,7 @@ print() -# Example 2: Per-query query tags (new approach) +# Example 2: Per-query query tags print("Example 2: Per-query query tags") with sql.connect( server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"), diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index efaf6ae4d..2aeea175e 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -36,6 +36,7 @@ ColumnQueue, build_client_context, get_session_config_value, + serialize_query_tags, ) from databricks.sql.parameters.native import ( DbsqlParameterBase, @@ -106,6 +107,7 @@ def __init__( schema: Optional[str] = None, _use_arrow_native_complex_types: Optional[bool] = True, ignore_transactions: bool = True, + query_tags: Optional[Dict[str, Optional[str]]] = None, **kwargs, ) -> None: """ @@ -281,6 +283,15 @@ def read(self) -> Optional[OAuthToken]: "spark.sql.thriftserver.metadata.metricview.enabled" ] = "true" + if query_tags is not None: + if session_configuration is None: + session_configuration = {} + serialized = serialize_query_tags(query_tags) + if serialized: + session_configuration["QUERY_TAGS"] = serialized + else: + session_configuration.pop("QUERY_TAGS", None) + self.disable_pandas = kwargs.get("_disable_pandas", False) self.lz4_compression = kwargs.get("enable_query_result_lz4_compression", True) self.use_cloud_fetch = kwargs.get("use_cloud_fetch", True) diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index 1d70ec4c4..3a43c1a75 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -202,3 +202,24 @@ def test_finalizer_closes_abandoned_connection(self, mock_client_class): close_session_call_args = instance.close_session.call_args[0][0] assert close_session_call_args.guid == b"\x22" assert close_session_call_args.secret == b"\x33" + + @patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME) + def test_query_tags_dict_sets_session_config(self, mock_client_class): + databricks.sql.connect( + query_tags={"team": "data-eng", "project": "etl"}, + **self.DUMMY_CONNECTION_ARGS, + ) + + call_kwargs = mock_client_class.return_value.open_session.call_args[1] + assert call_kwargs["session_configuration"]["QUERY_TAGS"] == "team:data-eng,project:etl" + + @patch("%s.session.ThriftDatabricksClient" % PACKAGE_NAME) + def test_query_tags_dict_takes_precedence_over_session_config(self, mock_client_class): + databricks.sql.connect( + query_tags={"team": "new-team"}, + session_configuration={"QUERY_TAGS": "team:old-team,other:value"}, + **self.DUMMY_CONNECTION_ARGS, + ) + + call_kwargs = mock_client_class.return_value.open_session.call_args[1] + assert call_kwargs["session_configuration"]["QUERY_TAGS"] == "team:new-team" From 12bfd5b7a86408da7ebbb61513dca72f90c50c4b Mon Sep 17 00:00:00 2001 From: Shubhambhusate <99666676+Shubhambhusate@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:29:20 +0530 Subject: [PATCH 2/6] =?UTF-8?q?Fix=20float=20inference=20to=20use=20Double?= =?UTF-8?q?Parameter=20(64-bit)=20instead=20of=20FloatP=E2=80=A6=20(#742)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix float inference to use DoubleParameter (64-bit) instead of FloatParameter (32-bit) Signed-off-by: Shubhambhusate * Add DoubleParameter with Primitive.DOUBLE to test_inference coverage --------- Signed-off-by: Shubhambhusate --- src/databricks/sql/parameters/native.py | 2 +- tests/unit/test_parameters.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/parameters/native.py b/src/databricks/sql/parameters/native.py index b7c448254..d0fb8d82c 100644 --- a/src/databricks/sql/parameters/native.py +++ b/src/databricks/sql/parameters/native.py @@ -659,7 +659,7 @@ def dbsql_parameter_from_primitive( elif isinstance(value, str): return StringParameter(value=value, name=name) elif isinstance(value, float): - return FloatParameter(value=value, name=name) + return DoubleParameter(value=value, name=name) elif isinstance(value, datetime.datetime): return TimestampParameter(value=value, name=name) elif isinstance(value, datetime.date): diff --git a/tests/unit/test_parameters.py b/tests/unit/test_parameters.py index cf2e24951..0588eb499 100644 --- a/tests/unit/test_parameters.py +++ b/tests/unit/test_parameters.py @@ -295,7 +295,8 @@ def test_tspark_param_ordinal(self): (BigIntegerParameter, Primitive.BIGINT), (BooleanParameter, Primitive.BOOL), (DateParameter, Primitive.DATE), - (FloatParameter, Primitive.FLOAT), + (DoubleParameter, Primitive.DOUBLE), + (DoubleParameter, Primitive.FLOAT), (VoidParameter, Primitive.NONE), (TimestampParameter, Primitive.TIMESTAMP), (MapParameter, Primitive.MAP), @@ -305,7 +306,7 @@ def test_tspark_param_ordinal(self): def test_inference(self, _type: TDbsqlParameter, prim: Primitive): """This method only tests inferrable types. - Not tested are TinyIntParameter, SmallIntParameter DoubleParameter and TimestampNTZParameter + Not tested are TinyIntParameter, SmallIntParameter, FloatParameter and TimestampNTZParameter """ inferred_type = dbsql_parameter_from_primitive(prim.value) From 36fb3760b9e1eec552376004ec9494fe6b425ff1 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Tue, 10 Mar 2026 16:29:57 +0530 Subject: [PATCH 3/6] Updated the PyArrow concatenation of tables to use promote_options as default (#751) Updated pyarrow-concat --- src/databricks/sql/utils.py | 2 +- tests/unit/test_cloud_fetch_queue.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index 125edbbaa..b1fff7202 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -895,7 +895,7 @@ def concat_table_chunks( result_table[j].extend(table_chunks[i].column_table[j]) return ColumnTable(result_table, table_chunks[0].column_names) else: - return pyarrow.concat_tables(table_chunks) + return pyarrow.concat_tables(table_chunks, promote_options="default") def serialize_query_tags( diff --git a/tests/unit/test_cloud_fetch_queue.py b/tests/unit/test_cloud_fetch_queue.py index 0c3fc7103..97bb99ad9 100644 --- a/tests/unit/test_cloud_fetch_queue.py +++ b/tests/unit/test_cloud_fetch_queue.py @@ -174,7 +174,7 @@ def test_next_n_rows_more_than_one_table(self, mock_create_next_table): assert ( result == pyarrow.concat_tables( - [self.make_arrow_table(), self.make_arrow_table()] + [self.make_arrow_table(), self.make_arrow_table()],promote_options="default" )[:7] ) @@ -266,7 +266,7 @@ def test_remaining_rows_multiple_tables_fully_returned( assert ( result == pyarrow.concat_tables( - [self.make_arrow_table(), self.make_arrow_table()] + [self.make_arrow_table(), self.make_arrow_table()], promote_options="default" )[3:] ) From ca4d7bcbcd6846bd9be1b079dcd84ce252e525b2 Mon Sep 17 00:00:00 2001 From: Sreekanth Date: Mon, 16 Mar 2026 14:50:33 +0530 Subject: [PATCH 4/6] Add statement-level query_tags support for SEA backend (#754) * Add statement-level query_tags support for SEA backend Signed-off-by: Sreekanth Vadigi * Simplify None handling in query_tags serialization Signed-off-by: Sreekanth Vadigi --------- Signed-off-by: Sreekanth Vadigi --- src/databricks/sql/backend/sea/backend.py | 5 +- .../sql/backend/sea/models/requests.py | 8 ++ tests/unit/test_sea_backend.py | 110 +++++++++++++++++- 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/databricks/sql/backend/sea/backend.py b/src/databricks/sql/backend/sea/backend.py index a6cff4913..ff130cd39 100644 --- a/src/databricks/sql/backend/sea/backend.py +++ b/src/databricks/sql/backend/sea/backend.py @@ -463,9 +463,7 @@ def execute_command( async_op: bool, enforce_embedded_schema_correctness: bool, row_limit: Optional[int] = None, - query_tags: Optional[ - Dict[str, Optional[str]] - ] = None, # TODO: implement query_tags for SEA backend + query_tags: Optional[Dict[str, Optional[str]]] = None, ) -> Union[SeaResultSet, None]: """ Execute a SQL command using the SEA backend. @@ -532,6 +530,7 @@ def execute_command( row_limit=row_limit, parameters=sea_parameters if sea_parameters else None, result_compression=result_compression, + query_tags=query_tags, ) response_data = self._http_client._make_request( diff --git a/src/databricks/sql/backend/sea/models/requests.py b/src/databricks/sql/backend/sea/models/requests.py index ad046ff54..eb156fb1a 100644 --- a/src/databricks/sql/backend/sea/models/requests.py +++ b/src/databricks/sql/backend/sea/models/requests.py @@ -31,6 +31,7 @@ class ExecuteStatementRequest: wait_timeout: str = "10s" on_wait_timeout: str = "CONTINUE" row_limit: Optional[int] = None + query_tags: Optional[Dict[str, Optional[str]]] = None def to_dict(self) -> Dict[str, Any]: """Convert the request to a dictionary for JSON serialization.""" @@ -60,6 +61,13 @@ def to_dict(self) -> Dict[str, Any]: for param in self.parameters ] + # SEA API expects query_tags as an array of {key, value} objects. + # None/empty values are left to the server to handle as key-only tags. + if self.query_tags: + result["query_tags"] = [ + {"key": k, "value": v} for k, v in self.query_tags.items() + ] + return result diff --git a/tests/unit/test_sea_backend.py b/tests/unit/test_sea_backend.py index 26a898cb8..f71e60943 100644 --- a/tests/unit/test_sea_backend.py +++ b/tests/unit/test_sea_backend.py @@ -185,7 +185,7 @@ def test_session_management(self, sea_client, mock_http_client, thrift_session_i session_config = { "ANSI_MODE": "FALSE", # Supported parameter "STATEMENT_TIMEOUT": "3600", # Supported parameter - "QUERY_TAGS": "team:marketing,dashboard:abc123", # Supported parameter + "QUERY_TAGS": "team:marketing,dashboard:abc123", # Supported parameter "unsupported_param": "value", # Unsupported parameter } catalog = "test_catalog" @@ -197,7 +197,7 @@ def test_session_management(self, sea_client, mock_http_client, thrift_session_i "session_confs": { "ansi_mode": "FALSE", "statement_timeout": "3600", - "query_tags": "team:marketing,dashboard:abc123", + "query_tags": "team:marketing,dashboard:abc123", }, "catalog": catalog, "schema": schema, @@ -416,6 +416,112 @@ def test_command_execution_advanced( ) assert "Command failed" in str(excinfo.value) + def _execute_response(self): + return { + "statement_id": "test-statement-123", + "status": {"state": "SUCCEEDED"}, + "manifest": {"schema": [], "total_row_count": 0, "total_byte_count": 0}, + "result": {"data": []}, + } + + def _run_execute_command(self, sea_client, sea_session_id, mock_cursor, **kwargs): + """Helper to invoke execute_command with default args.""" + return sea_client.execute_command( + operation="SELECT 1", + session_id=sea_session_id, + max_rows=100, + max_bytes=1000, + lz4_compression=False, + cursor=mock_cursor, + use_cloud_fetch=False, + parameters=[], + async_op=False, + enforce_embedded_schema_correctness=False, + **kwargs, + ) + + def test_execute_command_query_tags_string_values( + self, sea_client, mock_http_client, mock_cursor, sea_session_id + ): + """query_tags with string values are included in the request payload.""" + mock_http_client._make_request.return_value = self._execute_response() + with patch.object(sea_client, "_response_to_result_set"): + self._run_execute_command( + sea_client, + sea_session_id, + mock_cursor, + query_tags={"env": "prod", "team": "data"}, + ) + _, kwargs = mock_http_client._make_request.call_args + assert kwargs["data"]["query_tags"] == [ + {"key": "env", "value": "prod"}, + {"key": "team", "value": "data"}, + ] + + def test_execute_command_query_tags_none_value( + self, sea_client, mock_http_client, mock_cursor, sea_session_id + ): + """query_tags with a None value omit the value field (key-only tag).""" + mock_http_client._make_request.return_value = self._execute_response() + with patch.object(sea_client, "_response_to_result_set"): + self._run_execute_command( + sea_client, + sea_session_id, + mock_cursor, + query_tags={"env": "prod", "team": None}, + ) + _, kwargs = mock_http_client._make_request.call_args + assert kwargs["data"]["query_tags"] == [ + {"key": "env", "value": "prod"}, + {"key": "team", "value": None}, + ] + + def test_execute_command_no_query_tags_omitted( + self, sea_client, mock_http_client, mock_cursor, sea_session_id + ): + """query_tags field is absent from the request when not provided.""" + mock_http_client._make_request.return_value = self._execute_response() + with patch.object(sea_client, "_response_to_result_set"): + self._run_execute_command(sea_client, sea_session_id, mock_cursor) + _, kwargs = mock_http_client._make_request.call_args + assert "query_tags" not in kwargs["data"] + + def test_execute_command_empty_query_tags_omitted( + self, sea_client, mock_http_client, mock_cursor, sea_session_id + ): + """Empty query_tags dict is treated as absent — field omitted from request.""" + mock_http_client._make_request.return_value = self._execute_response() + with patch.object(sea_client, "_response_to_result_set"): + self._run_execute_command( + sea_client, sea_session_id, mock_cursor, query_tags={} + ) + _, kwargs = mock_http_client._make_request.call_args + assert "query_tags" not in kwargs["data"] + + def test_execute_command_async_query_tags( + self, sea_client, mock_http_client, mock_cursor, sea_session_id + ): + """query_tags are included in async execute requests (execute_async path).""" + mock_http_client._make_request.return_value = { + "statement_id": "test-statement-async", + "status": {"state": "PENDING"}, + } + sea_client.execute_command( + operation="SELECT 1", + session_id=sea_session_id, + max_rows=100, + max_bytes=1000, + lz4_compression=False, + cursor=mock_cursor, + use_cloud_fetch=False, + parameters=[], + async_op=True, + enforce_embedded_schema_correctness=False, + query_tags={"job": "nightly-etl"}, + ) + _, kwargs = mock_http_client._make_request.call_args + assert kwargs["data"]["query_tags"] == [{"key": "job", "value": "nightly-etl"}] + def test_command_management( self, sea_client, From 330c4454d2425a259f600b2126e01b9ce408274d Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Fri, 27 Mar 2026 15:14:42 +0530 Subject: [PATCH 5/6] Harden CI/CD workflows: fix secret exposure, script injection, and pin all actions to SHA (#762) Updaed the security --- .github/workflows/code-coverage.yml | 21 ++++++++------ .github/workflows/code-quality-checks.yml | 35 ++++++++++++----------- .github/workflows/daily-telemetry-e2e.yml | 13 +++++---- .github/workflows/dco-check.yml | 12 ++++---- .github/workflows/integration.yml | 21 ++++++++------ .github/workflows/publish-manual.yml | 9 ++++-- .github/workflows/publish-test.yml | 19 ++++++++---- .github/workflows/publish.yml | 14 +++++---- 8 files changed, 86 insertions(+), 58 deletions(-) diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index 5c961757e..9250d0ca5 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -20,14 +20,12 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 - ref: ${{ github.event.pull_request.head.ref || github.ref_name }} - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" #---------------------------------------------- @@ -41,7 +39,7 @@ jobs: # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -53,7 +51,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -105,8 +103,10 @@ jobs: #---------------------------------------------- - name: Check for coverage override id: override + env: + PR_BODY: ${{ github.event.pull_request.body }} run: | - OVERRIDE_COMMENT=$(echo "${{ github.event.pull_request.body }}" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "") + OVERRIDE_COMMENT=$(echo "$PR_BODY" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "") if [ -n "$OVERRIDE_COMMENT" ]; then echo "override=true" >> $GITHUB_OUTPUT REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/') @@ -153,9 +153,12 @@ jobs: # coverage enforcement summary #---------------------------------------------- - name: Coverage enforcement summary + env: + OVERRIDE: ${{ steps.override.outputs.override }} + REASON: ${{ steps.override.outputs.reason }} run: | - if [ "${{ steps.override.outputs.override }}" == "true" ]; then - echo "⚠️ Coverage checks bypassed: ${{ steps.override.outputs.reason }}" + if [ "$OVERRIDE" == "true" ]; then + echo "⚠️ Coverage checks bypassed: $REASON" echo "Please ensure this override is justified and temporary" else echo "✅ Coverage checks enforced - minimum 85% required" diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index cc3952920..13a889c56 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -2,6 +2,9 @@ name: Code Quality Checks on: [pull_request] +permissions: + contents: read + jobs: run-unit-tests: runs-on: ubuntu-latest @@ -23,17 +26,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -45,7 +48,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -107,17 +110,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@v2 + uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -129,7 +132,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv-pyarrow key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -185,17 +188,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -207,7 +210,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -238,17 +241,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -260,7 +263,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} diff --git a/.github/workflows/daily-telemetry-e2e.yml b/.github/workflows/daily-telemetry-e2e.yml index d60b7f5a9..c3d0da5df 100644 --- a/.github/workflows/daily-telemetry-e2e.yml +++ b/.github/workflows/daily-telemetry-e2e.yml @@ -12,6 +12,9 @@ on: default: 'tests/e2e/test_telemetry_e2e.py' type: string +permissions: + contents: read + jobs: telemetry-e2e-tests: runs-on: ubuntu-latest @@ -29,11 +32,11 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" @@ -41,7 +44,7 @@ jobs: # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -53,7 +56,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -82,7 +85,7 @@ jobs: #---------------------------------------------- - name: Upload test results on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: telemetry-test-results path: | diff --git a/.github/workflows/dco-check.yml b/.github/workflows/dco-check.yml index 050665ec9..770ffb9ec 100644 --- a/.github/workflows/dco-check.yml +++ b/.github/workflows/dco-check.yml @@ -2,17 +2,19 @@ name: DCO Check on: [pull_request] +permissions: + contents: read + pull-requests: write + jobs: check: - runs-on: - group: databricks-protected-runner-group - labels: linux-ubuntu-latest + runs-on: ubuntu-latest steps: - name: Check for DCO id: dco-check - uses: tisonkun/actions-dco@v1.1 + uses: tisonkun/actions-dco@6d1f8a197db1b04df1769707b46b9366b1eca902 # v1.1 - name: Comment about DCO status - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 if: ${{ failure() }} with: script: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c915ee6c1..49dedfd91 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,7 +4,10 @@ on: push: branches: - main - pull_request: + pull_request: + +permissions: + contents: read jobs: run-non-telemetry-tests: @@ -21,17 +24,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -43,7 +46,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -79,10 +82,10 @@ jobs: DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - name: Install system dependencies @@ -90,14 +93,14 @@ jobs: sudo apt-get update sudo apt-get install -y libkrb5-dev - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml index 2f2a7a4dd..1802664a0 100644 --- a/.github/workflows/publish-manual.yml +++ b/.github/workflows/publish-manual.yml @@ -4,6 +4,9 @@ name: Publish to PyPI Manual [Production] on: workflow_dispatch: {} +permissions: + contents: read + jobs: publish: name: Publish @@ -14,14 +17,14 @@ jobs: # Step 1: Check out the repository code #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v2 # Check out the repository to access the code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 #---------------------------------------------- # Step 2: Set up Python environment #---------------------------------------------- - name: Set up python id: setup-python - uses: actions/setup-python@v2 + uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: 3.9 # Specify the Python version to be used @@ -29,7 +32,7 @@ jobs: # Step 3: Install and configure Poetry #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 # Install Poetry, the Python package manager + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true diff --git a/.github/workflows/publish-test.yml b/.github/workflows/publish-test.yml index 97a444e68..e967b8085 100644 --- a/.github/workflows/publish-test.yml +++ b/.github/workflows/publish-test.yml @@ -1,5 +1,12 @@ name: Publish to PyPI [Test] -on: [push] +on: + push: + branches: + - main + +permissions: + contents: read + jobs: test-pypi: name: Create patch version number and push to test-pypi @@ -9,17 +16,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -30,7 +37,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -48,7 +55,7 @@ jobs: # Get the current version and increment it (test-pypi requires a unique version number) #---------------------------------------------- - name: Get next version - uses: reecetech/version-increment@2022.2.4 + uses: reecetech/version-increment@ddbbe72b7f76a996076fabfdce21a16384e8644a # 2022.2.4 id: version with: scheme: semver diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b101f421c..0fd04d992 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,6 +2,10 @@ name: Publish to PyPI [Production] on: release: types: [published] + +permissions: + contents: read + jobs: publish: name: Publish @@ -11,17 +15,17 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Set up python id: setup-python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: 3.9 #---------------------------------------------- # ----- install & configure poetry ----- #---------------------------------------------- - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 with: version: "2.2.1" virtualenvs-create: true @@ -32,7 +36,7 @@ jobs: #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} @@ -50,7 +54,7 @@ jobs: # Here we use version-increment to fetch the latest tagged version (we won't increment it though) #------------------------------------------------------------------------------------------------ - name: Get next version - uses: reecetech/version-increment@2022.2.4 + uses: reecetech/version-increment@ddbbe72b7f76a996076fabfdce21a16384e8644a # 2022.2.4 id: version with: scheme: semver From 47933533071c11141b3e94248863644aa1735989 Mon Sep 17 00:00:00 2001 From: Jothi Prakash Date: Mon, 30 Mar 2026 09:59:34 +0530 Subject: [PATCH 6/6] Removed Publish workflow (#764) --- .github/workflows/publish-manual.yml | 90 ---------------------------- .github/workflows/publish-test.yml | 82 ------------------------- .github/workflows/publish.yml | 76 ----------------------- 3 files changed, 248 deletions(-) delete mode 100644 .github/workflows/publish-manual.yml delete mode 100644 .github/workflows/publish-test.yml delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish-manual.yml b/.github/workflows/publish-manual.yml deleted file mode 100644 index 1802664a0..000000000 --- a/.github/workflows/publish-manual.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Publish to PyPI Manual [Production] - -# Allow manual triggering of the workflow -on: - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - publish: - name: Publish - runs-on: ubuntu-latest - - steps: - #---------------------------------------------- - # Step 1: Check out the repository code - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - - #---------------------------------------------- - # Step 2: Set up Python environment - #---------------------------------------------- - - name: Set up python - id: setup-python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 - with: - python-version: 3.9 # Specify the Python version to be used - - #---------------------------------------------- - # Step 3: Install and configure Poetry - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - #---------------------------------------------- - # Step 3.5: Install Kerberos system dependencies - #---------------------------------------------- - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - -# #---------------------------------------------- -# # Step 4: Load cached virtual environment (if available) -# #---------------------------------------------- -# - name: Load cached venv -# id: cached-poetry-dependencies -# uses: actions/cache@v2 -# with: -# path: .venv # Path to the virtual environment -# key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} -# # Cache key is generated based on OS, Python version, repo name, and the `poetry.lock` file hash - -# #---------------------------------------------- -# # Step 5: Install dependencies if the cache is not found -# #---------------------------------------------- -# - name: Install dependencies -# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' # Only run if the cache was not hit -# run: poetry install --no-interaction --no-root # Install dependencies without interaction - -# #---------------------------------------------- -# # Step 6: Update the version to the manually provided version -# #---------------------------------------------- -# - name: Update pyproject.toml with the specified version -# run: poetry version ${{ github.event.inputs.version }} # Use the version provided by the user input - - #---------------------------------------------- - # Step 7: Build and publish the first package to PyPI - #---------------------------------------------- - - name: Build and publish databricks sql connector to PyPI - working-directory: ./databricks_sql_connector - run: | - poetry build - poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }} # Publish with PyPI token - #---------------------------------------------- - # Step 7: Build and publish the second package to PyPI - #---------------------------------------------- - - - name: Build and publish databricks sql connector core to PyPI - working-directory: ./databricks_sql_connector_core - run: | - poetry build - poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }} # Publish with PyPI token \ No newline at end of file diff --git a/.github/workflows/publish-test.yml b/.github/workflows/publish-test.yml deleted file mode 100644 index e967b8085..000000000 --- a/.github/workflows/publish-test.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Publish to PyPI [Test] -on: - push: - branches: - - main - -permissions: - contents: read - -jobs: - test-pypi: - name: Create patch version number and push to test-pypi - runs-on: ubuntu-latest - steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.10" - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - #---------------------------------------------- - # Get the current version and increment it (test-pypi requires a unique version number) - #---------------------------------------------- - - name: Get next version - uses: reecetech/version-increment@ddbbe72b7f76a996076fabfdce21a16384e8644a # 2022.2.4 - id: version - with: - scheme: semver - increment: patch - #---------------------------------------------- - # Tell poetry to update the version number - #---------------------------------------------- - - name: Update pyproject.toml - run: poetry version ${{ steps.version.outputs.major-version }}.${{ steps.version.outputs.minor-version }}.dev$(date +%s) - #---------------------------------------------- - # Build the package (before publish action) - #---------------------------------------------- - - name: Build package - run: poetry build - #---------------------------------------------- - # Configure test-pypi repository - #---------------------------------------------- - - name: Configure test-pypi repository - run: poetry config repositories.testpypi https://test.pypi.org/legacy/ - #---------------------------------------------- - # Attempt push to test-pypi - #---------------------------------------------- - - name: Publish to test-pypi - run: poetry publish --username __token__ --password ${{ secrets.TEST_PYPI_TOKEN }} --repository testpypi diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 0fd04d992..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Publish to PyPI [Production] -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - publish: - name: Publish - runs-on: ubuntu-latest - steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: 3.9 - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install Kerberos system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libkrb5-dev - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - #------------------------------------------------------------------------------------------------ - # Here we use version-increment to fetch the latest tagged version (we won't increment it though) - #------------------------------------------------------------------------------------------------ - - name: Get next version - uses: reecetech/version-increment@ddbbe72b7f76a996076fabfdce21a16384e8644a # 2022.2.4 - id: version - with: - scheme: semver - increment: patch - #----------------------------------------------------------------------------- - # Tell poetry to use the `current-version` that was found by the previous step - #----------------------------------------------------------------------------- - - name: Update pyproject.toml - run: poetry version ${{ steps.version.outputs.current-version }} - #---------------------------------------------- - # Build the package (before publish) - #---------------------------------------------- - - name: Build package - run: poetry build - #---------------------------------------------- - # Publish to pypi - #---------------------------------------------- - - name: Publish to pypi - run: poetry publish --username __token__ --password ${{ secrets.PROD_PYPI_TOKEN }}