-
Notifications
You must be signed in to change notification settings - Fork 141
Expand file tree
/
Copy pathtest_query_as_arrow.py
More file actions
115 lines (87 loc) · 3.87 KB
/
Copy pathtest_query_as_arrow.py
File metadata and controls
115 lines (87 loc) · 3.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""Unit tests for FelderaClient.query_as_arrow and Pipeline.query_arrow."""
import io
from unittest.mock import MagicMock
import pyarrow as pa
import pyarrow.ipc as ipc
import pytest
from feldera.rest.feldera_client import FelderaClient
def _make_ipc_bytes(table: pa.Table) -> bytes:
"""Serialise a ``pyarrow.Table`` to Arrow IPC stream bytes."""
buf = io.BytesIO()
with ipc.new_stream(buf, table.schema) as writer:
if table.num_rows > 0:
writer.write_table(table)
return buf.getvalue()
def _mock_response(ipc_bytes: bytes) -> MagicMock:
"""Return a mock response whose ``raw`` is an Arrow IPC byte stream."""
resp = MagicMock()
resp.raw = io.BytesIO(ipc_bytes)
return resp
@pytest.fixture()
def client() -> FelderaClient:
"""A ``FelderaClient`` with a mocked HTTP layer (no real network calls)."""
c = FelderaClient.__new__(FelderaClient)
c.http = MagicMock()
return c
class TestQueryAsArrow:
def test_non_empty_result_yields_correct_data(self, client: FelderaClient):
schema = pa.schema([("id", pa.int64()), ("name", pa.utf8())])
expected = pa.table({"id": [1, 2, 3], "name": ["a", "b", "c"]}, schema=schema)
client.http.get.return_value = _mock_response(_make_ipc_bytes(expected))
batches = list(client.query_as_arrow("my_pipeline", "SELECT id, name FROM t"))
result = pa.Table.from_batches(batches, schema=schema)
assert len(batches) > 0
assert result.schema == schema
assert result.num_rows == 3
assert result.column("id").to_pylist() == [1, 2, 3]
assert result.column("name").to_pylist() == ["a", "b", "c"]
def test_http_called_with_correct_params(self, client: FelderaClient):
schema = pa.schema([("id", pa.int64())])
table = pa.table({"id": [42]}, schema=schema)
client.http.get.return_value = _mock_response(_make_ipc_bytes(table))
list(client.query_as_arrow("my_pipeline", "SELECT id FROM t"))
client.http.get.assert_called_once_with(
path="/pipelines/my_pipeline/query",
params={
"pipeline_name": "my_pipeline",
"sql": "SELECT id FROM t",
"format": "arrow_ipc",
},
stream=True,
)
def test_empty_result_yields_no_batches(self, client: FelderaClient):
schema = pa.schema([("id", pa.int64()), ("value", pa.float64())])
empty = pa.table(
{
"id": pa.array([], type=pa.int64()),
"value": pa.array([], type=pa.float64()),
},
schema=schema,
)
client.http.get.return_value = _mock_response(_make_ipc_bytes(empty))
result_batches = list(
client.query_as_arrow("my_pipeline", "SELECT id, value FROM t WHERE false")
)
assert result_batches == []
def test_response_closed_after_full_consumption(self, client: FelderaClient):
schema = pa.schema([("id", pa.int64())])
table = pa.table({"id": [1, 2]}, schema=schema)
resp = _mock_response(_make_ipc_bytes(table))
client.http.get.return_value = resp
list(client.query_as_arrow("my_pipeline", "SELECT id FROM t"))
resp.close.assert_called_once()
class TestPipelineQueryArrow:
def test_query_arrow_delegates_to_client(self):
"""Pipeline.query_arrow must forward to client.query_as_arrow."""
from feldera.pipeline import Pipeline
pipeline = Pipeline.__new__(Pipeline)
pipeline._inner = MagicMock()
pipeline._inner.name = "pipe1"
pipeline.client = MagicMock()
expected = object()
pipeline.client.query_as_arrow.return_value = expected
result = pipeline.query_arrow("SELECT x FROM v")
pipeline.client.query_as_arrow.assert_called_once_with(
"pipe1", "SELECT x FROM v"
)
assert result is expected