Skip to content

Commit 9e969e2

Browse files
mateusz834mateusz834
authored
Run Agentic SDK tests in CI (#35)
This change enables Agentic SDK tests to be run in CI, additionally it fixes system tests, these were broken, since the message refactor. Our CI now uses our internal OpenAI-compatible model, before GA we might also add support for OpenAI/Ollama for CI testing, such that external developers can also run our test suite.
1 parent 47b01d5 commit 9e969e2

File tree

11 files changed

+419
-344
lines changed

11 files changed

+419
-344
lines changed

‎.env.template‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,9 @@ version=9.0
1414
#splunkToken="<Bearer-token>"
1515
# Session key for authentication
1616
#token="<Session-Key>"
17+
18+
#internal_ai_client_id=""
19+
#internal_ai_client_secret=""
20+
#internal_ai_app_key=""
21+
#internal_ai_token_url=""
22+
#internal_ai_base_url=""

‎.github/workflows/test.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,22 @@ jobs:
1919
with:
2020
python-version: ${{ matrix.python-version }}
2121
- name: Install dependencies
22-
run: python -m pip install . --group test
22+
run: python -m pip install '.[openai]' --group test
2323
- name: Set up .env
2424
run: cp .env.template .env
25+
- name: Write internal AI secrets to .env
26+
run: |
27+
echo "internal_ai_app_key=$INTERNAL_AI_APP_KEY" >> .env
28+
echo "internal_ai_client_id=$INTERNAL_AI_CLIENT_ID" >> .env
29+
echo "internal_ai_client_secret=$INTERNAL_AI_CLIENT_SECRET" >> .env
30+
echo "internal_ai_token_url=$INTERNAL_AI_TOKEN_URL" >> .env
31+
echo "internal_ai_base_url=$INTERNAL_AI_BASE_URL" >> .env
32+
env:
33+
INTERNAL_AI_APP_KEY: ${{ secrets.INTERNAL_AI_APP_KEY }}
34+
INTERNAL_AI_CLIENT_ID: ${{ secrets.INTERNAL_AI_CLIENT_ID }}
35+
INTERNAL_AI_CLIENT_SECRET: ${{ secrets.INTERNAL_AI_CLIENT_SECRET }}
36+
INTERNAL_AI_TOKEN_URL: ${{ secrets.INTERNAL_AI_TOKEN_URL }}
37+
INTERNAL_AI_BASE_URL: ${{ secrets.INTERNAL_AI_BASE_URL }}
2538
- name: Run unit tests
2639
run: make test-unit
2740
- name: Run entire test suite

‎tests/ai_test_model.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import collections.abc
2+
from typing import override
3+
4+
import httpx
5+
from httpx import Auth, Request, Response
6+
from pydantic import BaseModel
7+
8+
from splunklib.ai import OpenAIModel
9+
from splunklib.ai.model import PredefinedModel
10+
11+
12+
class InternalAIModel(BaseModel):
13+
client_id: str
14+
client_secret: str
15+
app_key: str
16+
17+
token_url: str
18+
base_url: str
19+
20+
21+
class TestLLMSettings(BaseModel):
22+
# TODO: Currently we only support our internal OpenAI-compatible model,
23+
# once we are close to GA we should also support OpenAI and probably Ollama, such
24+
# that external developers can also run our test suite suite locally.
25+
internal_ai: InternalAIModel | None = None
26+
27+
28+
async def create_model(s: TestLLMSettings) -> PredefinedModel:
29+
if s.internal_ai is not None:
30+
return await _buildInternalAIModel(
31+
token_url=s.internal_ai.token_url,
32+
base_url=s.internal_ai.base_url,
33+
client_id=s.internal_ai.client_id,
34+
client_secret=s.internal_ai.client_secret,
35+
app_key=s.internal_ai.app_key,
36+
)
37+
raise Exception("unreachable")
38+
39+
40+
class _InternalAIAuth(Auth):
41+
token: str
42+
43+
def __init__(self, token: str) -> None:
44+
self.token = token
45+
46+
@override
47+
def auth_flow(
48+
self, request: Request
49+
) -> collections.abc.Generator[Request, Response, None]:
50+
request.headers["api-key"] = self.token
51+
yield request
52+
53+
54+
class _TokenResponse(BaseModel):
55+
access_token: str
56+
57+
58+
async def _buildInternalAIModel(
59+
token_url: str,
60+
base_url: str,
61+
client_id: str,
62+
client_secret: str,
63+
app_key: str,
64+
) -> OpenAIModel:
65+
headers = {
66+
"Accept": "*/*",
67+
"Content-Type": "application/x-www-form-urlencoded",
68+
}
69+
70+
http = httpx.AsyncClient()
71+
response = await http.post(
72+
url=token_url,
73+
headers=headers,
74+
data={"grant_type": "client_credentials"},
75+
auth=(client_id, client_secret),
76+
)
77+
78+
token = _TokenResponse.model_validate_json(response.text).access_token
79+
80+
auth_handler = _InternalAIAuth(token)
81+
model = "gpt-4.1"
82+
83+
return OpenAIModel(
84+
model=model,
85+
base_url=f"{base_url}/{model}",
86+
api_key="", # unused
87+
extra_body={"user": f'{{"appkey":"{app_key}"}}'},
88+
httpx_client=httpx.AsyncClient(auth=auth_handler),
89+
)

‎tests/ai_testlib.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
from splunklib.ai.model import PredefinedModel
2+
from tests.ai_test_model import InternalAIModel, TestLLMSettings, create_model
3+
from tests.testlib import SDKTestCase
4+
5+
6+
class AITestCase(SDKTestCase):
7+
_model: PredefinedModel | None = None
8+
9+
@property
10+
def test_llm_settings(self) -> TestLLMSettings:
11+
client_id: str = self.opts.kwargs["internal_ai_client_id"]
12+
client_secret: str = self.opts.kwargs["internal_ai_client_secret"]
13+
app_key: str = self.opts.kwargs["internal_ai_app_key"]
14+
token_url: str = self.opts.kwargs["internal_ai_token_url"]
15+
base_url: str = self.opts.kwargs["internal_ai_base_url"]
16+
return TestLLMSettings(
17+
internal_ai=InternalAIModel(
18+
client_id=client_id,
19+
client_secret=client_secret,
20+
app_key=app_key,
21+
token_url=token_url,
22+
base_url=base_url,
23+
)
24+
)
25+
26+
async def model(self) -> PredefinedModel:
27+
if self._model is not None:
28+
return self._model
29+
30+
model = await create_model(self.test_llm_settings)
31+
self._model = model
32+
return model

tests/cretestlib.py renamed to tests/cre_testlib.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,17 @@
1919
from abc import abstractmethod
2020
from http.cookies import SimpleCookie
2121

22+
from splunklib.ai.model import PredefinedModel
2223
from splunklib.binding import _spliturl
2324
from splunklib.client import Service, connect
25+
from tests.ai_test_model import TestLLMSettings, create_model
2426

2527
try:
2628
import splunk
2729

2830
class CRETestHandler(splunk.rest.BaseRestHandler):
2931
_service: Service | None = None
32+
_model: PredefinedModel | None = None
3033

3134
def handle_POST(self) -> None:
3235
async def run() -> None:
@@ -45,6 +48,16 @@ async def run() -> None:
4548
@abstractmethod
4649
async def run(self) -> None: ...
4750

51+
async def model(self) -> PredefinedModel:
52+
if self._model is not None:
53+
return self._model
54+
55+
raw_body = str(self.request["payload"])
56+
s = TestLLMSettings.model_validate_json(raw_body)
57+
model = await create_model(s)
58+
self._model = model
59+
return model
60+
4861
@property
4962
def service(self) -> Service:
5063
if self._service is not None:

0 commit comments

Comments
 (0)