Skip to content

Commit 302befb

Browse files
authored
Add E2E tests - Agent used inside of an App (#17)
This change adds few E2E tests, that make sure that out code works inside of an App properly. Such tests are defined using a Custom REST endpoint, as it is the simplest way to execute some custom code on demand in the App.
1 parent 1d2cd11 commit 302befb

File tree

17 files changed

+513
-44
lines changed

17 files changed

+513
-44
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,4 +278,6 @@ $RECYCLE.BIN/
278278
# End of https://www.toptal.com/developers/gitignore/api/windows,macos,linux,pycharm+all,python
279279

280280
.vscode/
281-
docs/_build/
281+
docs/_build/
282+
283+
/deps

Dockerfile

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
ARG SPLUNK_VERSION=latest
2+
FROM splunk/splunk:${SPLUNK_VERSION}
3+
4+
USER root
5+
6+
RUN mkdir /tmp/sdk
7+
COPY ./pyproject.toml /tmp/sdk/pyproject.toml
8+
COPY ./uv.lock /tmp/sdk/uv.lock
9+
COPY ./splunklib /tmp/sdk/splunklib
10+
11+
RUN mkdir /splunklib-deps
12+
RUN chown splunk:splunk /splunklib-deps
13+
RUN chown -R splunk:splunk /tmp/sdk
14+
RUN chown splunk:splunk /tmp/sdk
15+
16+
USER splunk
17+
18+
WORKDIR /tmp/sdk
19+
20+
RUN /opt/splunk/bin/python3.13 -m venv .venv
21+
RUN /bin/bash -c "source .venv/bin/activate && LD_LIBRARY_PATH=/opt/splunk/lib python -m pip install '.[openai]' --target=/splunklib-deps"
22+
23+
USER ${ANSIBLE_USER}
24+
WORKDIR /opt/splunk

Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ test-integration:
1818

1919
.PHONY: docker-up
2020
docker-up:
21-
@docker-compose up -d
21+
@DOCKER_BUILDKIT=0 docker-compose up -d --build
2222

2323
.PHONY: docker-ensure-up
2424
docker-ensure-up:
@@ -45,4 +45,4 @@ docker-remove:
4545
@docker-compose rm -f -s
4646

4747
.PHONY: docker-refresh
48-
docker-refresh: docker-remove docker-start
48+
docker-refresh: docker-remove docker-start

docker-compose.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
services:
22
splunk:
3-
image: "splunk/splunk:${SPLUNK_VERSION}"
3+
build:
4+
context: .
5+
dockerfile: Dockerfile
6+
platform: linux/amd64
47
container_name: splunk
58
environment:
69
- SPLUNK_START_ARGS=--accept-license
@@ -24,8 +27,14 @@ services:
2427
- "./tests/system/test_apps/streaming_app:/opt/splunk/etc/apps/streaming_app"
2528
- "./tests/system/test_apps/modularinput_app:/opt/splunk/etc/apps/modularinput_app"
2629
- "./tests/system/test_apps/cre_app:/opt/splunk/etc/apps/cre_app"
30+
- "./tests/system/test_apps/ai_agentic_test_app:/opt/splunk/etc/apps/ai_agentic_test_app"
31+
- "./tests/system/test_apps/ai_agentic_test_local_tools_app:/opt/splunk/etc/apps/ai_agentic_test_local_tools_app"
2732
- "./splunklib:/opt/splunk/etc/apps/eventing_app/bin/splunklib"
2833
- "./splunklib:/opt/splunk/etc/apps/generating_app/bin/splunklib"
2934
- "./splunklib:/opt/splunk/etc/apps/reporting_app/bin/splunklib"
3035
- "./splunklib:/opt/splunk/etc/apps/streaming_app/bin/splunklib"
3136
- "./splunklib:/opt/splunk/etc/apps/modularinput_app/bin/splunklib"
37+
- "./splunklib:/opt/splunk/etc/apps/ai_agentic_test_app/bin/lib/splunklib"
38+
- "./splunklib:/opt/splunk/etc/apps/ai_agentic_test_local_tools_app/bin/lib/splunklib"
39+
- "./tests:/opt/splunk/etc/apps/ai_agentic_test_app/bin/lib/tests"
40+
- "./tests:/opt/splunk/etc/apps/ai_agentic_test_local_tools_app/bin/lib/tests"

splunklib/ai/agent.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@
2525
from splunklib.ai.model import PredefinedModel
2626
from splunklib.ai.tools import load_mcp_tools, locate_tools_path_by_sdk_location
2727
from splunklib.ai.types import (
28+
AgentResponse,
2829
BaseAgent,
2930
Message,
30-
AgentResponse,
3131
OutputT,
3232
StopConditions,
3333
Tool,
@@ -42,11 +42,13 @@ class Agent(BaseAgent[OutputT], AbstractAsyncContextManager):
4242
_use_mcp_tools: bool
4343
_service: Service | None = None
4444

45+
# TODO: We should have a logger inside of an agent, debugging and such.
46+
4547
def __init__(
4648
self,
4749
model: PredefinedModel,
4850
system_prompt: str,
49-
use_mcp_tools: bool = False,
51+
use_mcp_tools: bool = False, # TODO: should we default to True?
5052
service: Service | None = None, # TODO: make it non-optional.
5153
agents: Sequence[BaseAgent[BaseModel | None]] | None = None,
5254
output_schema: type[OutputT] | None = None,
@@ -72,6 +74,8 @@ def __init__(
7274

7375
@override
7476
async def __aenter__(self):
77+
# TODO: replace these with if && raise
78+
# See: https://docs.python.org/3/reference/simple_stmts.html#the-assert-statement
7579
assert self._impl is None, "Agent is already in `async with` context"
7680

7781
if self._use_mcp_tools:

splunklib/ai/tools.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,21 @@ class RemoteCfg:
7676

7777
@asynccontextmanager
7878
async def _connect_local_mcp(cfg: LocalCfg):
79-
server_params = StdioServerParameters(command=sys.executable, args=[cfg.tools_path])
79+
server_params = StdioServerParameters(
80+
command=sys.executable,
81+
args=[cfg.tools_path],
82+
)
83+
84+
# Splunk starts processes with a custom LD_LIBRARY_PATH env var, the mcp lib
85+
# does not forward all env, but few restricted ones by default. If we don't do
86+
# so then the shared object that python loads would fail to succeed.
87+
# TODO: If needed we might in future pass all env vars, but we would have to investigate why
88+
# the mcp lib did that filtering in the first place. For now we only allow additionally
89+
# the LD_LIBRARY_PATH.
90+
ld = os.environ.get("LD_LIBRARY_PATH")
91+
if ld is not None:
92+
server_params.env = {"LD_LIBRARY_PATH": ld}
93+
8094
async with stdio_client(server_params) as (read, write):
8195
async with ClientSession(read, write) as session:
8296
await session.initialize()

tests/__init__.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright © 2011-2026 Splunk, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"): you may
4+
# not use this file except in compliance with the License. You may obtain
5+
# a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12+
# License for the specific language governing permissions and limitations
13+
# under the License.

tests/cretestlib.py

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#
2+
# Copyright © 2011-2026 Splunk, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"): you may
5+
# not use this file except in compliance with the License. You may obtain
6+
# a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations
14+
# under the License.
15+
16+
import asyncio
17+
import base64
18+
import traceback
19+
from abc import abstractmethod
20+
from http.cookies import SimpleCookie
21+
22+
from splunklib.binding import _spliturl
23+
from splunklib.client import Service, connect
24+
25+
try:
26+
import splunk
27+
28+
class CRETestHandler(splunk.rest.BaseRestHandler):
29+
_service: Service | None = None
30+
31+
def handle_POST(self) -> None:
32+
async def run() -> None:
33+
try:
34+
await self.run()
35+
except Exception:
36+
trace = traceback.format_exc()
37+
self.response.setStatus(500)
38+
self.response.write(trace)
39+
return
40+
41+
self.response.setStatus(200)
42+
43+
asyncio.run(run())
44+
45+
@abstractmethod
46+
async def run(self) -> None: ...
47+
48+
@property
49+
def service(self) -> Service:
50+
if self._service is not None:
51+
return self._service
52+
53+
mngmt_url: str = splunk.getLocalServerInfo()
54+
scheme, host, port, path = _spliturl(mngmt_url)
55+
56+
headers = self.request["headers"]
57+
58+
cookies: str | None = headers.get("cookie")
59+
authorizaiton: str | None = headers.get("authorization")
60+
61+
if cookies is not None:
62+
c = SimpleCookie()
63+
c.load(cookies)
64+
cookie_token = c.get("splunkd_8089")
65+
if cookie_token is not None:
66+
service = connect(
67+
scheme=scheme,
68+
host=host,
69+
port=port,
70+
path=path,
71+
autologin=True,
72+
cookie=f"splunkd_8089: {cookie_token}",
73+
)
74+
75+
# Make sure splunk connection works.
76+
assert service.info.startup_time
77+
78+
self._service = service
79+
return service
80+
81+
if authorizaiton is not None:
82+
authType, token = authorizaiton.split(" ", 1)
83+
if authType.lower() == "bearer" or authType.lower() == "splunk":
84+
service = connect(
85+
scheme=scheme,
86+
host=host,
87+
port=port,
88+
path=path,
89+
autologin=True,
90+
token=token,
91+
)
92+
93+
# Make sure splunk connection works.
94+
assert service.info.startup_time
95+
96+
self._service = service
97+
return service
98+
elif authType.lower() == "basic":
99+
decoded_bytes = base64.b64decode(token)
100+
username, password = decoded_bytes.decode("utf-8").split(":", 1)
101+
service = connect(
102+
scheme=scheme,
103+
host=host,
104+
port=port,
105+
path=path,
106+
autologin=True,
107+
username=username,
108+
password=password,
109+
)
110+
111+
# Make sure splunk connection works.
112+
assert service.info.startup_time
113+
114+
self._service = service
115+
return service
116+
117+
# We should not reach here, since Splunk requires that the request is authenticated.
118+
raise Exception("Missing auth")
119+
except ImportError as e:
120+
# The "splunk" package is only available on the Splunk instances, as it is only shipped
121+
# with the default splunk python interpreter. We can't use it reliabely if used outside of
122+
# splunk, in such cases, we don't expose the wrapped class.
123+
if e.name != "splunk":
124+
raise
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env python
2+
#
3+
# Copyright © 2011-2026 Splunk, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License"): you may
6+
# not use this file except in compliance with the License. You may obtain
7+
# a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
# License for the specific language governing permissions and limitations
15+
# under the License.
16+
17+
18+
import pytest
19+
20+
from tests import testlib
21+
22+
23+
class TestAgenticApp(testlib.SDKTestCase):
24+
def test_agetic_app(self) -> None:
25+
pytest.importorskip("langchain_openai")
26+
self.skip_splunk_10_2()
27+
28+
resp = self.service.post("agentic_app/agent-name")
29+
assert resp.status == 200
30+
assert "stefan" in str(resp.body)
31+
32+
def test_agentic_app_with_tools_weather(self) -> None:
33+
pytest.importorskip("langchain_openai")
34+
self.skip_splunk_10_2()
35+
36+
resp = self.service.post("agentic_app_with_local_tools/weather")
37+
assert resp.status == 200
38+
assert "31.5" in str(resp.body)
39+
40+
def test_agentic_app_with_tools_agent_name(self) -> None:
41+
pytest.importorskip("langchain_openai")
42+
self.skip_splunk_10_2()
43+
44+
resp = self.service.post("agentic_app_with_local_tools/agent-name")
45+
assert resp.status == 200
46+
assert "stefan" in str(resp.body)
47+
48+
# TODO: Would be nice to test remote tool execution, such test would need to install the
49+
# MCP Server App and define a custom tool (tools.conf). For now we only test remote tools ececution
50+
# with a mock mcp server, outside of Splunk environment, see ../integration/ai/test_agent_mcp_tools.py.
51+
52+
def skip_splunk_10_2(self) -> None:
53+
if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2:
54+
self.skipTest("Python 3.13 not available on splunk < 10.2")

0 commit comments

Comments
 (0)