Skip to content

Commit 875e05f

Browse files
mateusz834mateusz834
authored
Add an E2E test with Splunk MCP Server App (#85)
1 parent 31801e7 commit 875e05f

12 files changed

Lines changed: 284 additions & 60 deletions

File tree

‎.basedpyright/baseline.json‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40023,14 +40023,6 @@
4002340023
"lineCount": 1
4002440024
}
4002540025
},
40026-
{
40027-
"code": "reportUndefinedVariable",
40028-
"range": {
40029-
"startColumn": 12,
40030-
"endColumn": 16,
40031-
"lineCount": 1
40032-
}
40033-
},
4003440026
{
4003540027
"code": "reportImplicitOverride",
4003640028
"range": {

‎.github/workflows/test.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,20 @@ jobs:
1212
steps:
1313
- name: Checkout code
1414
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
15-
- name: Launch Splunk Docker instance
16-
run: SPLUNK_VERSION=${{ matrix.splunk-version }} docker compose up -d
1715
- name: Setup Python ${{ matrix.python-version }}
1816
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
1917
with:
2018
python-version: ${{ matrix.python-version }}
2119
cache: "pip"
2220
- name: Install dependencies
2321
run: python -m pip install '.[openai, anthropic]' --group test
24-
22+
- name: Download Splunk MCP Server App
23+
run: python ./scripts/download_splunk_mcp_server_app.py
24+
env:
25+
SPLUNKBASE_USERNAME: ${{ secrets.SPLUNKBASE_USERNAME }}
26+
SPLUNKBASE_PASSWORD: ${{ secrets.SPLUNKBASE_PASSWORD }}
27+
- name: Launch Splunk Docker instance
28+
run: SPLUNK_VERSION=${{ matrix.splunk-version }} docker compose up -d
2529
- name: Set up .env
2630
run: cp .env.template .env
2731
- name: Write internal AI secrets to .env

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,5 +279,8 @@ $RECYCLE.BIN/
279279

280280
.vscode/
281281
docs/_build/
282+
282283
!*.conf.spec
283284
**/metadata/local.meta
285+
286+
splunk-mcp-server*.{spl,tar,tar.gz,tgz}

‎Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@ FROM splunk/splunk:${SPLUNK_VERSION}
33

44
USER root
55

6+
# Copy splunk-mcp-server.tgz, we need to copy entire sdk since
7+
# splunk-mcp-server.tgz might not exist and we don't want to fail in such case.
8+
RUN mkdir /tmp/sdk
9+
COPY . /tmp/sdk
10+
RUN /bin/bash -c 'if [ -f /tmp/sdk/splunk-mcp-server.tgz ]; then cp /tmp/sdk/splunk-mcp-server.tgz /splunk-mcp-server.tgz; fi'
11+
RUN rm -rf /tmp/sdk
12+
613
RUN mkdir /tmp/sdk
714
COPY ./pyproject.toml /tmp/sdk/pyproject.toml
815
COPY ./uv.lock /tmp/sdk/uv.lock

‎scripts/download_splunk_mcp_server_app.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright © 2011-2026 Splunk, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"): you may
4+
# not use this file except in compliance with the License. You may obtain
5+
# a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12+
# License for the specific language governing permissions and limitations
13+
# under the License.
14+
15+
# This script uses unsupported API to download the Splunk MCP Server App
16+
# from splunkbase for CI purposes.
17+
#
18+
# Use at your own risk.
19+
20+
21+
import os
22+
import xml.etree.ElementTree as ET
23+
24+
import httpx
25+
from pydantic import BaseModel
26+
27+
SPLUNK_MCP_APP_ID = 7931
28+
MCP_PATH = "splunk-mcp-server.tgz"
29+
SPLUNKBASE_URL = "https://splunkbase.splunk.com"
30+
31+
32+
class Release(BaseModel):
33+
path: str
34+
35+
36+
class Response(BaseModel):
37+
release: Release
38+
39+
40+
def run() -> None:
41+
username = os.environ["SPLUNKBASE_USERNAME"]
42+
password = os.environ["SPLUNKBASE_PASSWORD"]
43+
44+
client = httpx.Client(follow_redirects=True)
45+
response = client.post(
46+
f"{SPLUNKBASE_URL}/api/account:login",
47+
data={
48+
"username": username,
49+
"password": password,
50+
},
51+
headers={"Content-Type": "application/x-www-form-urlencoded"},
52+
)
53+
54+
response.raise_for_status()
55+
56+
root = ET.fromstring(response.text)
57+
58+
token = next(elem.text for elem in root if elem.tag.endswith("id"))
59+
if token is None:
60+
raise AssertionError("token not found in the response")
61+
62+
response = client.get(
63+
f"{SPLUNKBASE_URL}/api/v1/app/{SPLUNK_MCP_APP_ID}/?include=release",
64+
headers={"Authorization": f"Bearer {token}"},
65+
)
66+
response.raise_for_status()
67+
68+
result = Response.model_validate_json(response.text)
69+
70+
response = client.get(
71+
result.release.path,
72+
headers={"Authorization": f"Bearer {token}"},
73+
)
74+
response.raise_for_status()
75+
76+
with open(MCP_PATH, "wb") as f:
77+
f.write(response.content)
78+
79+
80+
if __name__ == "__main__":
81+
run()

‎tests/ai_testlib.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from typing import override
12
from splunklib.ai.model import PredefinedModel
23
from tests.ai_test_model import InternalAIModel, TestLLMSettings, create_model
34
from tests.testlib import SDKTestCase
@@ -6,6 +7,17 @@
67
class AITestCase(SDKTestCase):
78
_model: PredefinedModel | None = None
89

10+
@override
11+
def setUp(self) -> None:
12+
super().setUp()
13+
14+
# Our tests don't expect this app to be installed, if needed it is
15+
# installed on demand.
16+
for app in self.service.apps.list(): # pyright: ignore[reportUnknownVariableType]
17+
if app.name.lower() == "splunk_mcp_server":
18+
app.delete()
19+
self.restart_splunk()
20+
921
@property
1022
def test_llm_settings(self) -> TestLLMSettings:
1123
client_id: str = self.opts.kwargs["internal_ai_client_id"]

‎tests/integration/ai/test_agent_mcp_tools.py‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import asyncio
44
import contextlib
55
import json
6-
import logging
76
import os
87
import socket
98
from collections.abc import AsyncGenerator
@@ -568,46 +567,6 @@ async def lifespan(app: Starlette) -> AsyncGenerator[None, Any]:
568567
response = result.final_message.content
569568
assert "31.5" in response, "Invalid LLM response"
570569

571-
@patch(
572-
"splunklib.ai.agent._testing_local_tools_path",
573-
os.path.join(os.path.dirname(__file__), "testdata", "non_existent.py"),
574-
)
575-
@patch("splunklib.ai.agent._testing_app_id", "app_id")
576-
@pytest.mark.asyncio
577-
async def test_splunk_mcp_server_app(self) -> None:
578-
pytest.skip("Remove this test once we have an E2E with Splunk MCP Server app.")
579-
580-
# Skip if the langchain_openai package is not installed
581-
pytest.importorskip("langchain_openai") # pyright: ignore[reportUnreachable]
582-
583-
logger = logging.getLogger("test")
584-
logger.setLevel(logging.DEBUG)
585-
586-
service = connect(
587-
port=8090,
588-
host="localhost",
589-
username="admin",
590-
password="",
591-
autologin=True,
592-
)
593-
594-
async with Agent(
595-
model=(await self.model()),
596-
system_prompt="You must use the available tools to perform requested operations",
597-
service=service,
598-
use_mcp_tools=True,
599-
logger=logger,
600-
) as agent:
601-
for tool in agent.tools: # pyright: ignore[reportUnreachable]
602-
if tool.name == "splunk_get_indexes": # pyright: ignore[reportUnreachable]
603-
result = await tool.func() # pyright: ignore[reportUnreachable]
604-
assert (
605-
len((result.structured_content or {}).get("results", [])) != 0
606-
)
607-
return
608-
609-
pytest.fail("Tool splunk_get_indexes not found")
610-
611570

612571
class TestHandlingToolNameCollision(AITestCase):
613572
@patch(

‎tests/system/test_ai_agentic_test_app.py‎

Lines changed: 58 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@
1515

1616
import pytest
1717

18+
from splunklib.binding import HTTPError
1819
from tests.ai_testlib import AITestCase
1920

2021

2122
class TestAgenticApp(AITestCase):
2223
def test_agetic_app(self) -> None:
2324
pytest.importorskip("langchain_openai")
24-
self.skip_splunk_10_2()
25+
self.requires_splunk_10_2()
2526

2627
resp = self.service.post(
2728
"agentic_app/agent-name",
@@ -32,7 +33,7 @@ def test_agetic_app(self) -> None:
3233

3334
def test_agentic_app_with_tools_weather(self) -> None:
3435
pytest.importorskip("langchain_openai")
35-
self.skip_splunk_10_2()
36+
self.requires_splunk_10_2()
3637

3738
resp = self.service.post(
3839
"agentic_app_with_local_tools/weather",
@@ -43,7 +44,7 @@ def test_agentic_app_with_tools_weather(self) -> None:
4344

4445
def test_agentic_app_with_tools_agent_name(self) -> None:
4546
pytest.importorskip("langchain_openai")
46-
self.skip_splunk_10_2()
47+
self.requires_splunk_10_2()
4748

4849
resp = self.service.post(
4950
"agentic_app_with_local_tools/agent-name",
@@ -52,10 +53,60 @@ def test_agentic_app_with_tools_agent_name(self) -> None:
5253
assert resp.status == 200
5354
assert "stefan" in str(resp.body)
5455

55-
# TODO: Would be nice to test remote tool execution, such test would need to install the
56-
# MCP Server App and define a custom tool (tools.conf). For now we only test remote tools ececution
57-
# with a mock mcp server, outside of Splunk environment, see ../integration/ai/test_agent_mcp_tools.py.
56+
# To execute this test locally, download the Splunk MCP Server App tarball from
57+
# https://splunkbase.splunk.com/app/7931 and place it in a file named
58+
# splunk-mcp-server.tgz at the root of this repo (i.e. ../../splunk-mcp-server.tgz).
59+
#
60+
# Note: that the downloaded file could have a: .spl, .tar, .tar.gz or .tgz extension,
61+
# if it is not .tgz, then you must change it to .tgz.
62+
#
63+
# Our CI does this automatically.
64+
def test_agentic_app_with_remote_tools(self) -> None:
65+
pytest.importorskip("langchain_openai")
66+
self.requires_splunk_10_2()
67+
68+
INDEX_NAME = "needle-index"
69+
70+
# Delete the index if already exists.
71+
for index in self.service.indexes: # pyright: ignore[reportUnknownVariableType]
72+
if index.name == INDEX_NAME:
73+
index.delete()
74+
75+
# Skip test in case the instance does not have a /splunk-mcp-server.tgz file.
76+
# We do so, not to require app download for local development of the SDK.
77+
# Note that: our CI always has this file available.
78+
#
79+
# We check that through a separate endpoint call, since we want to have tests
80+
# that don't assume that our CI splunk instance is a docker container.
81+
try:
82+
resp = self.service.get("agentic_app/has_mcp_app_file")
83+
assert resp.status == 200
84+
except HTTPError as e:
85+
if e.status == 404:
86+
self.skipTest("Splunk MCP Server App file not found on Splunk instance")
87+
raise
88+
89+
# AITestCase already removes the Splunk MCP Server App in case it is already
90+
# installed, so here we will always end up installing it, thus having a fresh
91+
# version of the app.
92+
93+
# Install the Splunk MCP Server App.
94+
app = self.service.apps.create(name="/splunk-mcp-server.tgz", filename=True) # pyright: ignore[reportUnknownVariableType]
95+
96+
index = self.service.indexes.create(name=INDEX_NAME) # pyright: ignore[reportUnknownVariableType]
97+
98+
resp = self.service.post(
99+
"agentic_app/indexes",
100+
body=self.test_llm_settings.model_dump_json(),
101+
)
102+
103+
assert resp.status == 200
104+
assert INDEX_NAME in str(resp.body) # pyright: ignore[reportUnknownArgumentType]
105+
106+
index.delete()
107+
app.delete()
108+
self.restart_splunk() # app removal requires a restart
58109

59-
def skip_splunk_10_2(self) -> None:
110+
def requires_splunk_10_2(self) -> None:
60111
if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2:
61112
self.skipTest("Python 3.13 not available on splunk < 10.2")

‎tests/system/test_apps/ai_agentic_test_app/bin/indexes.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Copyright © 2011-2026 Splunk, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"): you may
4+
# not use this file except in compliance with the License. You may obtain
5+
# a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12+
# License for the specific language governing permissions and limitations
13+
# under the License.
14+
15+
import os
16+
import sys
17+
18+
sys.path.insert(0, "/splunklib-deps")
19+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib"))
20+
21+
from typing import override
22+
23+
from pydantic import BaseModel, Field
24+
25+
from splunklib.ai.agent import Agent
26+
from splunklib.ai.messages import HumanMessage
27+
from splunklib.ai.tool_filtering import ToolFilters
28+
from tests.cre_testlib import CRETestHandler
29+
30+
# BUG: For some reason the CRE process is started with a overridden trust store path, that
31+
# does not exist on the filesystem. As a workaround in such case if it does not exist,
32+
# remove the env, this causes the default CAs to be used instead.
33+
CA_TRUST_STORE = "/opt/splunk/openssl/cert.pem"
34+
if os.environ.get("SSL_CERT_FILE") == CA_TRUST_STORE and not os.path.exists(
35+
CA_TRUST_STORE
36+
):
37+
os.environ["SSL_CERT_FILE"] = ""
38+
39+
# This app uses the splunk_get_indexes remote tool (from Splunk MCP Server App).
40+
# Requires that the MCP Server App is installed.
41+
42+
43+
class IndexesHandler(CRETestHandler):
44+
@override
45+
async def run(self) -> None:
46+
class Output(BaseModel):
47+
indexes: list[str] = Field(description="list of index names")
48+
49+
async with Agent(
50+
model=(await self.model()),
51+
system_prompt="You are a helpful Splunk assistant",
52+
use_mcp_tools=True,
53+
service=self.service,
54+
tool_filters=ToolFilters(
55+
allowed_names=["splunk_get_indexes"], allowed_tags=[]
56+
),
57+
output_schema=Output,
58+
) as agent:
59+
assert len(agent.tools) == 1, "Invalid tool count"
60+
assert (
61+
len([tool for tool in agent.tools if tool.name == "splunk_get_indexes"])
62+
== 1
63+
), "splunk_get_indexes not present"
64+
65+
result = await agent.invoke(
66+
[
67+
HumanMessage(
68+
content="List all indexes available on the splunk instance.",
69+
)
70+
]
71+
)
72+
73+
self.response.write(result.structured_output.model_dump_json())

0 commit comments

Comments
 (0)