Skip to content

Commit 1d3798d

Browse files
hiro-vhiro-v
authored
feat: Add /v1/audio/transcriptions and /v1/audio/translations with whisper.cpp (janhq#276)
1 parent 7e245c5 commit 1d3798d

18 files changed

Lines changed: 8135 additions & 52 deletions

.github/scripts/e2e-test-linux-and-mac.sh renamed to .github/scripts/e2e-test-llama-linux-and-mac.sh

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@ range=$((max - min + 1))
2121
PORT=$((RANDOM % range + min))
2222

2323
# Start the binary file
24-
"$BINARY_PATH" 1 127.0.0.1 $PORT > /tmp/nitro.log 2>&1 &
24+
"$BINARY_PATH" 1 127.0.0.1 $PORT >/tmp/nitro.log 2>&1 &
2525

2626
# Get the process id of the binary file
2727
pid=$!
2828

29-
if ! ps -p $pid > /dev/null; then
29+
if ! ps -p $pid >/dev/null; then
3030
echo "nitro failed to start. Logs:"
3131
cat /tmp/nitro.log
3232
exit 1
@@ -35,26 +35,27 @@ fi
3535
# Wait for a few seconds to let the server start
3636
sleep 5
3737

38-
# Check if /tmp/testmodel exists, if not, download it
39-
if [[ ! -f "/tmp/testmodel" ]]; then
40-
wget $DOWNLOAD_URL -O /tmp/testmodel
38+
# Check if /tmp/testllm exists, if not, download it
39+
if [[ ! -f "/tmp/testllm" ]]; then
40+
wget $DOWNLOAD_URL -O /tmp/testllm
4141
fi
4242

4343
# Run the curl commands
4444
response1=$(curl -o /tmp/response1.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/inferences/llamacpp/loadModel" \
45-
--header 'Content-Type: application/json' \
46-
--data '{
47-
"llama_model_path": "/tmp/testmodel",
45+
--header 'Content-Type: application/json' \
46+
--data '{
47+
"llama_model_path": "/tmp/testllm",
4848
"ctx_len": 50,
4949
"ngl": 32,
5050
"embedding": false
5151
}' 2>&1)
5252

53-
response2=$(curl -o /tmp/response2.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/inferences/llamacpp/chat_completion" \
54-
--header 'Content-Type: application/json' \
55-
--header 'Accept: text/event-stream' \
56-
--header 'Access-Control-Allow-Origin: *' \
57-
--data '{
53+
response2=$(
54+
curl -o /tmp/response2.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/v1/chat/completions" \
55+
--header 'Content-Type: application/json' \
56+
--header 'Accept: text/event-stream' \
57+
--header 'Access-Control-Allow-Origin: *' \
58+
--data '{
5859
"messages": [
5960
{"content": "Hello there", "role": "assistant"},
6061
{"content": "Write a long and sad story for me", "role": "user"}
@@ -98,7 +99,6 @@ echo "----------------------"
9899
echo "Log run test:"
99100
cat /tmp/response2.log
100101

101-
102102
echo "Nitro test run successfully!"
103103

104104
# Kill the server process

.github/scripts/e2e-test-windows.bat renamed to .github/scripts/e2e-test-llama-windows.bat

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
@echo off
22

33
set "TEMP=C:\Users\%UserName%\AppData\Local\Temp"
4-
set "MODEL_PATH=%TEMP%\testmodel"
4+
set "MODEL_PATH=%TEMP%\testllm"
55

66
rem Check for required arguments
77
if "%~2"=="" (

‎.github/scripts/e2e-test-whisper-linux-and-mac.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/bin/bash
2+
3+
## Example run command
4+
# ./linux-and-mac.sh './jan/plugins/@janhq/inference-plugin/dist/nitro/nitro_mac_arm64' https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/resolve/main/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
5+
6+
# Check for required arguments
7+
if [[ $# -ne 2 ]]; then
8+
echo "Usage: $0 <path_to_binary> <url_to_download>"
9+
exit 1
10+
fi
11+
12+
rm /tmp/response1.log /tmp/response2.log /tmp/nitro.log
13+
14+
BINARY_PATH=$1
15+
DOWNLOAD_URL=$2
16+
17+
# Random port to ensure it's not used
18+
min=10000
19+
max=11000
20+
range=$((max - min + 1))
21+
PORT=$((RANDOM % range + min))
22+
23+
# Start the binary file
24+
"$BINARY_PATH" 1 127.0.0.1 $PORT >/tmp/nitro.log 2>&1 &
25+
26+
# Get the process id of the binary file
27+
pid=$!
28+
29+
if ! ps -p $pid >/dev/null; then
30+
echo "nitro failed to start. Logs:"
31+
cat /tmp/nitro.log
32+
exit 1
33+
fi
34+
35+
# Wait for a few seconds to let the server start
36+
sleep 5
37+
38+
# Check if /tmp/testwhisper exists, if not, download it
39+
if [[ ! -f "/tmp/testwhisper" ]]; then
40+
wget $DOWNLOAD_URL -O /tmp/testwhisper
41+
fi
42+
43+
# Run the curl commands
44+
response1=$(curl -o /tmp/response1.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/v1/audio/load_model" \
45+
--header 'Content-Type: application/json' \
46+
--data '{
47+
"model_path": "/tmp/testwhisper",
48+
"model_id": "whisper.cpp"
49+
}' 2>&1)
50+
51+
response2=$(
52+
curl -o /tmp/response2.log -s -w "%{http_code}" --location "http://127.0.0.1:$PORT/v1/audio/transcriptions" \
53+
--header 'Access-Control-Allow-Origin: *' \
54+
--form 'file=@"../whisper.cpp/samples/jfk.wav"' \
55+
--form 'model_id="whisper.cpp"' \
56+
--form 'temperature="0.0"' \
57+
--form 'prompt="The transcript is about OpenAI which makes technology like DALL·E, GPT-3, and ChatGPT with the hope of one day building an AGI system that benefits all of humanity. The president is trying to raly people to support the cause."' \
58+
2>&1
59+
)
60+
61+
error_occurred=0
62+
if [[ "$response1" -ne 200 ]]; then
63+
echo "The first curl command failed with status code: $response1"
64+
cat /tmp/response1.log
65+
error_occurred=1
66+
fi
67+
68+
if [[ "$response2" -ne 200 ]]; then
69+
echo "The second curl command failed with status code: $response2"
70+
cat /tmp/response2.log
71+
error_occurred=1
72+
fi
73+
74+
if [[ "$error_occurred" -eq 1 ]]; then
75+
echo "Nitro test run failed!!!!!!!!!!!!!!!!!!!!!!"
76+
echo "Nitro Error Logs:"
77+
cat /tmp/nitro.log
78+
kill $pid
79+
exit 1
80+
fi
81+
82+
echo "----------------------"
83+
echo "Log load model:"
84+
cat /tmp/response1.log
85+
86+
echo "----------------------"
87+
echo "Log run test:"
88+
cat /tmp/response2.log
89+
90+
echo "Nitro test run successfully!"
91+
92+
# Kill the server process
93+
kill $pid

‎.github/scripts/e2e-test-whisper-windows.bat‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
@echo off
2+
3+
set "TEMP=C:\Users\%UserName%\AppData\Local\Temp"
4+
set "MODEL_PATH=%TEMP%\testwhisper"
5+
6+
rem Check for required arguments
7+
if "%~2"=="" (
8+
echo Usage: %~0 ^<path_to_binary^> ^<url_to_download^>
9+
exit /b 1
10+
)
11+
12+
set "BINARY_PATH=%~1"
13+
set "DOWNLOAD_URL=%~2"
14+
15+
for %%i in ("%BINARY_PATH%") do set "BINARY_NAME=%%~nxi"
16+
17+
echo BINARY_NAME=%BINARY_NAME%
18+
19+
del %TEMP%\response1.log 2>nul
20+
del %TEMP%\response2.log 2>nul
21+
del %TEMP%\nitro.log 2>nul
22+
23+
set /a min=9999
24+
set /a max=11000
25+
set /a range=max-min+1
26+
set /a PORT=%min% + %RANDOM% %% %range%
27+
28+
rem Start the binary file
29+
start /B "" "%BINARY_PATH%" 1 "127.0.0.1" %PORT% > %TEMP%\nitro.log 2>&1
30+
31+
ping -n 6 127.0.0.1 %PORT% > nul
32+
33+
rem Capture the PID of the started process with "nitro" in its name
34+
for /f "tokens=2" %%a in ('tasklist /fi "imagename eq %BINARY_NAME%" /fo list ^| findstr /B "PID:"') do (
35+
set "pid=%%a"
36+
)
37+
38+
echo pid=%pid%
39+
40+
if not defined pid (
41+
echo nitro failed to start. Logs:
42+
type %TEMP%\nitro.log
43+
exit /b 1
44+
)
45+
46+
rem Wait for a few seconds to let the server start
47+
48+
rem Check if %TEMP%\testmodel exists, if not, download it
49+
if not exist "%MODEL_PATH%" (
50+
bitsadmin.exe /transfer "DownloadTestModel" %DOWNLOAD_URL% "%MODEL_PATH%"
51+
)
52+
53+
rem Define JSON strings for curl data
54+
call set "MODEL_PATH_STRING=%%MODEL_PATH:\=\\%%"
55+
set "curl_data1={\"model_path\":\"%MODEL_PATH_STRING%\",\"model_id\":\"whisper.cpp\"}"
56+
57+
rem Print the values of curl_data1 for debugging
58+
echo curl_data1=%curl_data1%
59+
60+
rem Run the curl commands and capture the status code
61+
curl.exe -o %TEMP%\response1.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/load_model" --header "Content-Type: application/json" --data "%curl_data1%" > %TEMP%\response1_code.log 2>&1
62+
63+
curl.exe -o %TEMP%\response2.log -s -w "%%{http_code}" --location "http://127.0.0.1:%PORT%/v1/audio/transcriptions" ^
64+
--header "Access-Control-Allow-Origin: *" ^
65+
--form 'model_id="whisper.cpp"' ^
66+
--form 'file=@"..\whisper.cpp\samples\jfk.wav"' ^
67+
--form 'temperature="0.0"' ^
68+
--form 'prompt="The transcript is about OpenAI which makes technology like DALL·E, GPT-3, and ChatGPT with the hope of one day building an AGI system that benefits all of humanity. The president is trying to raly people to support the cause."' ^
69+
> %TEMP%\response2_code.log 2>&1
70+
71+
set "error_occurred=0"
72+
73+
rem Read the status codes from the log files
74+
for /f %%a in (%TEMP%\response1_code.log) do set "response1=%%a"
75+
for /f %%a in (%TEMP%\response2_code.log) do set "response2=%%a"
76+
77+
if "%response1%" neq "200" (
78+
echo The first curl command failed with status code: %response1%
79+
type %TEMP%\response1.log
80+
set "error_occurred=1"
81+
)
82+
83+
if "%response2%" neq "200" (
84+
echo The second curl command failed with status code: %response2%
85+
type %TEMP%\response2.log
86+
set "error_occurred=1"
87+
)
88+
89+
if "%error_occurred%"=="1" (
90+
echo Nitro test run failed!!!!!!!!!!!!!!!!!!!!!!
91+
echo Nitro Error Logs:
92+
type %TEMP%\nitro.log
93+
taskkill /f /pid %pid%
94+
exit /b 1
95+
)
96+
97+
98+
echo ----------------------
99+
echo Log load model:
100+
type %TEMP%\response1.log
101+
102+
echo ----------------------
103+
echo "Log run test:"
104+
type %TEMP%\response2.log
105+
106+
echo Nitro test run successfully!
107+
108+
rem Kill the server process
109+
taskkill /f /pid %pid%

0 commit comments

Comments
 (0)