Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion unstract/sdk1/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ dependencies = [
"pdfplumber>=0.11.2",
"redis>=5.2.1",
# # LLMWhisperer client
"llmwhisperer-client>=2.6.2",
"llmwhisperer-client>=2.8.0",
# # Core utilities (Redis Sentinel-aware client factory, etc.)
"unstract-core",
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,14 @@ class WhispererConfig:
MEDIAN_FILTER_SIZE = "median_filter_size"
GAUSSIAN_BLUR_RADIUS = "gaussian_blur_radius"
LINE_SPLITTER_TOLERANCE = "line_splitter_tolerance"
LINE_SPLITTER_STRATEGY = "line_spitter_strategy"
LINE_SPLITTER_STRATEGY = "line_splitter_strategy"
HORIZONTAL_STRETCH_FACTOR = "horizontal_stretch_factor"
PAGES_TO_EXTRACT = "pages_to_extract"
MARK_VERTICAL_LINES = "mark_vertical_lines"
MARK_HORIZONTAL_LINES = "mark_horizontal_lines"
# Misspelling retained: correcting the JSON schema key means migrating saved configs
PAGE_SEPARATOR = "page_seperator"
PAGE_SEPARATOR_PARAM = "page_separator"
URL_IN_POST = "url_in_post"
TAG = "tag"
USE_WEBHOOK = "use_webhook"
Expand All @@ -78,6 +80,7 @@ class WhispererConfig:
INCLUDE_LINE_CONFIDENCE = "include_line_confidence"
EXTRACT_ALL_LINES = "extract_all_lines"
LINES = "lines"
FILE_NAME = "file_name"


class WhisperStatus:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ def get_whisperer_params(
WhispererConfig.MARK_HORIZONTAL_LINES,
WhispererDefaults.MARK_HORIZONTAL_LINES,
),
WhispererConfig.PAGE_SEPARATOR: config.get(
WhispererConfig.PAGE_SEPARATOR_PARAM: config.get(
WhispererConfig.PAGE_SEPARATOR,
WhispererDefaults.PAGE_SEPARATOR,
),
Expand Down Expand Up @@ -260,6 +260,8 @@ def send_whisper_request(
params = LLMWhispererHelper.get_whisperer_params(
config=config, extra_params=extra_params
)
# Recorded against the extraction for cross referencing in usage reports
params[WhispererConfig.FILE_NAME] = Path(input_file_path).name
response: requests.Response
try:
input_file_data = BytesIO(fs.read(path=input_file_path, mode="rb"))
Expand Down
27 changes: 27 additions & 0 deletions unstract/sdk1/tests/test_llm_whisperer_v2_params.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"""Tests for the query params the LLMWhisperer V2 adapter sends."""

from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.dto import (
WhispererRequestParams,
)
from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src.helper import LLMWhispererHelper


def _params(config: dict) -> dict:
return LLMWhispererHelper.get_whisperer_params(
config=config, extra_params=WhispererRequestParams()
)


def test_line_splitter_strategy_from_config() -> None:
"""The key stored by the adapter's JSON schema is the one that is read."""
params = _params({"line_splitter_strategy": "right-priority"})

assert params["line_splitter_strategy"] == "right-priority"


def test_page_separator_read_under_legacy_config_key() -> None:
"""Existing configs store the misspelled key but the client kwarg is correct."""
params = _params({"page_seperator": "<<< {{page_no}} >>>"})

assert params["page_separator"] == "<<< {{page_no}} >>>"
assert "page_seperator" not in params
Loading