diff --git a/.claude/skills/adapter-ops/SKILL.md b/.claude/skills/adapter-ops/SKILL.md index 521111db32..417355b03a 100644 --- a/.claude/skills/adapter-ops/SKILL.md +++ b/.claude/skills/adapter-ops/SKILL.md @@ -56,7 +56,7 @@ LiteLLM requires provider prefixes on model names: 1. **Run initialization script**: ```bash - python .claude/skills/unstract-adapter-extension/scripts/init_llm_adapter.py \ + python .claude/skills/adapter-ops/scripts/init_llm_adapter.py \ --provider newprovider \ --name "New Provider" \ --description "New Provider LLM adapter" \ @@ -122,7 +122,7 @@ LiteLLM requires provider prefixes on model names: 1. **Run initialization script**: ```bash - python .claude/skills/unstract-adapter-extension/scripts/init_embedding_adapter.py \ + python .claude/skills/adapter-ops/scripts/init_embedding_adapter.py \ --provider newprovider \ --name "New Provider" \ --description "New Provider embedding adapter" \ @@ -190,10 +190,10 @@ LiteLLM requires provider prefixes on model names: 3. **Run management script** for automated updates: ```bash - python .claude/skills/unstract-adapter-extension/scripts/manage_models.py \ + python .claude/skills/adapter-ops/scripts/manage_models.py \ --adapter llm \ --provider openai \ - --action add \ + --action add-enum \ --models "gpt-4-turbo,gpt-4o-mini" ``` @@ -238,17 +238,17 @@ Compare existing adapter schemas against known LiteLLM features to identify pote 1. **Run the update checker**: ```bash # Check all adapters - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py # Check specific adapter type - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py --adapter llm - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py --adapter embedding + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py --adapter llm + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py --adapter embedding # Check specific provider - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py --provider openai + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py --provider openai # Output as JSON - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py --json + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py --json ``` 2. **Review the report**: @@ -278,7 +278,8 @@ Before submitting adapter changes: - [ ] Adapter class inherits from correct parameter class AND `BaseAdapter` - [ ] `get_id()` returns unique `{provider}|{uuid}` format - [ ] `get_metadata()` returns dict with `name`, `version`, `adapter`, `description`, `is_active` -- [ ] **CRITICAL: `get_provider()` returns value matching `litellm_provider` in LiteLLM pricing data** (see below) +- [ ] `get_provider()` matches the static JSON filename (`static/{get_provider()}.json`) +- [ ] **CRITICAL: the model string produced by `validate_model()` resolves in LiteLLM's cost map** (see below) - [ ] `get_adapter_type()` returns correct `AdapterTypes.LLM` or `AdapterTypes.EMBEDDING` - [ ] JSON schema has `adapter_name` as required field - [ ] `validate()` method adds correct model prefix @@ -286,43 +287,86 @@ Before submitting adapter changes: - [ ] All static methods decorated with `@staticmethod` - [ ] Icon path follows pattern `/icons/adapter-icons/{Name}.png` -### Provider Name Verification (MANDATORY) +### Model Prefix Verification (MANDATORY) -The `get_provider()` return value is used for **cost calculation**. It MUST match the `litellm_provider` field in LiteLLM's pricing data, otherwise costs will show as $0. +Cost is looked up from the **validated model string**, not from `get_provider()`. The string +that `validate_model()` produces (e.g. `mistral/mistral-embed`) is passed straight to +`litellm.cost_per_token()`. If LiteLLM's cost map has no entry for it, the lookup raises, the +exception is swallowed, and usage records **$0**. -**Before implementing any adapter, verify the provider name:** +The lookup sites: -```bash -# Fetch LiteLLM pricing data and check provider name -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.value.litellm_provider != null)) | - map({key: .key, provider: .value.litellm_provider}) | - unique_by(.provider) | - sort_by(.provider) | - .[].provider' | sort -u -``` +| Path | Site | +|------|------| +| LLM | `unstract/sdk1/src/unstract/sdk1/audit.py` — `cost_per_token(model=model_name)` | +| Embedding | `unstract/sdk1/src/unstract/sdk1/usage_handler.py` — `litellm.cost_per_token(...)` | + +`model_name` is `self._cost_model or self.kwargs["model"]` — the prefixed string. Both call +sites catch every exception and fall back to `0.0`, so a miss is **silent**. It will not fail a +test, a build, or a run. The only symptom is revenue-affecting: zero-cost usage rows. + +**Before implementing any adapter, verify the prefix resolves.** Use the pinned LiteLLM in the +sdk1 venv rather than curling upstream JSON — the pinned version is what actually runs: -**Common provider name mappings:** -| Display Name | `get_provider()` Value | LiteLLM Provider | -|--------------|------------------------|------------------| -| OpenAI | `openai` | `openai` | -| Anthropic | `anthropic` | `anthropic` | -| Azure OpenAI | `azure` | `azure` | -| Azure AI Foundry | `azure_ai` | `azure_ai` | -| AWS Bedrock | `bedrock` | `bedrock` | -| Google VertexAI | `vertex_ai` | `vertex_ai` | -| Mistral | `mistral` | `mistral` | -| Ollama | `ollama` | `ollama` | - -**Example verification for Azure AI Foundry:** ```bash -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.key | startswith("azure_ai"))) | .[0].value.litellm_provider' -# Output: "azure_ai" (NOT "azure_ai_foundry") +cd unstract/sdk1 && uv run python -c " +import litellm +from litellm import cost_per_token + +# 1. Does LiteLLM have a native provider for this vendor? +print([p for p in litellm.provider_list if 'YOUR_VENDOR' in str(p).lower()]) + +# 2. Which of its models are priced? +print([k for k in litellm.model_cost if k.lower().startswith('YOUR_PROVIDER/')]) + +# 3. Does the string your validate_model() emits actually resolve? +for m in ['YOUR_PROVIDER/some-model', 'custom_openai/some-model']: + try: + print(m, cost_per_token(model=m, prompt_tokens=1_000_000, completion_tokens=1_000_000)) + except Exception as e: + print(m, 'RAISES', type(e).__name__, '=> cost silently recorded as 0.0') +" ``` -**Why this matters:** -The cost calculation in `platform-service/src/unstract/platform_service/helper/cost_calculation.py` filters models by checking if the provider string is contained in `litellm_provider`. A mismatch (e.g., returning `"azure_ai_foundry"` when LiteLLM uses `"azure_ai"`) causes the cost lookup to fail silently, returning $0. +**Branded OpenAI-compatible adapters: pick the right base class.** + +Many vendors speak the OpenAI wire protocol, which tempts you to subclass +`OpenAICompatibleLLMParameters` and just pin `api_base`. But its `validate_model()` +unconditionally prepends `custom_openai/`, and **nothing** in LiteLLM's cost map is keyed that +way. That silently forfeits cost tracking. + +| If… | Then | Cost tracking | +|-----|------|---------------| +| LiteLLM has a native provider for the vendor | Extend `BaseChatCompletionParameters`, emit `{provider}/{model}` — follow `OpenRouterLLMParameters` | ✅ resolves | +| LiteLLM has **no** priced models for the vendor | Extend `OpenAICompatibleLLMParameters`, pin `api_base` — follow `NvidiaBuildLLMParameters` | ⚠️ $0 either way, nothing forfeited | + +`NvidiaBuildLLMParameters` is only a safe template because LiteLLM prices zero `nvidia_nim/` +chat models — there is no cost to lose. Do not copy that shape for a vendor LiteLLM *does* +price. Check first with the snippet above. + +**What `get_provider()` is actually for:** + +1. Resolving the static schema path — `{type}1/static/{get_provider()}.json` (case-sensitive `open()`). +2. The `provider` column on the usage row (`audit.py`) — metadata only, not used for pricing. + +Keeping it equal to LiteLLM's `litellm_provider` value is still good hygiene, and matters when +you route natively (the prefix and the provider name coincide). But a correct `get_provider()` +does **not** on its own guarantee cost resolution — a branded adapter can return `"minimax"`, +match `litellm_provider` exactly, and still bill $0 because its model string carries the +`custom_openai/` prefix. + +**Common provider names:** + +| Display Name | `get_provider()` Value | +|--------------|------------------------| +| OpenAI | `openai` | +| Anthropic | `anthropic` | +| Azure OpenAI | `azure` | +| Azure AI Foundry | `azure_ai` | +| AWS Bedrock | `bedrock` | +| Google VertexAI | `vertex_ai` | +| Mistral | `mistral` | +| Ollama | `ollama` | ## Maintenance Workflow @@ -332,7 +376,7 @@ Periodic maintenance to keep adapters current with LiteLLM features: 1. **Run the update checker**: ```bash - python .claude/skills/unstract-adapter-extension/scripts/check_adapter_updates.py + python .claude/skills/adapter-ops/scripts/check_adapter_updates.py ``` 2. **Review LiteLLM changelog** for new provider features: diff --git a/.claude/skills/adapter-ops/references/adapter_patterns.md b/.claude/skills/adapter-ops/references/adapter_patterns.md index a3b200335b..9bab1d5ea1 100644 --- a/.claude/skills/adapter-ops/references/adapter_patterns.md +++ b/.claude/skills/adapter-ops/references/adapter_patterns.md @@ -568,48 +568,87 @@ print(schema) 4. **Missing `is_active: True`** - Adapter won't be registered without it 5. **Wrong inheritance order** - Parameter class must come before BaseAdapter 6. **Incorrect provider in get_provider()** - Must match filename and schema path -7. **Provider name not matching LiteLLM pricing data** - Causes $0 cost calculation (see below) +7. **Model prefix not resolving in LiteLLM's cost map** - Causes silent $0 cost calculation (see below) -## Provider Name Verification (CRITICAL) +## Model Prefix Verification (CRITICAL) -The `get_provider()` return value MUST match the `litellm_provider` field in LiteLLM's pricing JSON. A mismatch causes cost calculation to silently fail, returning $0. +Cost is derived from the **validated model string**, not from `get_provider()`. Whatever +`validate_model()` emits is handed to `litellm.cost_per_token()`. No cost-map entry means the +call raises, the exception is swallowed, and the usage row records $0. + +### Cost Calculation Flow + +1. **Usage recording**: `LLM._record_usage()` logs tokens (no cost math here). +2. **Cost lookup**: + - LLM → `Audit.push_usage_data()` → `cost_per_token(model=model_name)` in `audit.py` + - Embedding → `UsageHandler` → `litellm.cost_per_token(...)` in `usage_handler.py` + - `model_name` is `self._cost_model or self.kwargs["model"]` — the **prefixed** string. +3. **Failure mode**: both call sites wrap the lookup in a bare `except` and fall back to + `cost_in_dollars = 0.0`, logged at `debug`/`warning`. + +The `provider` value travels alongside the usage payload and lands in the `provider` DB column. +It is **not** consulted for pricing. + +Because the failure is swallowed, nothing breaks loudly. Tests pass. Runs succeed. Usage rows +just quietly say $0. ### How to Verify -**Before implementing any new adapter:** +Query the pinned LiteLLM in the sdk1 venv — that's the version that actually runs: ```bash -# Step 1: Identify the correct provider name from LiteLLM pricing data -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.key | startswith("YOUR_PROVIDER"))) | .[0].value.litellm_provider' - -# Step 2: Use that exact value in get_provider() +cd unstract/sdk1 && uv run python -c " +import litellm +from litellm import cost_per_token + +print([p for p in litellm.provider_list if 'YOUR_VENDOR' in str(p).lower()]) +print([k for k in litellm.model_cost if k.lower().startswith('YOUR_PROVIDER/')]) + +for m in ['YOUR_PROVIDER/some-model', 'custom_openai/some-model']: + try: + print(m, cost_per_token(model=m, prompt_tokens=1_000_000, completion_tokens=1_000_000)) + except Exception: + print(m, 'RAISES => cost silently recorded as 0.0') +" ``` -### Real Example: Azure AI Foundry Bug +### Real Example: the `custom_openai/` trap -**Wrong implementation (caused $0 costs):** -```python -@staticmethod -def get_provider() -> str: - return "azure_ai_foundry" # WRONG - doesn't match litellm_provider -``` +A vendor speaking the OpenAI wire protocol invites subclassing +`OpenAICompatibleLLMParameters` and pinning `api_base`. That class's `validate_model()` +unconditionally prepends `custom_openai/`, and **no** LiteLLM cost-map key uses that prefix. -**Correct implementation:** ```python -@staticmethod -def get_provider() -> str: - return "azure_ai" # CORRECT - matches litellm_provider in pricing data +# Vendor has a native LiteLLM provider AND priced models. +# WRONG - emits "custom_openai/MiniMax-M3", which is not in the cost map -> $0 +class MiniMaxLLMParameters(OpenAICompatibleLLMParameters): + api_base: str = "https://api.minimax.io/v1" + +# CORRECT - emits "minimax/MiniMax-M3", priced at $0.30 / $1.20 per 1M tokens +class MiniMaxLLMParameters(BaseChatCompletionParameters): + ... # follow OpenRouterLLMParameters ``` -**How the bug was found:** -```bash -# Check what LiteLLM actually uses for azure_ai models -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.key | startswith("azure_ai"))) | .[0]' +Note that `get_provider()` returns `"minimax"` in *both* cases and matches `litellm_provider` +exactly. The provider string was never the problem — the model prefix was. -# Output shows: "litellm_provider": "azure_ai" (NOT "azure_ai_foundry") -``` +### Choosing a base class + +| If… | Then | Cost tracking | +|-----|------|---------------| +| LiteLLM has a native provider for the vendor | `BaseChatCompletionParameters`, emit `{provider}/{model}` — see `OpenRouterLLMParameters` | ✅ resolves | +| LiteLLM has **no** priced models for the vendor | `OpenAICompatibleLLMParameters`, pin `api_base` — see `NvidiaBuildLLMParameters` | ⚠️ $0 regardless, nothing forfeited | + +`NvidiaBuildLLMParameters` is a safe template *only because* LiteLLM prices zero `nvidia_nim/` +chat models. Don't copy its shape for a vendor LiteLLM prices — verify first. + +### What `get_provider()` is for + +1. Resolving the static schema path: `{type}1/static/{get_provider()}.json` (case-sensitive). +2. The `provider` column on the usage row — metadata only. + +Match it to LiteLLM's `litellm_provider` for hygiene and because it coincides with the prefix +when routing natively. But it does not, by itself, guarantee cost resolution. ### Provider Name Reference @@ -623,14 +662,3 @@ curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_ | Google VertexAI | `vertex_ai` | | Mistral | `mistral` | | Ollama | `ollama` | - -### Cost Calculation Flow - -Understanding why this matters: - -1. **Usage Recording**: `LLM._record_usage()` → `Audit.push_usage_data(provider=get_provider())` -2. **Platform Service**: Receives provider name with usage data -3. **Cost Lookup**: `CostCalculationHelper` checks `provider in model_info.get("litellm_provider", "")` -4. **Result**: If check fails, returns `cost = 0` - -The check `"azure_ai_foundry" in "azure_ai"` returns `False`, causing silent failure. diff --git a/.claude/skills/adapter-ops/references/provider_capabilities.md b/.claude/skills/adapter-ops/references/provider_capabilities.md index 4a004abae4..5a27a5705b 100644 --- a/.claude/skills/adapter-ops/references/provider_capabilities.md +++ b/.claude/skills/adapter-ops/references/provider_capabilities.md @@ -132,30 +132,39 @@ api_key, api_base, model, max_tokens, max_retries, timeout ### Verification Command -Always verify the provider name before implementing an adapter: +Always verify the provider name before implementing an adapter — against the **pinned** LiteLLM +in the sdk1 venv, not upstream `main`, since `main` may price models the pinned version doesn't: ```bash -# Check all unique litellm_provider values -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.value.litellm_provider != null)) | - map(.value.litellm_provider) | unique | sort' - -# Check specific provider (e.g., for azure_ai models) -curl -s https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | \ - jq 'to_entries | map(select(.key | startswith("azure_ai"))) | .[0].value.litellm_provider' +cd unstract/sdk1 && uv run python -c " +import litellm +# All unique litellm_provider values in the pinned cost map +print(sorted({v['litellm_provider'] for v in litellm.model_cost.values() + if isinstance(v, dict) and v.get('litellm_provider')})) +# Provider for a specific model-key prefix (e.g. azure_ai) +print([v['litellm_provider'] for k, v in litellm.model_cost.items() + if k.startswith('azure_ai')][:1]) +" ``` ### Why This Matters The cost calculation flow: -1. `LLM._record_usage()` calls `Audit.push_usage_data()` with provider from `get_provider()` -2. Platform service receives usage data with provider name -3. `CostCalculationHelper.calculate_cost()` filters models where `provider in litellm_provider` -4. If no match found, cost = $0 - -**Example bug**: If `get_provider()` returns `"azure_ai_foundry"` but LiteLLM uses `"azure_ai"`: -- Check: `"azure_ai_foundry" in "azure_ai"` = `False` -- Result: Cost calculation returns $0 +1. `LLM._record_usage()` logs tokens — no cost math. +2. `Audit.push_usage_data()` (LLM) / `UsageHandler` (embedding) calls + `litellm.cost_per_token(model=model_name)`, where `model_name` is the **prefixed** model + string emitted by `validate_model()`. +3. No cost-map entry for that string → the call raises → the bare `except` records $0. + +The `provider` value from `get_provider()` rides along into the usage row's `provider` column +but is **not** used for pricing. + +**Example bug**: a branded OpenAI-compatible adapter emits `custom_openai/MiniMax-M3`. LiteLLM +prices `minimax/MiniMax-M3` but has no `custom_openai/` keys, so cost silently resolves to $0 — +even though `get_provider()` correctly returns `"minimax"`. + +See `references/adapter_patterns.md` → *Model Prefix Verification* for how to choose a base +class so the prefix resolves. ## Models Supporting Advanced Features diff --git a/.claude/skills/adapter-ops/scripts/check_adapter_updates.py b/.claude/skills/adapter-ops/scripts/check_adapter_updates.py index ec50ed7f76..418046524b 100755 --- a/.claude/skills/adapter-ops/scripts/check_adapter_updates.py +++ b/.claude/skills/adapter-ops/scripts/check_adapter_updates.py @@ -440,7 +440,8 @@ def main(): args = parser.parse_args() - print(f"SDK1 Adapters Path: {SDK1_ADAPTERS}") + # stderr so --json output on stdout stays machine-parseable + print(f"SDK1 Adapters Path: {SDK1_ADAPTERS}", file=sys.stderr) adapter_types = ["llm", "embedding"] if args.adapter == "all" else [args.adapter] results = [] diff --git a/.claude/skills/adapter-ops/scripts/init_embedding_adapter.py b/.claude/skills/adapter-ops/scripts/init_embedding_adapter.py index 0122c39ed7..9835d79233 100755 --- a/.claude/skills/adapter-ops/scripts/init_embedding_adapter.py +++ b/.claude/skills/adapter-ops/scripts/init_embedding_adapter.py @@ -25,9 +25,7 @@ SCRIPT_DIR = Path(__file__).parent SKILL_DIR = SCRIPT_DIR.parent # Find the sdk1 adapters directory relative to the repo root -REPO_ROOT = ( - SKILL_DIR.parent.parent.parent -) # .claude/skills/unstract-adapter-extension -> repo root +REPO_ROOT = SKILL_DIR.parent.parent.parent # .claude/skills/adapter-ops -> repo root SDK1_ADAPTERS = REPO_ROOT / "unstract" / "sdk1" / "src" / "unstract" / "sdk1" / "adapters" ICONS_DIR = REPO_ROOT / "frontend" / "public" / "icons" / "adapter-icons" diff --git a/.claude/skills/adapter-ops/scripts/init_llm_adapter.py b/.claude/skills/adapter-ops/scripts/init_llm_adapter.py index 8226f69eeb..828e04c3dc 100755 --- a/.claude/skills/adapter-ops/scripts/init_llm_adapter.py +++ b/.claude/skills/adapter-ops/scripts/init_llm_adapter.py @@ -26,9 +26,7 @@ SCRIPT_DIR = Path(__file__).parent SKILL_DIR = SCRIPT_DIR.parent # Find the sdk1 adapters directory relative to the repo root -REPO_ROOT = ( - SKILL_DIR.parent.parent.parent -) # .claude/skills/unstract-adapter-extension -> repo root +REPO_ROOT = SKILL_DIR.parent.parent.parent # .claude/skills/adapter-ops -> repo root SDK1_ADAPTERS = REPO_ROOT / "unstract" / "sdk1" / "src" / "unstract" / "sdk1" / "adapters" ICONS_DIR = REPO_ROOT / "frontend" / "public" / "icons" / "adapter-icons" diff --git a/.github/workflows/ci-container-build.yaml b/.github/workflows/ci-container-build.yaml deleted file mode 100644 index aacd0d60e4..0000000000 --- a/.github/workflows/ci-container-build.yaml +++ /dev/null @@ -1,52 +0,0 @@ -name: Container Image Build Test for PRs - -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: - - main - workflow_dispatch: - -jobs: - build: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v6 - - name: Login to Docker Hub - if: github.event.pull_request == null || github.event.pull_request.head.repo.fork == false - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Container Run - run: | - ./run-platform.sh -b - sleep 30 - docker compose -f docker/docker-compose.yaml ps -a - # Get the names of exited containers - custom_format="{{.Name}}\t{{.Image}}\t{{.Service}}" - EXITED_CONTAINERS=$(docker compose -f docker/docker-compose.yaml ps -a --filter status=exited --format "$custom_format") - - line_count=$(echo "$EXITED_CONTAINERS" | wc -l) - - if [ "$line_count" -gt 1 ]; then - echo "Exited Containers: $EXITED_CONTAINERS" - - SERVICE=$(echo "$EXITED_CONTAINERS" | awk 'NR>0 {print $3}') - echo "Exited Services:" - echo "$SERVICE" - echo "There are exited containers." - # Print logs of exited containers - IFS=$'\n' - for SERVICE in $SERVICE; do - docker compose -f docker/docker-compose.yaml logs "$SERVICE" - done - docker compose -f docker/docker-compose.yaml down -v - exit 1 - fi - docker compose -f docker/docker-compose.yaml down -v diff --git a/.github/workflows/ci-test-e2e.yaml b/.github/workflows/ci-test-e2e.yaml deleted file mode 100644 index fb93d3aeaa..0000000000 --- a/.github/workflows/ci-test-e2e.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: Run e2e tests (rig + docker compose) - -# Image-build wiring is not yet in place: the base docker/docker-compose.yaml -# references ${VERSION} for ~15 services without a default, and this workflow -# does not build/pull images or export VERSION/UNSTRACT_TEST_VERSION. Running -# the compose lane on push:main + nightly would fail deterministically (empty- -# tag image refs) before any test executes. Restore the `push: main` and -# `schedule:` triggers once a build step that exports the SHA tag lands. -on: - pull_request: - types: [labeled, synchronize] - branches: [main] - workflow_dispatch: - -jobs: - e2e: - # Only run on PRs that opt in via the `run-e2e` label, plus manual dispatch. - if: > - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-e2e') - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 - with: - version: "0.6.14" - python-version: 3.12.9 - - - name: Install tox with UV - run: uv tool install tox --with tox-uv - - - name: Validate test manifests - run: tox -e rig -- validate - - - name: Restore main-branch test baseline - # See ci-test.yaml for the rationale on namespacing per-workflow. - uses: actions/cache@v5 - with: - path: reports/previous-summary.json - key: unstract-test-baseline-e2e-main-${{ github.run_id }} - restore-keys: | - unstract-test-baseline-e2e-main- - unstract-test-baseline-e2e- - - - name: Run e2e tier via docker compose - env: - UNSTRACT_E2E_RUNTIME: compose - run: | - # Use --tier e2e (not `all`) so this workflow runs only e2e groups. - # The main-branch baseline-update path is preserved for the eventual - # `workflow_dispatch` from main and future restoration of push: main. - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tox -e e2e -- --fail-on-critical-gap --update-baseline - else - tox -e e2e - fi - - - name: Capture docker compose logs on failure - if: failure() - run: | - mkdir -p reports - docker compose -p unstract-test \ - -f docker/docker-compose.yaml \ - -f tests/compose/docker-compose.test.yaml \ - logs --no-color > reports/docker-compose-logs.txt || true - - - name: Output e2e report to job summary - if: always() && hashFiles('reports/summary.md') != '' - shell: bash - run: | - cat reports/summary.md >> $GITHUB_STEP_SUMMARY - - - name: Upload e2e reports artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-reports-e2e - path: reports/ - if-no-files-found: ignore - retention-days: 30 diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml index 13de1475f4..acb7a466ff 100644 --- a/.github/workflows/ci-test.yaml +++ b/.github/workflows/ci-test.yaml @@ -1,31 +1,69 @@ -name: Run unit + integration tests (rig) +name: Run tests (unit + integration + e2e) + +# Multi-job, one report. unit/integration run as parallel matrix legs on +# testcontainers; e2e builds images and boots the full platform via compose on +# its own runner (images don't travel across jobs). The report job needs all +# three, merges every tier's junit into ONE critical-path evaluation, and posts +# a single sticky PR comment — so e2e coverage (e.g. auth-login) shows up next +# to unit/integration. Tiers never run in one rig invocation: the rig picks one +# infra mode per call, so mixing compose e2e with testcontainers integration +# would point the backend at the compose DB hostname, unreachable from pytest. on: push: branches: - main - paths-ignore: - - "docker/**" - - "frontend/**" - - "docs/**" pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: [main] - paths-ignore: - - "docker/**" - - "frontend/**" - - "docs/**" + branches: + - main + workflow_dispatch: jobs: - test: - if: github.event.pull_request.draft == false + # Path filtering at job level, not trigger paths-ignore: an untriggered + # workflow reports no check run, so a required check waits forever; a job + # skipped via `if:` reports conclusion `skipped`, which passes. Gates only + # unit/integration — e2e runs regardless (docker/** changes affect it). + changes: runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Check for changes outside ignored paths + uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: filter + with: + # `every`: a file counts as relevant only if it matches ALL the + # negated globs, i.e. lies outside every ignored dir. + predicate-quantifier: every + filters: | + relevant: + - "!docker/**" + - "!frontend/**" + - "!docs/**" + # Tiers share no state (separate runners, disjoint groups), so they run as + # parallel matrix legs; cross-tier reconciliation happens in `report`. + test: + needs: changes + if: needs.changes.outputs.relevant == 'true' && github.event.pull_request.draft == false + runs-on: ubuntu-latest + strategy: + # One tier failing must not cancel the other; report needs both results. + fail-fast: false + matrix: + tier: [unit, integration] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - fetch-depth: 0 # rig --changed-only needs git history + fetch-depth: 0 # rig --changed-only needs git history + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 @@ -37,62 +75,202 @@ jobs: uses: actions/cache@v5 with: path: .tox/ - key: ${{ runner.os }}-tox-uv-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} + # Tier in the key: legs build different tox envs, and parallel saves + # to one key race (first save wins). + key: ${{ runner.os }}-tox-uv-${{ matrix.tier }}-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} restore-keys: | + ${{ runner.os }}-tox-uv-${{ matrix.tier }}- ${{ runner.os }}-tox-uv- - - name: Restore main-branch test baseline (for regression detection) - # actions/cache only saves on a cache miss. Include the run id in the - # key so each main build writes a fresh cache entry; the prefix in - # restore-keys pulls the most recent baseline. - # - # The unit/integration lane keeps a SEPARATE baseline from the e2e - # workflow because their `scope_groups` don't overlap — restoring an - # e2e-tier baseline here would flag every e2e-covered path as a - # regression in this lane (and vice versa). Each workflow is the - # source of truth for the paths covered by its own tiers. + - name: Install tox with UV + run: uv tool install tox --with tox-uv + + - name: Validate test manifests + # Cheap pre-flight: catches manifest schema errors before tier runs. + run: tox -e rig -- validate + + - name: Run ${{ matrix.tier }} tier + # Gap check on PRs too: main is gap-free and PRs test the merge ref, so + # an in-scope gap here is PR-introduced. No baseline restore or + # --update-baseline — the report job is the sole regression authority + # (a per-tier eval would flag other tiers' paths as missing). + run: tox -e ${{ matrix.tier }} -- --fail-on-critical-gap + + - name: Suffix coverage data for cross-tier merge + # Every tier emits a bare reports/.coverage that would clobber on + # download; suffixed names match the `.coverage.*` glob combine unions. + if: always() + run: | + if [ -f reports/.coverage ]; then + mv reports/.coverage "reports/.coverage.tier-${{ matrix.tier }}" + fi + + - name: Upload tier reports artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: reports-${{ matrix.tier }} + path: reports/ + if-no-files-found: ignore + retention-days: 14 + + # Own runner: builds images then runs e2e against the full compose platform. + # Not gated by `changes` — docker/** edits don't count as "relevant" yet must + # be e2e-tested. + e2e: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + # One tag end to end: docker-compose.build.yaml tags images with VERSION, + # the base compose boots ${VERSION}, and the test overlay pins its + # services via UNSTRACT_TEST_VERSION. Keep them identical. + VERSION: ${{ github.sha }} + UNSTRACT_TEST_VERSION: ${{ github.sha }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Login to Docker Hub + # Authenticated pulls avoid the anonymous rate limit shared by all + # runners. Forks have no access to secrets. + if: github.event.pull_request == null || github.event.pull_request.head.repo.fork == false + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: "0.6.14" + python-version: 3.12.9 + + - name: Cache tox environments uses: actions/cache@v5 with: - path: reports/previous-summary.json - key: unstract-test-baseline-ut-main-${{ github.run_id }} + path: .tox/ + key: ${{ runner.os }}-tox-uv-e2e-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} restore-keys: | - unstract-test-baseline-ut-main- - unstract-test-baseline-ut- + ${{ runner.os }}-tox-uv-e2e- + ${{ runner.os }}-tox-uv- - name: Install tox with UV run: uv tool install tox --with tox-uv - name: Validate test manifests - # Cheap pre-flight: catches groups.yaml / critical_paths.yaml schema - # errors before we spend minutes on tier runs. Also catches malformed - # manifests on PRs that only touch paths-ignored files (because this - # step always runs). + # Cheap pre-flight before the multi-minute image build. run: tox -e rig -- validate - - name: Run unit tier - # Each tier runs as a separate rig invocation so its results land in - # reports// before the next tier starts. --update-baseline (on - # main only) merges this tier's covered paths into the cached - # previous-summary.json; later tiers union on top. + - name: Set up env files + run: ./run-platform.sh -e + + - name: Build service images + run: docker compose -f docker/docker-compose.build.yaml build + + - name: Run e2e tier (rig boots the platform via compose) + env: + UNSTRACT_E2E_RUNTIME: compose + # Gap check only; report owns baseline/regression across all tiers. + run: tox -e e2e -- --fail-on-critical-gap + + - name: Suffix coverage data for cross-tier merge + if: always() run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tox -e unit -- --fail-on-critical-gap --update-baseline - else - tox -e unit + if [ -f reports/.coverage ]; then + mv reports/.coverage "reports/.coverage.tier-e2e" fi - - name: Run integration tier + - name: Capture docker compose logs on failure + if: failure() + run: | + mkdir -p reports + docker compose -p unstract-test \ + -f docker/docker-compose.yaml \ + -f tests/compose/docker-compose.test.yaml \ + logs --no-color > reports/docker-compose-logs.txt || true + + - name: Upload e2e reports artifact if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: reports-e2e + path: reports/ + if-no-files-found: ignore + retention-days: 14 + + report: + needs: [changes, test, e2e] + # `always()` so a red tier still posts a report. e2e always runs on + # non-draft, so gate only on draft; the fail step below distinguishes a + # legitimate skip (irrelevant paths) from a real tier failure. + if: always() && github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: "0.6.14" + python-version: 3.12.9 + + - name: Cache tox environments + uses: actions/cache@v5 + with: + path: .tox/ + key: ${{ runner.os }}-tox-uv-rig-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} + restore-keys: | + ${{ runner.os }}-tox-uv-rig- + ${{ runner.os }}-tox-uv- + + - name: Install tox with UV + run: uv tool install tox --with tox-uv + + - name: Restore main-branch test baseline (for regression detection) + # One unified baseline across all tiers; this job is its sole + # reader+writer. Restore-only here, save below on green main. + uses: actions/cache/restore@v5 + with: + path: reports/previous-summary.json + key: unstract-test-baseline-main-${{ github.run_id }} + restore-keys: | + unstract-test-baseline-main- + + - name: Download tier reports + uses: actions/download-artifact@v4 + with: + pattern: reports-* + path: reports/ + merge-multiple: true + + - name: Combine tiers into one report + # Merges every tier's junit into a single critical-path evaluation + PR + # comment. --update-baseline is main-only and green-only — a PR must not + # write main's baseline, nor a red build a partial one. run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tox -e integration -- --fail-on-critical-gap --update-baseline - else - tox -e integration + flags="" + if [ "${{ github.ref }}" = "refs/heads/main" ] \ + && [ "${{ needs.test.result }}" = "success" ] \ + && [ "${{ needs.e2e.result }}" = "success" ]; then + flags="--update-baseline" fi + tox -e rig -- report combine $flags - - name: Re-aggregate reports from both tiers - if: always() - run: tox -e rig -- report combine + - name: Save updated baseline + # actions/cache saves only on a miss; the run id keys each main build a + # fresh entry, and restore-keys' prefix pulls the latest next run. + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@v5 + with: + path: reports/previous-summary.json + key: unstract-test-baseline-main-${{ github.run_id }} - name: Render combined test report to PR uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2 @@ -105,14 +283,24 @@ jobs: - name: Output combined report to job summary if: always() && hashFiles('reports/summary.md') != '' shell: bash - run: | - cat reports/summary.md >> $GITHUB_STEP_SUMMARY + run: cat reports/summary.md >> $GITHUB_STEP_SUMMARY - - name: Upload reports artifact + - name: Upload combined reports artifact if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: test-reports-unit-integration + name: test-reports path: reports/ if-no-files-found: ignore retention-days: 14 + + - name: Fail if any tier failed + # Single branch-protection check. A skipped `test` means irrelevant + # paths (path-filter skip == pass); e2e always runs, so it must be + # green. Runs after the comment so a red tier still reports. + if: always() + run: | + test_result="${{ needs.test.result }}" + e2e_result="${{ needs.e2e.result }}" + [ "$test_result" = "success" ] || [ "$test_result" = "skipped" ] || exit 1 + [ "$e2e_result" = "success" ] || exit 1 diff --git a/.github/workflows/create-release.yaml b/.github/workflows/create-release.yaml index c30c43fe9c..6f2f92c160 100644 --- a/.github/workflows/create-release.yaml +++ b/.github/workflows/create-release.yaml @@ -4,11 +4,12 @@ on: workflow_dispatch: inputs: version_bump: - description: "Version bump type (hotfix branches force 'patch')" + description: "Version bump ('auto' = minor if a FEAT/GATED-FEAT PR merged since the last release, else patch; hotfix branches force 'patch')" required: true - default: "patch" + default: "auto" type: choice options: + - auto - patch - minor - major @@ -39,6 +40,11 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + # 'auto' version_bump classifies merged PR titles via `gh pr list`, which + # needs read access to pull requests. Declaring an explicit permissions + # block sets every unlisted scope to 'none', so without this the query 403s + # and 'auto' falls back to patch (with a warning) on every run. + pull-requests: read defaults: run: shell: bash -euo pipefail {0} @@ -82,7 +88,9 @@ jobs: env: BUMP: ${{ github.event.inputs.version_bump }} run: | - if [[ "$BUMP" != "patch" ]]; then + # 'auto' is allowed here — the "Resolve auto bump" step maps it to 'patch' + # on a hotfix line (hotfixes are patch-only by definition). + if [[ "$BUMP" != "patch" && "$BUMP" != "auto" ]]; then echo "::error::Hotfix branches only support 'patch' bumps. Got: '$BUMP'" exit 1 fi @@ -158,11 +166,78 @@ jobs: echo "✅ $AHEAD new commit(s) on '$BRANCH' since $LATEST_TAG" fi + - name: Resolve auto bump + id: resolve-bump + if: github.event.inputs.version_bump == 'auto' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MODE: ${{ steps.branch-mode.outputs.mode }} + LATEST_TAG: ${{ steps.get-latest.outputs.latest_tag }} + run: | + # 'auto' only chooses minor-vs-patch on main; hotfix lines are patch-only. + if [[ "$MODE" != "main" ]]; then + echo "resolved=patch" >> "$GITHUB_OUTPUT" + echo "::notice::auto bump on hotfix line -> patch" + exit 0 + fi + + # Classify PRs merged into main after the base release was published: + # any [FEAT]/[GATED-FEAT] title (the only feature PR types in the + # contribution guide) => minor, otherwise patch. Every failure below + # degrades to patch — never over-bump the public version, and never + # hard-fail the release on a transient query hiccup. '// empty' keeps a + # JSON null from leaking through as the literal "null" (matches the + # get-latest step's convention). + SINCE=$(gh api "repos/${{ github.repository }}/releases/tags/$LATEST_TAG" --jq '.published_at // empty' 2>/tmp/gh_since.err || echo "") + if [[ -z "$SINCE" ]]; then + echo "::warning::Could not resolve publish time for $LATEST_TAG; defaulting to patch." + cat /tmp/gh_since.err >&2 || true + echo "resolved=patch" >> "$GITHUB_OUTPUT" + exit 0 + fi + + LIMIT=200 + if ! PR_JSON=$(gh pr list --repo "${{ github.repository }}" \ + --base main --state merged --limit "$LIMIT" --search "merged:>$SINCE" \ + --json title 2>/tmp/gh_pr.err); then + echo "::warning::PR classification query failed; defaulting to patch." + cat /tmp/gh_pr.err >&2 || true # surface 403/permission vs rate-limit/5xx + RESOLVED=patch + else + # Parse defensively: malformed/non-array stdout must degrade to patch, + # not crash the job under `set -e`/pipefail. + TOTAL=$(jq 'length' <<<"$PR_JSON" 2>/dev/null) || TOTAL="" + FEAT_COUNT=$(jq '[.[] | select(.title | test("\\[(FEAT|GATED-FEAT)\\]"; "i"))] | length' <<<"$PR_JSON" 2>/dev/null) || FEAT_COUNT="" + if ! [[ "$TOTAL" =~ ^[0-9]+$ && "$FEAT_COUNT" =~ ^[0-9]+$ ]]; then + echo "::warning::PR classification returned unparseable output; defaulting to patch." + RESOLVED=patch + elif [[ "$FEAT_COUNT" -gt 0 ]]; then + RESOLVED=minor + else + RESOLVED=patch + # Truncation only changes the outcome when no FEAT was found within + # the cap, so only warn in that case (a FEAT beyond it could be missed). + if [[ "$TOTAL" -ge "$LIMIT" ]]; then + echo "::warning::merged-PR query hit the $LIMIT-result cap since $SINCE; a FEAT PR beyond it may be missed — double-check the bump (override with version_bump=minor if needed)." + fi + fi + fi + + echo "resolved=$RESOLVED" >> "$GITHUB_OUTPUT" + if [[ "$RESOLVED" == "minor" ]]; then + echo "::notice::auto bump: $FEAT_COUNT FEAT/GATED-FEAT PR(s) merged since $LATEST_TAG -> minor" + else + echo "::notice::auto bump: no FEAT/GATED-FEAT PR merged since $LATEST_TAG -> patch. Re-run with version_bump=minor to override." + fi + - name: Compute next version id: compute-version env: LATEST_TAG: ${{ steps.get-latest.outputs.latest_tag }} - BUMP_TYPE: ${{ github.event.inputs.version_bump }} + # Non-auto: use the raw input. Auto: use the resolved value, falling back + # to 'patch' (never back to 'auto', which would hit compute-version's + # unknown-bump error). + BUMP_TYPE: ${{ github.event.inputs.version_bump != 'auto' && github.event.inputs.version_bump || steps.resolve-bump.outputs.resolved || 'patch' }} run: | VERSION="${LATEST_TAG#v}" MAJOR=$(echo "$VERSION" | cut -d. -f1) @@ -225,9 +300,12 @@ jobs: LATEST_TAG: ${{ steps.get-latest.outputs.latest_tag }} NEW_TAG: ${{ steps.compute-version.outputs.new_tag }} BUMP: ${{ github.event.inputs.version_bump }} + RESOLVED: ${{ steps.resolve-bump.outputs.resolved }} PRERELEASE: ${{ github.event.inputs.pre_release }} MODE: ${{ steps.branch-mode.outputs.mode }} run: | + BUMP_DISPLAY="$BUMP" + [[ "$BUMP" == "auto" ]] && BUMP_DISPLAY="auto -> ${RESOLVED}" { echo "## Dry Run Summary" echo "" @@ -235,7 +313,7 @@ jobs: echo "|-------|-------|" echo "| Mode | $MODE |" echo "| Current version | $LATEST_TAG |" - echo "| Bump type | $BUMP |" + echo "| Bump type | $BUMP_DISPLAY |" echo "| Next version | $NEW_TAG |" echo "| Pre-release | $PRERELEASE |" echo "" @@ -324,9 +402,12 @@ jobs: LATEST_TAG: ${{ steps.get-latest.outputs.latest_tag }} NEW_TAG: ${{ steps.compute-version.outputs.new_tag }} BUMP: ${{ github.event.inputs.version_bump }} + RESOLVED: ${{ steps.resolve-bump.outputs.resolved }} PRERELEASE: ${{ github.event.inputs.pre_release }} MODE: ${{ steps.branch-mode.outputs.mode }} run: | + BUMP_DISPLAY="$BUMP" + [[ "$BUMP" == "auto" ]] && BUMP_DISPLAY="auto -> ${RESOLVED}" { echo "## Release Created" echo "" @@ -335,7 +416,7 @@ jobs: echo "| Mode | $MODE |" echo "| Previous version | $LATEST_TAG |" echo "| New version | $NEW_TAG |" - echo "| Bump type | $BUMP |" + echo "| Bump type | $BUMP_DISPLAY |" echo "| Pre-release | $PRERELEASE |" echo "| Triggered by | ${{ github.actor }} |" echo "" diff --git a/.github/workflows/docker-tools-build-push.yaml b/.github/workflows/docker-tools-build-push.yaml index 439f4fbe81..b18507a9ac 100644 --- a/.github/workflows/docker-tools-build-push.yaml +++ b/.github/workflows/docker-tools-build-push.yaml @@ -10,13 +10,12 @@ on: service_name: description: "Tool to build" required: true - default: "tool-structure" # Provide a default value + default: "tool-sidecar" # Provide a default value type: choice options: # Define available options + - tool-sidecar - tool-classifier - - tool-structure - tool-text-extractor - - tool-sidecar add_latest_tag: description: "Also tag as 'latest'" required: false @@ -58,18 +57,15 @@ jobs: - name: Set build configuration id: build-config run: | - if [ "${{ github.event.inputs.service_name }}" == "tool-classifier" ]; then + if [ "${{ github.event.inputs.service_name }}" == "tool-sidecar" ]; then echo "context=." >> $GITHUB_OUTPUT - echo "dockerfile=./tools/classifier/Dockerfile" >> $GITHUB_OUTPUT - elif [ "${{ github.event.inputs.service_name }}" == "tool-structure" ]; then + echo "dockerfile=docker/dockerfiles/tool-sidecar.Dockerfile" >> $GITHUB_OUTPUT + elif [ "${{ github.event.inputs.service_name }}" == "tool-classifier" ]; then echo "context=." >> $GITHUB_OUTPUT - echo "dockerfile=./tools/structure/Dockerfile" >> $GITHUB_OUTPUT + echo "dockerfile=./tools/classifier/Dockerfile" >> $GITHUB_OUTPUT elif [ "${{ github.event.inputs.service_name }}" == "tool-text-extractor" ]; then echo "context=." >> $GITHUB_OUTPUT echo "dockerfile=./tools/text_extractor/Dockerfile" >> $GITHUB_OUTPUT - elif [ "${{ github.event.inputs.service_name }}" == "tool-sidecar" ]; then - echo "context=." >> $GITHUB_OUTPUT - echo "dockerfile=docker/dockerfiles/tool-sidecar.Dockerfile" >> $GITHUB_OUTPUT fi # Determine tags based on inputs diff --git a/.github/workflows/production-build.yaml b/.github/workflows/production-build.yaml index 21edb520f7..8a9745b309 100644 --- a/.github/workflows/production-build.yaml +++ b/.github/workflows/production-build.yaml @@ -101,7 +101,6 @@ jobs: backend, frontend, platform-service, - prompt-service, runner, worker-unified, x2text-service, @@ -225,7 +224,7 @@ jobs: id: summary run: | # Initialize variables - TOTAL_SERVICES=7 + TOTAL_SERVICES=6 OVERALL_RESULT='${{ needs.build-and-push.result }}' SUCCESS_COUNT=0 FAILED_COUNT=0 @@ -316,7 +315,7 @@ jobs: echo "|---------|--------|" >> $GITHUB_STEP_SUMMARY # Define services in order - for service in backend frontend platform-service prompt-service runner worker-unified x2text-service; do + for service in backend frontend platform-service runner worker-unified x2text-service; do if [ -f "build-status/${service}.json" ]; then STATUS=$(jq -r '.status' "build-status/${service}.json") if [ "$STATUS" = "success" ]; then diff --git a/backend/account_v2/migrations/0006_organization_restrict_llm_adapter_creation.py b/backend/account_v2/migrations/0006_organization_restrict_llm_adapter_creation.py new file mode 100644 index 0000000000..591b7a4f1b --- /dev/null +++ b/backend/account_v2/migrations/0006_organization_restrict_llm_adapter_creation.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("account_v2", "0005_alter_platformkey_organization"), + ] + + operations = [ + migrations.AddField( + model_name="organization", + name="restrict_llm_adapter_creation", + field=models.BooleanField( + db_comment=( + "Controlled mode: when True, only organization admins may " + "create LLM adapters in this org. Default False preserves " + "open creation." + ), + default=False, + ), + ), + ] diff --git a/backend/account_v2/migrations/0007_organization_restrict_connector_creation.py b/backend/account_v2/migrations/0007_organization_restrict_connector_creation.py new file mode 100644 index 0000000000..faeefc09a1 --- /dev/null +++ b/backend/account_v2/migrations/0007_organization_restrict_connector_creation.py @@ -0,0 +1,22 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("account_v2", "0006_organization_restrict_llm_adapter_creation"), + ] + + operations = [ + migrations.AddField( + model_name="organization", + name="restrict_connector_creation", + field=models.BooleanField( + db_comment=( + "Controlled mode: when True, only organization admins may " + "create connectors in this org. Default False preserves " + "open creation." + ), + default=False, + ), + ), + ] diff --git a/backend/account_v2/models.py b/backend/account_v2/models.py index eb6b81d8b3..e532a9a456 100644 --- a/backend/account_v2/models.py +++ b/backend/account_v2/models.py @@ -39,6 +39,20 @@ class Organization(models.Model): default=-1, db_comment="token limit set in case of frition less onbaoarded org", ) + restrict_llm_adapter_creation = models.BooleanField( + default=False, + db_comment=( + "Controlled mode: when True, only organization admins may create " + "LLM adapters in this org. Default False preserves open creation." + ), + ) + restrict_connector_creation = models.BooleanField( + default=False, + db_comment=( + "Controlled mode: when True, only organization admins may create " + "connectors in this org. Default False preserves open creation." + ), + ) class Meta: verbose_name = "Organization" diff --git a/backend/adapter_processor_v2/tests/test_adapter_api.py b/backend/adapter_processor_v2/tests/test_adapter_api.py new file mode 100644 index 0000000000..5df0d748f6 --- /dev/null +++ b/backend/adapter_processor_v2/tests/test_adapter_api.py @@ -0,0 +1,61 @@ +"""Critical path ``adapter-register-llm``: POST /api/v1/adapter/ registers an +LLM adapter. Exercises the real endpoint wiring — auth, serializer, metadata +encryption, org-scoped persistence — with only the SDK context-window lookup +(a provider-shaped call) mocked. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import secrets +from unittest.mock import patch + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from adapter_processor_v2.models import AdapterInstance +from adapter_processor_v2.views import AdapterInstanceViewSet + + +class AdapterRegisterLLMAPITest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-a", display_name="Org A", organization_id="org-a" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="owner@example.com", + email="owner@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.create_view = AdapterInstanceViewSet.as_view({"post": "create"}) + + @pytest.mark.critical_path("adapter-register-llm") + @patch.object(AdapterInstance, "get_context_window_size", return_value=4096) + def test_register_llm_adapter_persists_encrypted(self, _ctx_window) -> None: + payload = { + "adapter_id": "openai|test-llm", + "adapter_name": "my-openai", + "adapter_type": "LLM", + "adapter_metadata": {"api_key": "sk-test", "model": "gpt-4o-mini"}, + } + request = APIRequestFactory().post("/api/v1/adapter/", payload, format="json") + force_authenticate(request, user=self.user) + + response = self.create_view(request) + + assert response.status_code == status.HTTP_201_CREATED, response.data + instance = AdapterInstance.objects.get(adapter_name="my-openai") + # persisted under the request user's org, created_by the request user + assert instance.organization_id == self.org.id + assert instance.created_by == self.user + # metadata stored encrypted (binary), decrypts back via .metadata + assert instance.adapter_metadata_b is not None + assert instance.metadata["model"] == "gpt-4o-mini" diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.py index 4cfee77d01..2d500588ed 100644 --- a/backend/adapter_processor_v2/views.py +++ b/backend/adapter_processor_v2/views.py @@ -16,6 +16,7 @@ from plugins import get_plugin from rest_framework import status from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied from rest_framework.request import Request from rest_framework.response import Response from rest_framework.serializers import ModelSerializer @@ -24,6 +25,7 @@ from tenant_account_v2.organization_member_service import OrganizationMemberService from tool_instance_v2.models import ToolInstance from utils.filtering import FilterHelper +from utils.user_context import UserContext from adapter_processor_v2.adapter_processor import AdapterProcessor from adapter_processor_v2.constants import AdapterKeys @@ -173,6 +175,28 @@ def get_serializer_class( return AdapterListSerializer return AdapterInstanceSerializer + @staticmethod + def _enforce_llm_creation_restriction(request: Any, adapter_type: str) -> None: + """Controlled mode (UN-3584): only org admins may create LLM adapters + when the org has enabled the restriction. Service accounts (platform + API-key sessions) and non-LLM adapter types bypass; the default + (flag off) keeps creation open for everyone. + """ + if adapter_type != AdapterKeys.LLM: + return + if getattr(request.user, "is_service_account", False): + return + organization = UserContext.get_organization() + if ( + organization + and organization.restrict_llm_adapter_creation + and not OrganizationMemberService.is_user_organization_admin(request.user) + ): + raise PermissionDenied( + "LLM adapter creation is restricted to organization admins. " + "Please contact your organization admin." + ) + def create(self, request: Any) -> Response: serializer = self.get_serializer(data=request.data) @@ -184,9 +208,10 @@ def create(self, request: Any) -> Response: use_platform_unstract_key = True serializer.is_valid(raise_exception=True) - try: - adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE) + adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE) + self._enforce_llm_creation_restriction(request, adapter_type) + try: if adapter_type == AdapterKeys.X2TEXT and use_platform_unstract_key: adapter_metadata_b = serializer.validated_data.get( AdapterKeys.ADAPTER_METADATA_B @@ -199,7 +224,12 @@ def create(self, request: Any) -> Response: adapter_metadata_b ) - instance = serializer.save() + # Bind the adapter to the request-scoped organization, overriding any + # client-supplied `organization` in the payload (the serializer + # exposes it via fields="__all__"). This keeps the row's org + # consistent with the org the controlled-mode check above evaluated, + # so the per-org restriction can't be sidestepped via the payload. + instance = serializer.save(organization=UserContext.get_organization()) organization_member = OrganizationMemberService.get_user_by_id( request.user.id ) diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py index 28cdf939a6..b6b3a8d16f 100644 --- a/backend/api_v2/deployment_helper.py +++ b/backend/api_v2/deployment_helper.py @@ -236,13 +236,41 @@ def execute_workflow( f"API hub header caching failed for execution {execution_id}: {e}" ) - hash_values_of_files = SourceConnector.add_input_file_to_api_storage( - pipeline_id=pipeline_id, - workflow_id=workflow_id, - execution_id=execution_id, - file_objs=file_objs, - use_file_history=use_file_history, - ) + # Staging runs synchronously, before async dispatch. Scope it to its own + # try/except so a failure marks the still-PENDING execution ERROR (else the + # row is orphaned and the UI shows the run as stuck), and so error-marking + # never runs in the post-dispatch path where it could overwrite the status + # of an already-running execution. + try: + hash_values_of_files = SourceConnector.add_input_file_to_api_storage( + pipeline_id=pipeline_id, + workflow_id=workflow_id, + execution_id=execution_id, + file_objs=file_objs, + use_file_history=use_file_history, + ) + except Exception as error: + # Isolate the DB error-marking so cleanup still runs if it raises. + try: + WorkflowExecutionServiceHelper.update_execution_err( + str(execution_id), str(error) + ) + except Exception: + logger.exception(f"Failed to mark execution {execution_id} as ERROR") + + # Async job never started — release the rate limit slot and clean up. + APIDeploymentRateLimiter.release_slot(api.organization, str(execution_id)) + DestinationConnector.delete_api_storage_dir( + workflow_id=workflow_id, execution_id=execution_id + ) + return APIExecutionResponseSerializer( + ExecutionResponse( + workflow_id=workflow_id, + execution_id=execution_id, + execution_status=ExecutionStatus.ERROR.value, + error=str(error), + ) + ).data try: result = WorkflowHelper.execute_workflow_async( @@ -287,7 +315,9 @@ def execute_workflow( if not include_metrics: result.remove_result_metrics() except Exception as error: - # Release rate limit slot (workflow setup/dispatch failed, async job not started) + # Dispatch failures are marked ERROR internally by execute_workflow_async; + # post-dispatch failures (enrichment/config) must not overwrite a running + # execution's status, so only release the slot and clean up storage here. APIDeploymentRateLimiter.release_slot(api.organization, str(execution_id)) # Clean up storage diff --git a/backend/api_v2/postman_collection/constants.py b/backend/api_v2/postman_collection/constants.py index 7bcbe988e0..117bcad98c 100644 --- a/backend/api_v2/postman_collection/constants.py +++ b/backend/api_v2/postman_collection/constants.py @@ -6,4 +6,8 @@ class CollectionKey: EXECUTE_PIPELINE_API_KEY = "Process pipeline" STATUS_API_KEY = "Execution status" STATUS_EXEC_ID_DEFAULT = "REPLACE_WITH_EXECUTION_ID" + EXEC_ID_VARIABLE_NAME = "execution_id" + # Derive the Postman variable reference from the variable name so the two + # can never drift apart (-> "{{execution_id}}"). + STATUS_EXEC_ID_VARIABLE = f"{{{{{EXEC_ID_VARIABLE_NAME}}}}}" AUTH_QUERY_PARAM_DEFAULT = "REPLACE_WITH_API_KEY" diff --git a/backend/api_v2/postman_collection/dto.py b/backend/api_v2/postman_collection/dto.py index 17a6865eea..1118af3df0 100644 --- a/backend/api_v2/postman_collection/dto.py +++ b/backend/api_v2/postman_collection/dto.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import asdict, dataclass, field -from typing import Any +from typing import Any, Literal from urllib.parse import urlencode, urljoin from django.conf import settings @@ -18,6 +18,24 @@ class HeaderItem: value: str +@dataclass +class ScriptItem: + exec: list[str] + type: Literal["text/javascript"] = "text/javascript" + + +@dataclass +class EventItem: + listen: Literal["prerequest", "test"] + script: ScriptItem + + +@dataclass +class VariableItem: + key: str + value: str + + @dataclass class FormDataItem: key: str @@ -55,6 +73,7 @@ class RequestItem: class PostmanItem: name: str request: RequestItem + event: list[EventItem] = field(default_factory=list) @dataclass @@ -69,6 +88,12 @@ class APIBase(ABC): def get_form_data_items(self) -> list[FormDataItem]: pass + def get_collection_variables(self) -> list[VariableItem]: + """Collection-level variables; only needed when a request + references them. + """ + return [] + @abstractmethod def get_api_endpoint(self) -> str: pass @@ -137,20 +162,61 @@ def get_api_endpoint(self) -> str: def _get_status_api_request(self) -> RequestItem: header_list = [HeaderItem(key="Authorization", value=f"Bearer {self.api_key}")] status_query_param = { - "execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT, + "execution_id": CollectionKey.STATUS_EXEC_ID_VARIABLE, ApiExecution.INCLUDE_METADATA: "False", ApiExecution.INCLUDE_METRICS: "False", } - status_query_str = urlencode(status_query_param) + # Keep {{...}} unescaped so Postman resolves the collection variable + status_query_str = urlencode(status_query_param, safe="{}") abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.api_endpoint) status_url = urljoin(abs_api_endpoint, "?" + status_query_str) return RequestItem(method=HTTPMethod.GET, header=header_list, url=status_url) + def get_collection_variables(self) -> list[VariableItem]: + return [ + VariableItem( + key=CollectionKey.EXEC_ID_VARIABLE_NAME, + value=CollectionKey.STATUS_EXEC_ID_DEFAULT, + ) + ] + + def _get_execute_capture_event(self) -> EventItem: + """Post-response script that stores the execution_id from the execute + response into a collection variable, so the status request can use it + without manual copy-pasting. + + This is Postman's ``test`` hook (the emitted event uses + ``listen="test"``). It assumes the execute response shape + ``{"message": {"execution_id": ...}}``; if the body is non-JSON or + lacks that field it captures nothing and resets the variable to the + default sentinel so a stale id from a previous run is never reused. + """ + return EventItem( + listen="test", + script=ScriptItem( + exec=[ + "let response = null;", + "try {", + " response = pm.response.json();", + "} catch (error) {", + " // Non-JSON response (e.g. gateway error); nothing to capture", + "}", + "if (response && response.message && response.message.execution_id) {", # noqa: E501 + f' pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}", response.message.execution_id);', # noqa: E501 + "} else {", + f' pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}", "{CollectionKey.STATUS_EXEC_ID_DEFAULT}");', # noqa: E501 + ' console.warn("No execution_id in execute response; not reusing a stale value. Status:", pm.response.code);', # noqa: E501 + "}", + ] + ), + ) + def get_postman_items(self) -> list[PostmanItem]: postman_item_list = [ PostmanItem( name=CollectionKey.EXECUTE_API_KEY, request=self.get_create_api_request(), + event=[self._get_execute_capture_event()], ), PostmanItem( name=CollectionKey.STATUS_API_KEY, @@ -192,6 +258,7 @@ def get_postman_items(self) -> list[PostmanItem]: class PostmanCollection: info: PostmanInfo item: list[PostmanItem] = field(default_factory=list) + variable: list[VariableItem] = field(default_factory=list) @classmethod def create( @@ -228,7 +295,11 @@ def create( ) postman_info: PostmanInfo = data_object.get_postman_info() postman_item_list = data_object.get_postman_items() - return cls(info=postman_info, item=postman_item_list) + return cls( + info=postman_info, + item=postman_item_list, + variable=data_object.get_collection_variables(), + ) def to_dict(self) -> dict[str, Any]: """Convert PostmanCollection instance to a dict. @@ -237,4 +308,9 @@ def to_dict(self) -> dict[str, Any]: dict[str, Any]: PostmanCollection as a dict """ collection_dict = asdict(self) + # Omit empty event blocks; items without scripts (e.g. pipelines) + # should not carry an "event" key at all. + for item in collection_dict.get("item", []): + if not item.get("event"): + item.pop("event", None) return collection_dict diff --git a/prompt-service/src/unstract/prompt_service/__init__.py b/backend/api_v2/postman_collection/tests/__init__.py similarity index 100% rename from prompt-service/src/unstract/prompt_service/__init__.py rename to backend/api_v2/postman_collection/tests/__init__.py diff --git a/backend/api_v2/postman_collection/tests/test_dto.py b/backend/api_v2/postman_collection/tests/test_dto.py new file mode 100644 index 0000000000..b05048fb65 --- /dev/null +++ b/backend/api_v2/postman_collection/tests/test_dto.py @@ -0,0 +1,154 @@ +"""Regression tests for api_v2.postman_collection.dto — the builder that +turns an API deployment or pipeline into a Postman v2.1 collection. + +UN-2190: the execute request gained a post-response capture script and the +status request reuses the captured ``execution_id`` collection variable. +These tests pin the contract so a refactor can't reopen the gaps the +review flagged: the null-event strip, the variable/URL/JS all referencing +the SAME constant, and the status URL keeping ``{{execution_id}}`` literal +(not percent-encoded). + +The DTOs are constructed directly (no DB) and only ``WEB_APP_ORIGIN_URL`` +is overridden, so the suite is cheap and needs no fixtures. +""" + +from __future__ import annotations + +import pytest +from django.test import override_settings + +from api_v2.postman_collection.constants import CollectionKey +from api_v2.postman_collection.dto import ( + APIDeploymentDto, + PipelineDto, + PostmanCollection, +) + +WEB_APP_ORIGIN_URL = "https://example.unstract.com" +API_ENDPOINT = "deployment/api/org/my-api/" +API_KEY = "test-api-key" + + +def _api_deployment_dto() -> APIDeploymentDto: + return APIDeploymentDto( + display_name="My API", + description="An API deployment", + api_endpoint=API_ENDPOINT, + api_key=API_KEY, + ) + + +def _pipeline_dto() -> PipelineDto: + return PipelineDto( + pipeline_name="My Pipeline", + api_endpoint=API_ENDPOINT, + api_key=API_KEY, + ) + + +def _build_dict(data_object) -> dict: + """Mirror PostmanCollection.create without touching the DB. + + Pins WEB_APP_ORIGIN_URL so URL building is deterministic regardless of + the ambient test settings. + """ + with override_settings(WEB_APP_ORIGIN_URL=WEB_APP_ORIGIN_URL): + collection = PostmanCollection( + info=data_object.get_postman_info(), + item=data_object.get_postman_items(), + variable=data_object.get_collection_variables(), + ) + return collection.to_dict() + + +def _item_by_name(collection_dict: dict, name: str) -> dict: + for item in collection_dict["item"]: + if item["name"] == name: + return item + raise AssertionError(f"item {name!r} not found in collection") + + +class TestApiDeploymentCollection: + def test_execute_item_keeps_populated_event(self) -> None: + """to_dict keeps the populated execute event and its capture script.""" + collection_dict = _build_dict(_api_deployment_dto()) + execute_item = _item_by_name(collection_dict, CollectionKey.EXECUTE_API_KEY) + + assert "event" in execute_item + assert execute_item["event"], "execute event must not be stripped" + script_lines = execute_item["event"][0]["script"]["exec"] + joined = "\n".join(script_lines) + # The capture must target the same constant the variable/URL use. + assert ( + f'pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}"' + in joined + ) + + def test_capture_script_resets_on_missing_execution_id(self) -> None: + """The else branch resets to the sentinel so a stale id is never reused.""" + collection_dict = _build_dict(_api_deployment_dto()) + execute_item = _item_by_name(collection_dict, CollectionKey.EXECUTE_API_KEY) + joined = "\n".join(execute_item["event"][0]["script"]["exec"]) + + assert "else" in joined + assert ( + f'pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}", ' + f'"{CollectionKey.STATUS_EXEC_ID_DEFAULT}")' in joined + ) + assert "console.warn" in joined + + def test_status_item_has_no_event_key(self) -> None: + """The status request carries no scripts -> no 'event' key at all.""" + collection_dict = _build_dict(_api_deployment_dto()) + status_item = _item_by_name(collection_dict, CollectionKey.STATUS_API_KEY) + assert "event" not in status_item + + def test_variable_array_references_the_constant(self) -> None: + """The collection 'variable' array exposes the exec-id variable.""" + collection_dict = _build_dict(_api_deployment_dto()) + variables = collection_dict["variable"] + assert any( + v["key"] == CollectionKey.EXEC_ID_VARIABLE_NAME for v in variables + ) + + def test_status_url_keeps_unencoded_variable_reference(self) -> None: + """Status URL keeps execution_id={{execution_id}} literal, not %7B-encoded.""" + collection_dict = _build_dict(_api_deployment_dto()) + status_item = _item_by_name(collection_dict, CollectionKey.STATUS_API_KEY) + url = status_item["request"]["url"] + + assert ( + f"{CollectionKey.EXEC_ID_VARIABLE_NAME}=" + f"{CollectionKey.STATUS_EXEC_ID_VARIABLE}" in url + ) + assert "%7B" not in url and "%7D" not in url + + +class TestCollectionShapeRegression: + def test_pipeline_carries_no_event_and_empty_variable(self) -> None: + """PipelineDto: no scripts and no collection variables.""" + collection_dict = _build_dict(_pipeline_dto()) + assert collection_dict["variable"] == [] + for item in collection_dict["item"]: + assert "event" not in item + + def test_api_deployment_carries_event_and_variable(self) -> None: + """APIDeploymentDto: execute event present and variable populated.""" + collection_dict = _build_dict(_api_deployment_dto()) + execute_item = _item_by_name(collection_dict, CollectionKey.EXECUTE_API_KEY) + assert execute_item.get("event") + assert collection_dict["variable"], "api deployment must expose variables" + + +class TestConstantCoupling: + def test_status_variable_derived_from_name(self) -> None: + """STATUS_EXEC_ID_VARIABLE must be the {{...}} form of the var name.""" + assert ( + CollectionKey.STATUS_EXEC_ID_VARIABLE + == f"{{{{{CollectionKey.EXEC_ID_VARIABLE_NAME}}}}}" + ) + assert CollectionKey.STATUS_EXEC_ID_VARIABLE == "{{execution_id}}" + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index 282adc8a4a..58bcd8c175 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -523,6 +523,7 @@ class DeploymentResponseSerializer(Serializer): class APIExecutionResponseSerializer(Serializer): + execution_id = CharField() execution_status = CharField() status_api = CharField() error = CharField() diff --git a/prompt-service/src/unstract/prompt_service/helpers/__init__.py b/backend/api_v2/tests/__init__.py similarity index 100% rename from prompt-service/src/unstract/prompt_service/helpers/__init__.py rename to backend/api_v2/tests/__init__.py diff --git a/backend/api_v2/tests/test_deployment_auth.py b/backend/api_v2/tests/test_deployment_auth.py new file mode 100644 index 0000000000..ccbf307fa7 --- /dev/null +++ b/backend/api_v2/tests/test_deployment_auth.py @@ -0,0 +1,109 @@ +"""Critical path ``api-deployment-auth``: the public deployment endpoint +rejects unauthenticated callers before any work is dispatched. + +``POST /deployment/api///`` is the only unauthenticated surface +in the product — it is guarded by the ``@DeploymentHelper.validate_api_key`` +decorator rather than DRF permission classes, so a regression here silently +opens the endpoint. Every rejection must land before ``execute_workflow``, i.e. +before a ``WorkflowExecution`` row is written or a Celery task is queued; +the mock below asserts exactly that. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import uuid +from unittest.mock import patch + +import pytest +from account_v2.models import Organization +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from rest_framework.test import APIRequestFactory +from utils.user_context import UserContext +from workflow_manager.workflow_v2.models.workflow import Workflow + +from api_v2.api_deployment_views import DeploymentExecution +from api_v2.models import APIDeployment, APIKey + +ORG_ID = "org-auth" + + +class APIDeploymentAuthTest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name=ORG_ID, display_name="Org Auth", organization_id=ORG_ID + ) + UserContext.set_organization_identifier(ORG_ID) + workflow = Workflow.objects.create(workflow_name="wf-auth", is_active=True) + + self.api = APIDeployment.objects.create(api_name="live-api", workflow=workflow) + self.key = APIKey.objects.create(api=self.api) + self.inactive_key = APIKey.objects.create(api=self.api, is_active=False) + + self.inactive_api = APIDeployment.objects.create( + api_name="dead-api", workflow=workflow, is_active=False + ) + self.other_api = APIDeployment.objects.create( + api_name="other-api", workflow=workflow + ) + self.other_key = APIKey.objects.create(api=self.other_api) + + self.view = DeploymentExecution.as_view() + self.factory = APIRequestFactory() + + def _post(self, api_name: str, auth: str | None, org: str = ORG_ID): + headers = {"HTTP_AUTHORIZATION": auth} if auth is not None else {} + payload = {"files": SimpleUploadedFile("doc.txt", b"hello")} + request = self.factory.post( + f"/deployment/api/{org}/{api_name}/", payload, format="multipart", **headers + ) + return self.view(request, org_name=org, api_name=api_name) + + @pytest.mark.critical_path("api-deployment-auth") + @patch("api_v2.api_deployment_views.DeploymentHelper.execute_workflow") + def test_bad_credentials_rejected_before_dispatch(self, execute_workflow) -> None: + cases = [ + ("missing header", "live-api", None, ORG_ID, 403), + ("no bearer prefix", "live-api", f"Token {self.key.api_key}", ORG_ID, 403), + ("empty bearer", "live-api", "Bearer ", ORG_ID, 403), + ("unknown key", "live-api", f"Bearer {uuid.uuid4()}", ORG_ID, 401), + ("not a uuid", "live-api", "Bearer not-a-uuid", ORG_ID, 401), + ( + "inactive key", + "live-api", + f"Bearer {self.inactive_key.api_key}", + ORG_ID, + 401, + ), + ( + "key of another api", + "live-api", + f"Bearer {self.other_key.api_key}", + ORG_ID, + 401, + ), + ("unknown api", "ghost-api", f"Bearer {self.key.api_key}", ORG_ID, 404), + ("inactive api", "dead-api", f"Bearer {self.key.api_key}", ORG_ID, 404), + ("wrong org", "live-api", f"Bearer {self.key.api_key}", "no-such-org", 404), + ] + for label, api_name, auth, org, expected in cases: + with self.subTest(label): + response = self._post(api_name, auth, org) + assert response.status_code == expected, response.data + + execute_workflow.assert_not_called() + + @pytest.mark.critical_path("api-deployment-auth") + @patch("api_v2.api_deployment_views.APIDeploymentRateLimiter.check_and_acquire") + @patch("api_v2.api_deployment_views.DeploymentHelper.execute_workflow") + def test_valid_key_reaches_execution(self, execute_workflow, rate_limit) -> None: + """Guard the inverse of the rejection cases: a guard that rejected + everything would pass them all. + """ + rate_limit.return_value = (True, {}) + execute_workflow.return_value = {"execution_status": "COMPLETED"} + + response = self._post("live-api", f"Bearer {self.key.api_key}") + + assert response.status_code == 200, response.data + execute_workflow.assert_called_once() diff --git a/backend/api_v2/tests/test_deployment_helper.py b/backend/api_v2/tests/test_deployment_helper.py new file mode 100644 index 0000000000..39b23e5b16 --- /dev/null +++ b/backend/api_v2/tests/test_deployment_helper.py @@ -0,0 +1,91 @@ +"""Regression tests for synchronous staging failures in ``execute_workflow``. + +When an API-deployment run fails synchronously at the "Staging files in API +storage" step (``SourceConnector.add_input_file_to_api_storage``, before async +dispatch), the PENDING ``WorkflowExecution`` row created earlier must be marked +ERROR — otherwise the UI shows the run as stuck/running forever. The +error-marking is isolated so the rate-limit slot release and storage cleanup +still run even if that DB write fails. + +Unit tests: the real ``execute_workflow`` control flow runs with every +DB/storage-touching collaborator patched on the imported module, so no +database is needed. +""" + +from unittest import mock +from unittest.mock import MagicMock + +import pytest + +import api_v2.deployment_helper as dh + + +@pytest.fixture +def collaborators(): + """Patch execute_workflow's collaborators; staging fails with 'boom'.""" + with mock.patch.multiple( + dh, + WorkflowExecutionServiceHelper=mock.DEFAULT, + SourceConnector=mock.DEFAULT, + DestinationConnector=mock.DEFAULT, + APIDeploymentRateLimiter=mock.DEFAULT, + WorkflowHelper=mock.DEFAULT, + Tag=mock.DEFAULT, + logger=mock.DEFAULT, + ) as mocks: + execution_row = MagicMock() + execution_row.id = "exec-123" + mocks[ + "WorkflowExecutionServiceHelper" + ].create_workflow_execution.return_value = execution_row + mocks["SourceConnector"].add_input_file_to_api_storage.side_effect = ( + RuntimeError("boom") + ) + yield mocks + + +def _api() -> MagicMock: + api = MagicMock() + api.workflow.id = "wf-1" + api.id = "pipe-1" + return api + + +def test_staging_failure_marks_execution_error(collaborators) -> None: + """A staging failure marks the execution ERROR instead of leaving it PENDING.""" + # Must NOT raise — the failure should be handled, not propagated. + dh.DeploymentHelper.execute_workflow( + organization_name="org", + api=_api(), + file_objs=[], + timeout=-1, + ) + + # The PENDING row is marked ERROR with the surfaced reason. + collaborators[ + "WorkflowExecutionServiceHelper" + ].update_execution_err.assert_called_once_with("exec-123", "boom") + # And the slot/storage cleanup still runs. + collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once() + collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once() + # Async dispatch is never reached when staging fails. + collaborators["WorkflowHelper"].execute_workflow_async.assert_not_called() + + +def test_staging_failure_cleanup_survives_db_marking_error(collaborators) -> None: + """If marking the row ERROR itself raises, cleanup must still run (not propagate).""" + collaborators["WorkflowExecutionServiceHelper"].update_execution_err.side_effect = ( + RuntimeError("db down") + ) + + # Must NOT raise — a failed error-marking should not break cleanup. + dh.DeploymentHelper.execute_workflow( + organization_name="org", + api=_api(), + file_objs=[], + timeout=-1, + ) + + # Cleanup still runs even though error-marking raised. + collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once() + collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once() diff --git a/backend/api_v2/tests/test_deployment_provision.py b/backend/api_v2/tests/test_deployment_provision.py new file mode 100644 index 0000000000..5f59630194 --- /dev/null +++ b/backend/api_v2/tests/test_deployment_provision.py @@ -0,0 +1,88 @@ +"""Critical path ``api-deployment-provision``: deploying a workflow as an API +mints a usable key and a resolvable endpoint. + +Provisioning is the only place an API key is created for a deployment, and the +endpoint string it derives is what callers POST to. Both are returned exactly +once — at create time — so a regression here is unrecoverable for the caller. +Fully synchronous. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import secrets + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext +from workflow_manager.endpoint_v2.models import WorkflowEndpoint +from workflow_manager.workflow_v2.models.workflow import Workflow + +from api_v2.api_deployment_views import APIDeploymentViewSet +from api_v2.api_key_views import APIKeyViewSet +from api_v2.models import APIDeployment, APIKey + +ORG_ID = "org-provision" + + +class APIDeploymentProvisionTest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name=ORG_ID, display_name="Org Provision", organization_id=ORG_ID + ) + UserContext.set_organization_identifier(ORG_ID) + self.user = User.objects.create_user( + username="deployer@example.com", + email="deployer@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.workflow = Workflow.objects.create(workflow_name="wf-deploy", is_active=True) + for endpoint_type in WorkflowEndpoint.EndpointType: + WorkflowEndpoint.objects.create( + workflow=self.workflow, + endpoint_type=endpoint_type, + connection_type=WorkflowEndpoint.ConnectionType.API, + ) + self.factory = APIRequestFactory() + + @pytest.mark.critical_path("api-deployment-provision") + def test_deploy_mints_key_and_endpoint(self) -> None: + create = APIDeploymentViewSet.as_view({"post": "create"}) + request = self.factory.post( + "/api/v1/api/deployment/", + { + "workflow": str(self.workflow.id), + "display_name": "my api", + "api_name": "my-api", + "description": "provisioned in a test", + }, + format="json", + ) + force_authenticate(request, user=self.user) + response = create(request) + + assert response.status_code == status.HTTP_201_CREATED, response.data + assert response.data["api_endpoint"] == f"deployment/api/{ORG_ID}/my-api/" + + api = APIDeployment.objects.get(api_name="my-api") + assert api.is_active + assert api.organization_id == self.org.id + + # The key is returned once, at create time, and never again. + key = APIKey.objects.get(api=api) + assert str(key.api_key) == response.data["api_key"] + assert key.is_active + + list_keys = APIKeyViewSet.as_view({"get": "api_keys"}) + list_request = self.factory.get(f"/api/v1/api/keys/api/{api.id}/") + force_authenticate(list_request, user=self.user) + listed = list_keys(list_request, api_id=str(api.id)) + + assert listed.status_code == status.HTTP_200_OK, listed.data + assert [k["id"] for k in listed.data] == [str(key.id)] diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index d12374f44e..94071ac3b8 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -160,8 +160,6 @@ def get_required_setting(setting_key: str, default: str | None = None) -> str | FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005") PLATFORM_HOST = os.environ.get("PLATFORM_SERVICE_HOST", "http://localhost") PLATFORM_PORT = os.environ.get("PLATFORM_SERVICE_PORT", 3001) -PROMPT_HOST = os.environ.get("PROMPT_HOST", "http://localhost") -PROMPT_PORT = os.environ.get("PROMPT_PORT", 3003) PROMPT_STUDIO_FILE_PATH = os.environ.get( "PROMPT_STUDIO_FILE_PATH", "/app/prompt-studio-data" ) diff --git a/backend/conftest.py b/backend/conftest.py index 16452bb9ef..bbf1910425 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -22,3 +22,23 @@ # to make a mis-located file debuggable instead of silently empty. if not load_dotenv(Path(__file__).parent / "test.env", override=False): print("[conftest] backend/test.env not found; using ambient environment", flush=True) + + +def pytest_collection_modifyitems(items): + """Auto-mark every DB-bound test as ``integration`` so the rig's unit tier + (``-m 'not integration'``) skips it while the integration tier (live + Postgres) runs it. Detects Django ``TestCase``/``TransactionTestCase`` + subclasses and any item using the ``django_db`` marker — the two ways a + backend test needs a database. Kept central so tests declare their DB need + by how they're written, not by a hand-maintained marker on each file. + """ + import pytest + from django.test import TestCase, TransactionTestCase + + for item in items: + cls = getattr(item, "cls", None) + needs_db = item.get_closest_marker("django_db") is not None or ( + cls is not None and issubclass(cls, (TestCase, TransactionTestCase)) + ) + if needs_db: + item.add_marker(pytest.mark.integration) diff --git a/prompt-service/src/unstract/prompt_service/services/__init__.py b/backend/connector_v2/tests/__init__.py similarity index 100% rename from prompt-service/src/unstract/prompt_service/services/__init__.py rename to backend/connector_v2/tests/__init__.py diff --git a/backend/connector_v2/tests/conftest.py b/backend/connector_v2/tests/conftest.py deleted file mode 100644 index 89f1715a16..0000000000 --- a/backend/connector_v2/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -from django.core.management import call_command - - -@pytest.fixture(scope="session") -def django_db_setup(django_db_blocker): # type: ignore - fixtures = ["./connector/tests/fixtures/fixtures_0001.json"] - with django_db_blocker.unblock(): - call_command("loaddata", *fixtures) diff --git a/backend/connector_v2/tests/connector_tests.py b/backend/connector_v2/tests/connector_tests.py deleted file mode 100644 index 2f7960de6b..0000000000 --- a/backend/connector_v2/tests/connector_tests.py +++ /dev/null @@ -1,314 +0,0 @@ -# mypy: ignore-errors -import pytest -from connector_v2.models import ConnectorInstance -from django.urls import reverse -from rest_framework import status -from rest_framework.test import APITestCase - -pytestmark = pytest.mark.django_db - - -@pytest.mark.connector -class TestConnector(APITestCase): - def test_connector_list(self) -> None: - """Tests to List the connectors.""" - url = reverse("connectors_v1-list") - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_detail(self) -> None: - """Tests to fetch a connector with given pk.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_detail_not_found(self) -> None: - """Tests for negative case to fetch non exiting key.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 768}) - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - - def test_connectors_create(self) -> None: - """Tests to create a new ConnectorInstance.""" - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_with_json_list(self) -> None: - """Tests to create a new connector with list included in the json - field. - """ - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "file_name_list": ["a1", "a2"], - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_with_nested_json(self) -> None: - """Tests to create a new connector with json field as nested json.""" - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_bad_request(self) -> None: - """Tests for negative case to throw error on a wrong access.""" - url = reverse("connectors_v1-list") - data = { - "org": 5, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - - def test_connectors_update_json_field(self) -> None: - """Tests to update connector with json field update.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - drive_link = response.data["connector_metadata"]["drive_link"] - self.assertEqual(drive_link, "new_sample_url") - - def test_connectors_update(self) -> None: - """Tests to update connector update single field.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 1, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - modified_by = response.data["modified_by"] - self.assertEqual(modified_by, 2) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_update_pk(self) -> None: - """Tests the PUT method for 400 error.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 2, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - - def test_connectors_update_json_fields(self) -> None: - """Tests to update ConnectorInstance.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - nested_value = response.data["connector_metadata"]["sample_metadata_json"]["key1"] - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(nested_value, "value1") - - def test_connectors_update_json_list_fields(self) -> None: - """Tests to update connector to the third second level of json.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - "file_list": ["a1", "a2", "a3"], - }, - } - response = self.client.put(url, data, format="json") - nested_value = response.data["connector_metadata"]["sample_metadata_json"]["key1"] - nested_list = response.data["connector_metadata"]["file_list"] - last_val = nested_list.pop() - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(nested_value, "value1") - self.assertEqual(last_val, "a3") - - # @pytest.mark.xfail(raises=KeyError) - # def test_connectors_update_json_fields_failed(self) -> None: - # """Tests to update connector to the second level of JSON with a wrong - # key.""" - - # url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - # data = { - # "org": 1, - # "project": 1, - # "created_by": 2, - # "modified_by": 2, - # "modified_at": "2023-06-14T05:28:47.759Z", - # "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - # "connector_metadata": { - # "drive_link": "new_sample_url", - # "sharable_link": True, - # "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - # }, - # } - # response = self.client.put(url, data, format="json") - # nested_value = response.data["connector_metadata"]["sample_metadata_json"][ - # "key00" - # ] - - # @pytest.mark.xfail(raises=KeyError) - # def test_connectors_update_json_nested_failed(self) -> None: - # """Tests to update connector to test a first level of json with a wrong - # key.""" - - # url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - # data = { - # "org": 1, - # "project": 1, - # "created_by": 2, - # "modified_by": 2, - # "modified_at": "2023-06-14T05:28:47.759Z", - # "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - # "connector_metadata": { - # "drive_link": "new_sample_url", - # "sharable_link": True, - # "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - # }, - # } - # response = self.client.put(url, data, format="json") - # nested_value = response.data["connector_metadata"]["sample_metadata_jsonNew"] - - def test_connectors_update_field(self) -> None: - """Tests the PATCH method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = {"connector_id": "e3a4512m-efgb-48d5-98a9-3983ntest"} - response = self.client.patch(url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - connector_id = response.data["connector_id"] - - self.assertEqual( - connector_id, - ConnectorInstance.objects.get(connector_id=connector_id).connector_id, - ) - - def test_connectors_update_json_field_patch(self) -> None: - """Tests the PATCH method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "connector_metadata": { - "drive_link": "patch_update_url", - "sharable_link": True, - "sample_metadata_json": { - "key1": "patch_update1", - "key2": "value2", - }, - } - } - - response = self.client.patch(url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - drive_link = response.data["connector_metadata"]["drive_link"] - - self.assertEqual(drive_link, "patch_update_url") - - def test_connectors_delete(self) -> None: - """Tests the DELETE method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.delete(url, format="json") - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) - - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.get(url) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) diff --git a/backend/connector_v2/tests/fixtures/fixtures_0001.json b/backend/connector_v2/tests/fixtures/fixtures_0001.json deleted file mode 100644 index 55b39e6d86..0000000000 --- a/backend/connector_v2/tests/fixtures/fixtures_0001.json +++ /dev/null @@ -1,67 +0,0 @@ -[ - { - "model": "account.org", - "pk": 1, - "fields": { - "org_name": "Zipstack", - "created_by": 1, - "modified_by": 1, - "modified_at": "2023-06-14T05:28:47.739Z" - } - }, - { - "model": "account.user", - "pk": 1, - "fields": { - "org": 1, - "email": "johndoe@gmail.com", - "first_name": "John", - "last_name": "Doe", - "is_admin": true, - "created_by": null, - "modified_by": null, - "modified_at": "2023-06-14T05:28:47.744Z" - } - }, - { - "model": "account.user", - "pk": 2, - "fields": { - "org": 1, - "email": "user1@gmail.com", - "first_name": "Ron", - "last_name": "Stone", - "is_admin": false, - "created_by": 1, - "modified_by": 1, - "modified_at": "2023-06-14T05:28:47.750Z" - } - }, - { - "model": "project.project", - "pk": 1, - "fields": { - "org": 1, - "project_name": "Unstract Test", - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z" - } - }, - { - "model": "connector.connector", - "pk": 1, - "fields": { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e38a59b7-efbb-48d5-9da6-3a0cf2d882a0", - "connector_metadata": { - "connector_type": "gdrive", - "auth_type": "oauth" - } - } - } -] diff --git a/backend/connector_v2/tests/test_connector_register.py b/backend/connector_v2/tests/test_connector_register.py new file mode 100644 index 0000000000..889a961eca --- /dev/null +++ b/backend/connector_v2/tests/test_connector_register.py @@ -0,0 +1,123 @@ +"""Critical path ``connector-register-test``: credentials are validated against +the live external system, and a registered connector persists them encrypted. + +``POST /test_connectors/`` is the only place the platform proves it can actually +reach a customer's storage before a workflow depends on it; a version that +always answers "valid" would surface as a run-time failure deep inside an +execution. Runs against the rig's MinIO. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import os +import secrets + +import pytest +from account_v2.models import Organization, User +from django.db import connection +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from connector_v2.models import ConnectorInstance + +MINIO_CONNECTOR_ID = "minio|c799f6e3-2b57-434e-aaac-b5daa415da19" + +pytestmark = pytest.mark.skipif( + not all( + os.environ.get(var) + for var in ( + "MINIO_ACCESS_KEY_ID", + "MINIO_SECRET_ACCESS_KEY", + "MINIO_ENDPOINT_URL", + ) + ), + reason="needs a live MinIO (provisioned by the rig for this group)", +) + + +def _credentials(secret: str | None = None) -> dict: + return { + "connectorName": "rig-minio", + "key": os.environ["MINIO_ACCESS_KEY_ID"], + "secret": secret or os.environ["MINIO_SECRET_ACCESS_KEY"], + "endpoint_url": os.environ["MINIO_ENDPOINT_URL"], + "region_name": "", + "path": "/", + } + + +class ConnectorRegisterTest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-conn", display_name="Org Conn", organization_id="org-conn" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="connector@example.com", + email="connector@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.factory = APIRequestFactory() + + def _test_connector(self, credentials: dict): + from connector_processor.views import ConnectorViewSet + + view = ConnectorViewSet.as_view({"post": "test"}) + request = self.factory.post( + "/api/v1/test_connectors/", + {"connector_id": MINIO_CONNECTOR_ID, "connector_metadata": credentials}, + format="json", + ) + force_authenticate(request, user=self.user) + return view(request) + + @pytest.mark.critical_path("connector-register-test") + def test_good_credentials_validate_bad_ones_do_not(self) -> None: + valid = self._test_connector(_credentials()) + assert valid.status_code == status.HTTP_200_OK, valid.data + assert valid.data["is_valid"] is True + + invalid = self._test_connector(_credentials(secret="wrong-secret")) + assert invalid.status_code >= status.HTTP_400_BAD_REQUEST, invalid.data + assert "is_valid" not in invalid.data + + @pytest.mark.critical_path("connector-register-test") + def test_register_persists_credentials_encrypted(self) -> None: + from connector_v2.views import ConnectorInstanceViewSet + + view = ConnectorInstanceViewSet.as_view({"post": "create"}) + request = self.factory.post( + "/api/v1/connector/", + { + "connector_id": MINIO_CONNECTOR_ID, + "connector_name": "rig-minio", + "connector_metadata": _credentials(), + }, + format="json", + ) + force_authenticate(request, user=self.user) + response = view(request) + + assert response.status_code == status.HTTP_201_CREATED, response.data + instance = ConnectorInstance.objects.get(id=response.data["id"]) + assert instance.organization_id == self.org.id + # Decrypts back to the input on read... + assert instance.connector_metadata["key"] == os.environ["MINIO_ACCESS_KEY_ID"] + + # ...but the raw column must be ciphertext. Read it past the field's + # decrypting descriptor, else a regression to plaintext still passes. + with connection.cursor() as cursor: + cursor.execute( + "SELECT connector_metadata FROM connector_instance WHERE id = %s", + [str(instance.id)], + ) + raw = bytes(cursor.fetchone()[0]) + secret = os.environ["MINIO_SECRET_ACCESS_KEY"].encode() + assert secret not in raw + assert os.environ["MINIO_ACCESS_KEY_ID"].encode() not in raw diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py index f6803b047d..e81da3c3b2 100644 --- a/backend/connector_v2/views.py +++ b/backend/connector_v2/views.py @@ -12,10 +12,13 @@ from permissions.resource_share_views import ResourceShareManagementMixin from plugins import get_plugin from rest_framework import status, viewsets +from rest_framework.exceptions import PermissionDenied from rest_framework.request import Request from rest_framework.response import Response from rest_framework.versioning import URLPathVersioning +from tenant_account_v2.organization_member_service import OrganizationMemberService from utils.filtering import FilterHelper +from utils.user_context import UserContext from backend.constants import RequestKey from connector_v2.constants import ConnectorInstanceKey as CIKey @@ -44,6 +47,27 @@ def get_permissions(self) -> list[Any]: return [IsOwnerOrSharedUserOrSharedToOrg()] + @staticmethod + def _enforce_connector_creation_restriction(request: Any) -> None: + """Controlled mode (UN-3585): only org admins may create connectors + when the org has enabled the restriction. Service accounts (platform + API-key sessions) bypass; the default (flag off) keeps creation open + for everyone. Applies to all connector types (all connect to external + systems). + """ + if getattr(request.user, "is_service_account", False): + return + organization = UserContext.get_organization() + if ( + organization + and organization.restrict_connector_creation + and not OrganizationMemberService.is_user_organization_admin(request.user) + ): + raise PermissionDenied( + "Connector creation is restricted to organization admins. " + "Please contact your organization admin." + ) + def get_queryset(self) -> QuerySet | None: queryset = ConnectorInstance.objects.for_user(self.request.user) @@ -161,11 +185,18 @@ def perform_create(self, serializer: ConnectorInstanceSerializer) -> None: except Exception as exc: logger.error(f"Error while obtaining ConnectorAuth: {exc}") raise OAuthTimeOut + # Explicitly bind the connector to the request-scoped organization. + # Defense-in-depth: DefaultOrganizationMixin already declares + # `organization` as editable=False (so DRF drops any client-supplied + # value) and its save() backfills the org from UserContext when unset. + # Binding here makes that explicit at the callsite and keeps the row's + # org consistent with the org the controlled-mode check evaluated. serializer.save( connector_id=connector_id, connector_metadata=connector_metadata, created_by=self.request.user, modified_by=self.request.user, + organization=UserContext.get_organization(), ) # type: ignore # Clean up OAuth cache after successful create @@ -173,6 +204,10 @@ def perform_create(self, serializer: ConnectorInstanceSerializer) -> None: def create(self, request: Any) -> Response: # Overriding default exception behavior + # Fail fast on the admin restriction before validating the payload — + # the check depends only on the request/org, not on validated data, so + # a denied caller shouldn't get input-validation feedback first. + self._enforce_connector_creation_restriction(request) serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) try: diff --git a/backend/dashboard_metrics/tests/test_tasks.py b/backend/dashboard_metrics/tests/test_tasks.py index 1e38e2ca89..ac45887d4c 100644 --- a/backend/dashboard_metrics/tests/test_tasks.py +++ b/backend/dashboard_metrics/tests/test_tasks.py @@ -1,11 +1,11 @@ """Unit tests for Dashboard Metrics Celery tasks.""" -import uuid from datetime import datetime, timedelta from django.test import TestCase, TransactionTestCase from django.utils import timezone +from account_v2.models import Organization from dashboard_metrics.models import ( EventMetricsDaily, EventMetricsHourly, @@ -86,7 +86,10 @@ class TestCleanupTasks(TransactionTestCase): def setUp(self): """Set up test fixtures.""" - self.org_id = str(uuid.uuid4()) + # organization FK targets Organization's int PK, not a UUID. + self.org = Organization.objects.create( + organization_id="test-org", name="test-org", display_name="Test Org" + ) def test_cleanup_hourly_metrics_deletes_old_records(self): """Test that cleanup deletes hourly records older than retention.""" @@ -96,7 +99,7 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): # Create old record EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=old_timestamp, metric_name="old_metric", metric_type=MetricType.COUNTER, @@ -107,7 +110,7 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): # Create recent record EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=recent_timestamp, metric_name="recent_metric", metric_type=MetricType.COUNTER, @@ -122,9 +125,10 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): assert result["deleted"] == 1 assert result["retention_days"] == 30 - # Verify old is deleted, recent remains - assert not EventMetricsHourly.objects.filter(metric_name="old_metric").exists() - assert EventMetricsHourly.objects.filter(metric_name="recent_metric").exists() + # _base_manager bypasses the org-scoped default manager, which filters + # by UserContext.get_organization() — None here, so .objects sees nothing. + assert not EventMetricsHourly._base_manager.filter(metric_name="old_metric").exists() + assert EventMetricsHourly._base_manager.filter(metric_name="recent_metric").exists() def test_cleanup_daily_metrics_deletes_old_records(self): """Test that cleanup deletes daily records older than retention.""" @@ -134,7 +138,7 @@ def test_cleanup_daily_metrics_deletes_old_records(self): # Create old record EventMetricsDaily.objects.create( - organization_id=self.org_id, + organization=self.org, date=old_date, metric_name="old_daily_metric", metric_type=MetricType.COUNTER, @@ -145,7 +149,7 @@ def test_cleanup_daily_metrics_deletes_old_records(self): # Create recent record EventMetricsDaily.objects.create( - organization_id=self.org_id, + organization=self.org, date=recent_date, metric_name="recent_daily_metric", metric_type=MetricType.COUNTER, @@ -160,10 +164,10 @@ def test_cleanup_daily_metrics_deletes_old_records(self): assert result["deleted"] == 1 # Verify old is deleted, recent remains - assert not EventMetricsDaily.objects.filter( + assert not EventMetricsDaily._base_manager.filter( metric_name="old_daily_metric" ).exists() - assert EventMetricsDaily.objects.filter( + assert EventMetricsDaily._base_manager.filter( metric_name="recent_daily_metric" ).exists() @@ -173,7 +177,7 @@ def test_cleanup_hourly_with_custom_retention(self): old_timestamp = now - timedelta(days=10) EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=old_timestamp, metric_name="custom_retention_metric", metric_type=MetricType.COUNTER, diff --git a/backend/platform_api/permissions.py b/backend/platform_api/permissions.py index 12f23e63d5..32d6de2f33 100644 --- a/backend/platform_api/permissions.py +++ b/backend/platform_api/permissions.py @@ -3,6 +3,8 @@ from account_v2.authentication_controller import AuthenticationController from rest_framework.permissions import BasePermission +from platform_api.models import ApiKeyPermission + logger = logging.getLogger(__name__) @@ -21,3 +23,43 @@ def has_permission(self, request, view): except Exception: logger.exception("Error checking admin role for user %s", request.user.id) return False + + +class CanRotatePlatformApiKey(BasePermission): + """Permission for the ``rotate`` action (UN-3586). + + - **Session callers** must be organization admins (an admin may rotate any + key in their org). + - **Platform API key callers** must present a ``full_access`` key — the + API/automation path that the admin-only ``IsOrganizationAdmin`` gate + otherwise blocks (it rejects service accounts). ``full_access`` is the + admin-equivalent tier (it already permits DELETE), so key management + belongs there. + + Why ``full_access`` only (not any bearer key): ``rotate`` returns the new + key value in its response, so allowing a lower-tier ``read_write`` key to + rotate a ``full_access`` key would let it read that key's new secret and + escalate its privileges. Requiring ``full_access`` closes that path — a + ``full_access`` caller rotating any key gains no privilege (it is already + the top tier), matching what a session admin can do. + + Rotation stays confined to the caller's own organization via the auth + middleware (URL org must match the key's org) and the org-scoped queryset. + """ + + message = ( + "Rotating platform API keys requires an organization admin session or " + "a full_access platform API key." + ) + + def has_permission(self, request, view): + if not request.user or not request.user.is_authenticated: + return False + platform_key = getattr(request, "platform_api_key", None) + if platform_key is not None: + # Key-based caller: only a full_access key may rotate. Prevents a + # read_write key from rotating a full_access key and reading its new + # secret from the response (privilege escalation) — see docstring. + return platform_key.permission == ApiKeyPermission.FULL_ACCESS + # Session caller: must be an org admin (may rotate any key in the org). + return IsOrganizationAdmin().has_permission(request, view) diff --git a/backend/platform_api/views.py b/backend/platform_api/views.py index 2abf49812e..6886e51ddf 100644 --- a/backend/platform_api/views.py +++ b/backend/platform_api/views.py @@ -7,7 +7,7 @@ from utils.user_context import UserContext from platform_api.models import PlatformApiKey -from platform_api.permissions import IsOrganizationAdmin +from platform_api.permissions import CanRotatePlatformApiKey, IsOrganizationAdmin from platform_api.serializers import ( PlatformApiKeyCreateSerializer, PlatformApiKeyDetailSerializer, @@ -20,6 +20,13 @@ class PlatformApiKeyViewSet(viewsets.ModelViewSet): permission_classes = [IsAuthenticated, IsOrganizationAdmin] + def get_permissions(self): + # Rotate allows self-service rotation by a platform API key (UN-3586); + # all other key-management actions stay admin/session-only. + if self.action == "rotate": + return [IsAuthenticated(), CanRotatePlatformApiKey()] + return super().get_permissions() + def get_queryset(self): return PlatformApiKey.objects.filter( organization=UserContext.get_organization(), diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py index adb5713243..d82156b140 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py +++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py @@ -13,293 +13,21 @@ 4. On an error inside ``check_extraction_status``, swallow the error and fall back to full extraction — the dispatch must not fail. -The backend test environment has no ``pytest-django``, no SQLite -fallback, and the helper has a heavy Django-coupled import surface. -Rather than spin up Django, we stub every collaborator as a -``MagicMock`` on ``sys.modules`` *before* importing the helper, and -then patch ``PromptStudioHelper`` class methods per-test. This mirrors -the ``usage_v2/tests/test_helper.py`` approach. - -If the helper module cannot be imported in a given environment (for -example because the stub surface has drifted), all tests in the module -are skipped with a clear reason. +Unit tests: the real helper module is imported (Django is loaded by the +rig's test env) and every collaborator is patched on it per-test, so no +database is touched. """ from __future__ import annotations -import sys -import types +from contextlib import ExitStack from typing import Any from unittest.mock import MagicMock, patch -import pytest - - -# --------------------------------------------------------------------------- -# Stub every collaborator module on sys.modules before importing the helper. -# These stubs are intentionally broad MagicMocks — the tests patch the -# specific attributes they care about via ``unittest.mock.patch``. -# --------------------------------------------------------------------------- - - -def _install(name: str, attrs: dict[str, Any] | None = None) -> types.ModuleType: - """Install (or replace) a fake module into ``sys.modules``. - - Always creates a fresh ``ModuleType``; this is important because the - real module may already have been imported before these stubs run - (via pytest collection, conftest, etc.), and we need our fake to - actually take effect. - """ - mod = types.ModuleType(name) - if attrs: - for key, value in attrs.items(): - setattr(mod, key, value) - sys.modules[name] = mod - return mod - - -def _install_package(name: str) -> types.ModuleType: - """Install a fake package (marked with ``__path__``). - - Only stubs the package if it is not already in ``sys.modules``. - This prevents clobbering packages like ``unstract.core`` that must - retain their real ``__path__`` for submodule resolution. The child - modules we care about are always replaced explicitly via - ``_install``. - """ - if name in sys.modules: - return sys.modules[name] - mod = types.ModuleType(name) - mod.__path__ = [] # type: ignore[attr-defined] - sys.modules[name] = mod - return mod - - -try: - # Account / adapter stubs - _install_package("account_v2") - _install( - "account_v2.constants", - {"Common": type("Common", (), {"LOG_EVENTS_ID": "log_events_id", - "REQUEST_ID": "request_id"})}, - ) - _install("account_v2.models", {"User": MagicMock(name="User")}) - _install_package("adapter_processor_v2") - _install( - "adapter_processor_v2.constants", - {"AdapterKeys": type("AdapterKeys", (), {})}, - ) - _install( - "adapter_processor_v2.models", - {"AdapterInstance": MagicMock(name="AdapterInstance")}, - ) - - # Plugins stub - _install("plugins", {"get_plugin": MagicMock(return_value=None)}) - - # utils stubs - _install_package("utils") - _install_package("utils.file_storage") - _install( - "utils.file_storage.constants", - { - "FileStorageKeys": type( - "FileStorageKeys", - (), - {"PERMANENT_REMOTE_STORAGE": "permanent"}, - ) - }, - ) - _install_package("utils.file_storage.helpers") - _install( - "utils.file_storage.helpers.prompt_studio_file_helper", - {"PromptStudioFileHelper": MagicMock(name="PromptStudioFileHelper")}, - ) - _install( - "utils.local_context", - {"StateStore": MagicMock(name="StateStore")}, - ) - - # backend.celery_service stub - _install_package("backend") - _install( - "backend.celery_service", - {"app": MagicMock(name="celery_app")}, - ) - - # prompt_studio stubs - _install_package("prompt_studio") - _install_package("prompt_studio.prompt_profile_manager_v2") - _install( - "prompt_studio.prompt_profile_manager_v2.models", - {"ProfileManager": MagicMock(name="ProfileManager")}, - ) - _install( - "prompt_studio.prompt_profile_manager_v2.profile_manager_helper", - {"ProfileManagerHelper": MagicMock(name="ProfileManagerHelper")}, - ) - - _install_package("prompt_studio.prompt_studio_document_manager_v2") - _install( - "prompt_studio.prompt_studio_document_manager_v2.models", - {"DocumentManager": MagicMock(name="DocumentManager")}, - ) - - _install_package("prompt_studio.prompt_studio_index_manager_v2") - _install( - "prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper", - {"PromptStudioIndexHelper": MagicMock(name="PromptStudioIndexHelper")}, - ) +from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as _psh_mod - _install_package("prompt_studio.prompt_studio_output_manager_v2") - _install( - "prompt_studio.prompt_studio_output_manager_v2.output_manager_helper", - {"OutputManagerHelper": MagicMock(name="OutputManagerHelper")}, - ) - - _install_package("prompt_studio.prompt_studio_v2") - _install( - "prompt_studio.prompt_studio_v2.models", - {"ToolStudioPrompt": MagicMock(name="ToolStudioPrompt")}, - ) - - # Stub the prompt_studio_core_v2 sibling modules too — several of them - # transitively import modules (like ``utils.cache_service``) that we - # don't want to pull in for these unit tests. - _install_package("prompt_studio.prompt_studio_core_v2") - _install( - "prompt_studio.prompt_studio_core_v2.document_indexing_service", - {"DocumentIndexingService": MagicMock(name="DocumentIndexingService")}, - ) - - # Real exception classes — build_index_payload uses ``raise``. - class _FakeExc(Exception): - pass - - _install( - "prompt_studio.prompt_studio_core_v2.exceptions", - { - "AnswerFetchError": type("AnswerFetchError", (_FakeExc,), {}), - "DefaultProfileError": type("DefaultProfileError", (_FakeExc,), {}), - "EmptyPromptError": type("EmptyPromptError", (_FakeExc,), {}), - "ExtractionAPIError": type("ExtractionAPIError", (_FakeExc,), {}), - "IndexingAPIError": type("IndexingAPIError", (_FakeExc,), {}), - "NoPromptsFound": type("NoPromptsFound", (_FakeExc,), {}), - "OperationNotSupported": type("OperationNotSupported", (_FakeExc,), {}), - "PermissionError": type("PermissionError", (_FakeExc,), {}), - }, - ) - _install( - "prompt_studio.prompt_studio_core_v2.migration_utils", - {"SummarizeMigrationUtils": MagicMock(name="SummarizeMigrationUtils")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.models", - {"CustomTool": MagicMock(name="CustomTool")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool", - {"PromptIdeBaseTool": MagicMock(name="PromptIdeBaseTool")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.prompt_variable_service", - {"PromptStudioVariableService": MagicMock(name="PromptStudioVariableService")}, - ) - - # unstract.core.pubsub_helper stub (LogPublisher isn't used by - # build_index_payload but the module-level import must succeed). - _install_package("unstract.core") - _install( - "unstract.core.pubsub_helper", - {"LogPublisher": MagicMock(name="LogPublisher")}, - ) - - # unstract.sdk1 stubs — these heavy modules transitively pull in - # ``unstract.core.cache.redis_client`` which isn't on the python - # path for the backend tests. We only need the leaf classes. - _install_package("unstract.sdk1") - _install( - "unstract.sdk1.constants", - { - "LogLevel": type( - "LogLevel", (), {"INFO": "INFO", "WARN": "WARN", "ERROR": "ERROR"} - ) - }, - ) - _install( - "unstract.sdk1.exceptions", - { - "IndexingError": type("IndexingError", (Exception,), {}), - "SdkError": type("SdkError", (Exception,), {}), - }, - ) - _install_package("unstract.sdk1.execution") - - class _FakeExecutionContext: - """Minimal ExecutionContext that keeps ``executor_params`` as - the real dict we pass in (the tests inspect it).""" - - def __init__(self, **kwargs: Any) -> None: - self.executor_name = kwargs.get("executor_name") - self.operation = kwargs.get("operation") - self.run_id = kwargs.get("run_id") - self.execution_source = kwargs.get("execution_source") - self.organization_id = kwargs.get("organization_id") - self.executor_params = kwargs.get("executor_params") or {} - self.request_id = kwargs.get("request_id") - self.log_events_id = kwargs.get("log_events_id") - - _install( - "unstract.sdk1.execution.context", - {"ExecutionContext": _FakeExecutionContext}, - ) - _install( - "unstract.sdk1.execution.dispatcher", - {"ExecutionDispatcher": MagicMock(name="ExecutionDispatcher")}, - ) - _install_package("unstract.sdk1.file_storage") - _install( - "unstract.sdk1.file_storage.constants", - {"StorageType": type("StorageType", (), {"PERMANENT": "permanent"})}, - ) - _install( - "unstract.sdk1.file_storage.env_helper", - {"EnvHelper": MagicMock(name="EnvHelper")}, - ) - _install_package("unstract.sdk1.utils") - _install( - "unstract.sdk1.utils.indexing", - {"IndexingUtils": MagicMock(name="IndexingUtils")}, - ) - _install( - "unstract.sdk1.utils.tool", - {"ToolUtils": MagicMock(name="ToolUtils")}, - ) - - # Now import the helper module. If this fails, all tests below will - # be skipped via the ``_IMPORT_ERROR`` sentinel. - from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as _psh_mod # noqa: E402 - - PromptStudioHelper = _psh_mod.PromptStudioHelper - IKeys = _psh_mod.IKeys - _IMPORT_ERROR: str | None = None -except Exception as exc: # pragma: no cover — environment guard - _IMPORT_ERROR = ( - f"prompt_studio_helper could not be imported in this environment: " - f"{type(exc).__name__}: {exc}" - ) - PromptStudioHelper = None # type: ignore[assignment] - IKeys = None # type: ignore[assignment] - - -pytestmark = pytest.mark.skipif( - _IMPORT_ERROR is not None, reason=_IMPORT_ERROR or "" -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- +PromptStudioHelper = _psh_mod.PromptStudioHelper +IKeys = _psh_mod.IKeys def _make_tool(enable_highlight: bool = False, summarize_context: bool = False): @@ -354,72 +82,35 @@ def _dispatch_build( else: check_mock.return_value = check_return - # Patch everything via context managers so each test starts clean. - patches = [ - patch.object( - _psh_mod.CustomTool, - "objects", - MagicMock(get=MagicMock(return_value=tool)), - ), - patch.object( - _psh_mod.PromptStudioFileHelper, - "get_or_create_prompt_studio_subdirectory", - return_value="/prompt-studio/org/user/tool", - ), - patch.object( - _psh_mod.ProfileManager, - "get_default_llm_profile", - return_value=profile, - ), - patch.object( - PromptStudioHelper, - "validate_adapter_status", - return_value=None, - ), - patch.object( - PromptStudioHelper, - "validate_profile_manager_owner_access", - return_value=None, - ), - patch.object( - PromptStudioHelper, - "_get_platform_api_key", - return_value="pk-test", - ), - patch.object( - PromptStudioHelper, - "_build_summarize_params", - return_value=(None, "", MagicMock()), - ), - patch.object( - _psh_mod.EnvHelper, - "get_storage", - return_value=fs_instance, - ), - patch.object( - _psh_mod.PromptStudioIndexHelper, - "check_extraction_status", - check_mock, - ), - patch.object( - _psh_mod.IndexingUtils, - "generate_index_key", - return_value="doc-key-1", - ), - patch.object( - _psh_mod, - "PromptIdeBaseTool", - MagicMock(return_value=MagicMock()), - ), - patch.object( - _psh_mod.StateStore, - "get", - return_value="", - ), - ] - for p in patches: - p.start() - try: + with ExitStack() as stack: + for target, attr, value in ( + (_psh_mod.CustomTool, "objects", MagicMock(get=MagicMock(return_value=tool))), + ( + _psh_mod.PromptStudioFileHelper, + "get_or_create_prompt_studio_subdirectory", + MagicMock(return_value="/prompt-studio/org/user/tool"), + ), + (_psh_mod.ProfileManager, "get_default_llm_profile", MagicMock(return_value=profile)), + (PromptStudioHelper, "validate_adapter_status", MagicMock(return_value=None)), + ( + PromptStudioHelper, + "validate_profile_manager_owner_access", + MagicMock(return_value=None), + ), + (PromptStudioHelper, "_get_platform_api_key", MagicMock(return_value="pk-test")), + ( + PromptStudioHelper, + "_build_summarize_params", + MagicMock(return_value=(None, "", MagicMock())), + ), + (_psh_mod.EnvHelper, "get_storage", MagicMock(return_value=fs_instance)), + (_psh_mod.PromptStudioIndexHelper, "check_extraction_status", check_mock), + (_psh_mod.IndexingUtils, "generate_index_key", MagicMock(return_value="doc-key-1")), + (_psh_mod, "PromptIdeBaseTool", MagicMock(return_value=MagicMock())), + (_psh_mod.StateStore, "get", MagicMock(return_value="")), + ): + stack.enter_context(patch.object(target, attr, value)) + context, cb_kwargs = PromptStudioHelper.build_index_payload( tool_id="tool-1", file_name="doc.pdf", @@ -429,14 +120,6 @@ def _dispatch_build( run_id="run-1", ) return context, cb_kwargs, fs_instance, check_mock - finally: - for p in patches: - p.stop() - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- class TestBuildIndexPayloadMarker: diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_prompt_studio_author.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_prompt_studio_author.py new file mode 100644 index 0000000000..1b5946ab9c --- /dev/null +++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_prompt_studio_author.py @@ -0,0 +1,85 @@ +"""Critical path ``prompt-studio-author``: create a Prompt Studio project and +add a prompt to it. + +Authoring is the synchronous half of prompt-studio-fetch-response — running a +prompt needs a real LLM, but composing one does not. A project must be creatable +before any adapter is configured (a fresh org has none), and prompts must bind +to their project. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import secrets + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from prompt_studio.prompt_studio_core_v2.models import CustomTool +from prompt_studio.prompt_studio_core_v2.views import PromptStudioCoreView +from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt + + +class PromptStudioAuthorAPITest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-ps", display_name="Org PS", organization_id="org-ps" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="prompter@example.com", + email="prompter@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.factory = APIRequestFactory() + + def _call(self, view, method: str, path: str, payload: dict, **kwargs): + request = getattr(self.factory, method)(path, payload, format="json") + force_authenticate(request, user=self.user) + return view(request, **kwargs) + + @pytest.mark.critical_path("prompt-studio-author") + def test_create_project_then_add_prompt(self) -> None: + create_project = PromptStudioCoreView.as_view({"post": "create"}) + project = self._call( + create_project, + "post", + "/api/v1/prompt-studio/", + { + "tool_name": "invoice-parser", + "description": "extracts invoice fields", + "author": "test", + }, + ) + assert project.status_code == status.HTTP_201_CREATED, project.data + + tool = CustomTool.objects.get(tool_id=project.data["tool_id"]) + assert tool.organization_id == self.org.id + assert tool.created_by == self.user + + create_prompt = PromptStudioCoreView.as_view({"post": "create_prompt"}) + prompt = self._call( + create_prompt, + "post", + f"/api/v1/prompt-studio/prompt-studio-prompt/{tool.tool_id}/", + { + "tool_id": str(tool.tool_id), + "prompt_key": "invoice_number", + "prompt": "What is the invoice number?", + "prompt_type": ToolStudioPrompt.PromptType.PROMPT, + "sequence_number": 1, + }, + pk=str(tool.tool_id), + ) + assert prompt.status_code == status.HTTP_201_CREATED, prompt.data + + persisted = ToolStudioPrompt.objects.get(prompt_id=prompt.data["prompt_id"]) + assert persisted.tool_id_id == tool.tool_id + assert persisted.active diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3ef2c187a4..46f0821131 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -71,7 +71,10 @@ dev = [ "responses>=0.25.7", "psutil>=7.0.0", ] -test = ["pytest>=8.0.1"] +test = [ + "pytest>=8.0.1", + "pytest-django>=4.12.0", +] deploy = [ "gunicorn~=23.0", # For serving the application # Keep versions empty and let uv decide version @@ -101,6 +104,11 @@ constraint-dependencies = [ # Note: test.env is loaded by backend/conftest.py via python-dotenv directly # (replaces the unmaintained pytest-dotenv plugin). addopts = "-s" +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] +markers = [ + "integration: needs live infra (Postgres/Redis); runs in the rig's integration tier, not unit (select with -m integration / exclude with -m 'not integration')", + "critical_path(path_id): attests coverage of declared critical path(s) from tests/critical_paths.yaml when this test passes", +] [tool.poe] envfile = ".env" diff --git a/backend/sample.env b/backend/sample.env index 743a31de8e..e04445822d 100644 --- a/backend/sample.env +++ b/backend/sample.env @@ -92,10 +92,6 @@ UNSTRACT_RUNNER_API_TIMEOUT=240 # (in seconds) 2 mins UNSTRACT_RUNNER_API_RETRY_COUNT=5 # Number of retries for failed requests UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3 # Exponential backoff factor for retries -# Prompt Service -PROMPT_HOST=http://unstract-prompt-service -PROMPT_PORT=3003 - #Prompt Studio PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data diff --git a/backend/tenant_account_v2/urls.py b/backend/tenant_account_v2/urls.py index 91831b8c11..4b7aec849b 100644 --- a/backend/tenant_account_v2/urls.py +++ b/backend/tenant_account_v2/urls.py @@ -1,13 +1,23 @@ from django.urls import include, path from tenant_account_v2 import groups_urls, invitation_urls, users_urls -from tenant_account_v2.views import get_organization, get_roles, reset_password +from tenant_account_v2.views import ( + get_organization, + get_roles, + organization_settings, + reset_password, +) urlpatterns = [ path("roles", get_roles, name="roles"), path("users/", include(users_urls)), path("invitation/", include(invitation_urls)), path("organization", get_organization, name="get_organization"), + path( + "organization/settings", + organization_settings, + name="organization_settings", + ), path("reset_password", reset_password, name="reset_password"), path("", include(groups_urls)), ] diff --git a/backend/tenant_account_v2/views.py b/backend/tenant_account_v2/views.py index 838e6adff7..0e824bd7e1 100644 --- a/backend/tenant_account_v2/views.py +++ b/backend/tenant_account_v2/views.py @@ -4,10 +4,13 @@ from account_v2.authentication_controller import AuthenticationController from account_v2.dto import UserRoleData from account_v2.models import Organization +from platform_api.permissions import IsOrganizationAdmin from rest_framework import status -from rest_framework.decorators import api_view +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response +from utils.user_context import UserContext from utils.user_session import UserSessionUtils from tenant_account_v2.dto import OrganizationLoginResponse, ResetUserPasswordDto @@ -52,6 +55,65 @@ def reset_password(request: Request) -> Response: ) +# Boolean org-level controlled-mode flags exposed by ``organization_settings``. +# Each maps a request/response key to the ``Organization`` field it updates. +ORGANIZATION_SETTING_FLAGS = ( + "restrict_llm_adapter_creation", + "restrict_connector_creation", +) + + +@api_view(["GET", "PATCH"]) +@permission_classes([IsAuthenticated, IsOrganizationAdmin]) +def organization_settings(request: Request) -> Response: + """Read or update org-level settings. Admin-only. + + Exposes the controlled-mode flags in ``ORGANIZATION_SETTING_FLAGS`` + (``restrict_llm_adapter_creation``, ``restrict_connector_creation``). GET + returns the current values; PATCH updates any subset of them. + """ + organization = UserContext.get_organization() + if not organization: + return Response( + status=status.HTTP_404_NOT_FOUND, + data={"message": "Org Not Found"}, + ) + + if request.method == "PATCH": + provided = { + flag: request.data[flag] + for flag in ORGANIZATION_SETTING_FLAGS + if flag in request.data + } + if not provided: + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "message": ( + "Provide at least one of: " + f"{', '.join(ORGANIZATION_SETTING_FLAGS)}" + ) + }, + ) + # Validate everything before mutating so a bad value in a later flag + # doesn't leave the in-memory org partially updated. + for flag, value in provided.items(): + if not isinstance(value, bool): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={"message": f"{flag} must be a boolean"}, + ) + for flag, value in provided.items(): + setattr(organization, flag, value) + organization.modified_by = request.user + organization.save(update_fields=[*provided, "modified_by", "modified_at"]) + + return Response( + status=status.HTTP_200_OK, + data={flag: getattr(organization, flag) for flag in ORGANIZATION_SETTING_FLAGS}, + ) + + @api_view(["GET"]) def get_organization(request: Request) -> Response: auth_controller = AuthenticationController() diff --git a/backend/usage_v2/tests/test_helper.py b/backend/usage_v2/tests/test_helper.py index 4f6ffc9c40..f0311c22aa 100644 --- a/backend/usage_v2/tests/test_helper.py +++ b/backend/usage_v2/tests/test_helper.py @@ -5,69 +5,34 @@ bare ``"llm"`` bucket from leaking into API deployment responses when a producer-side LLM call site forgets to set ``llm_usage_reason``. -The tests deliberately do not require a live Django database — the -backend test environment has no ``pytest-django``, no SQLite fallback, -and uses ``django-tenants`` against Postgres in production. Instead -the tests stub ``account_usage.models`` and ``usage_v2.models`` in -``sys.modules`` *before* importing the helper, so the helper module -loads cleanly without triggering Django's app registry checks. The -fake ``Usage.objects.filter`` chain returns a deterministic list of -row dicts shaped exactly like the real ``.values(...).annotate(...)`` -queryset rows the helper iterates over. +The tests exercise only the helper's in-memory aggregation logic, not +the ORM. We rebind the ``Usage`` symbol the helper resolved at import +to a fake whose ``objects.filter`` chain returns a deterministic list +of row dicts shaped exactly like the real +``.values(...).annotate(...)`` queryset rows the helper iterates over. """ from __future__ import annotations -import sys -import types from typing import Any from unittest.mock import MagicMock +import pytest +import usage_v2.helper as helper_mod +from usage_v2.helper import UsageHelper -# --------------------------------------------------------------------------- -# Module-level stubs. Must run BEFORE ``usage_v2.helper`` is imported, so we -# do it at import time and capture the helper reference for the tests below. -# --------------------------------------------------------------------------- - - -def _install_stubs() -> tuple[Any, Any]: - """Install fake ``account_usage.models`` and ``usage_v2.models`` modules - so that ``usage_v2.helper`` can be imported without Django being set up. - - Returns ``(UsageHelper, FakeUsage)`` — the helper class to test and the - fake Usage class whose ``objects.filter`` we will swap per-test. - """ - # Fake account_usage package + models module - if "account_usage" not in sys.modules: - account_usage_pkg = types.ModuleType("account_usage") - account_usage_pkg.__path__ = [] # mark as package - sys.modules["account_usage"] = account_usage_pkg - if "account_usage.models" not in sys.modules: - account_usage_models = types.ModuleType("account_usage.models") - account_usage_models.PageUsage = MagicMock(name="PageUsage") - sys.modules["account_usage.models"] = account_usage_models - - # Fake usage_v2.models with a Usage class whose ``objects`` is a - # MagicMock (so each test can rebind ``filter.return_value``). - if "usage_v2.models" not in sys.modules or not hasattr( - sys.modules["usage_v2.models"], "_is_test_stub" - ): - usage_v2_models = types.ModuleType("usage_v2.models") - usage_v2_models._is_test_stub = True - - class _FakeUsage: - objects = MagicMock(name="Usage.objects") - - usage_v2_models.Usage = _FakeUsage - sys.modules["usage_v2.models"] = usage_v2_models - - # Now import the helper — this picks up our stubs. - from usage_v2.helper import UsageHelper - return UsageHelper, sys.modules["usage_v2.models"].Usage +class FakeUsage: + # objects is a MagicMock so each test can rebind filter.return_value. + objects = MagicMock(name="Usage.objects") -UsageHelper, FakeUsage = _install_stubs() +@pytest.fixture(autouse=True) +def _swap_usage(monkeypatch: pytest.MonkeyPatch) -> None: + # Swap the symbol get_usage_by_model resolves, per-test, so monkeypatch + # restores the real model afterwards — a module-level rebind would leak + # FakeUsage into every later test in the same process. + monkeypatch.setattr(helper_mod, "Usage", FakeUsage) # --------------------------------------------------------------------------- diff --git a/backend/usage_v2/tests/test_usage_aggregate.py b/backend/usage_v2/tests/test_usage_aggregate.py new file mode 100644 index 0000000000..6efd45dd5c --- /dev/null +++ b/backend/usage_v2/tests/test_usage_aggregate.py @@ -0,0 +1,91 @@ +"""Critical path ``usage-aggregate-read``: per-run token usage aggregates +correctly and never crosses organization boundaries. + +Usage rows are written by workers, but the aggregation that bills against them +is a synchronous read. Under-counting loses revenue; leaking another org's rows +into the sum is both a billing and a disclosure bug — the org scoping lives in a +default manager, so a stray ``objects`` -> ``_base_manager`` change would break +it silently. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import secrets +import uuid + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from usage_v2.models import Usage, UsageType + + +def _usage(organization: Organization, run_id: uuid.UUID, tokens: int) -> Usage: + return Usage.objects.create( + organization=organization, + run_id=run_id, + adapter_instance_id=str(uuid.uuid4()), + usage_type=UsageType.LLM, + model_name="gpt-4o-mini", + embedding_tokens=0, + prompt_tokens=tokens, + completion_tokens=tokens, + total_tokens=tokens * 2, + cost_in_dollars=0.5, + ) + + +class UsageAggregateReadTest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-usage", display_name="Org Usage", organization_id="org-usage" + ) + self.other_org = Organization.objects.create( + name="org-other", display_name="Org Other", organization_id="org-other" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="biller@example.com", + email="biller@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.run_id = uuid.uuid4() + self.factory = APIRequestFactory() + + def tearDown(self) -> None: + # UserContext is thread-local, not rolled back with the test DB; clear it + # so the org identifier can't leak into a later manager-filtered query. + UserContext.set_organization_identifier(None) + super().tearDown() + + @pytest.mark.critical_path("usage-aggregate-read") + def test_token_usage_sums_own_org_only(self) -> None: + _usage(self.org, self.run_id, tokens=100) + _usage(self.org, self.run_id, tokens=50) + _usage(self.org, uuid.uuid4(), tokens=999) # different run + _usage(self.other_org, self.run_id, tokens=777) # same run, other org + + # Deferred: UsageView's filterset resolves a queryset at class-body + # evaluation, so a module-scope import queries the DB during pytest + # collection — before any test has DB access. + from usage_v2.views import UsageView + + view = UsageView.as_view({"get": "get_token_usage"}) + request = self.factory.get( + "/api/v1/usage/get_token_usage/", {"run_id": self.run_id} + ) + force_authenticate(request, user=self.user) + response = view(request) + + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["prompt_tokens"] == 150 + assert response.data["completion_tokens"] == 150 + assert response.data["total_tokens"] == 300 + assert response.data["cost_in_dollars"] == pytest.approx(1.0) diff --git a/prompt-service/src/unstract/prompt_service/tests/unit/__init__.py b/backend/utils/file_storage/__init__.py similarity index 100% rename from prompt-service/src/unstract/prompt_service/tests/unit/__init__.py rename to backend/utils/file_storage/__init__.py diff --git a/prompt-service/src/unstract/prompt_service/utils/__init__.py b/backend/utils/file_storage/helpers/__init__.py similarity index 100% rename from prompt-service/src/unstract/prompt_service/utils/__init__.py rename to backend/utils/file_storage/helpers/__init__.py diff --git a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py index df09776651..c8b75612fe 100644 --- a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py +++ b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py @@ -6,13 +6,13 @@ from file_management.exceptions import InvalidFileType from file_management.file_management_helper import FileManagerHelper -from utils.file_storage.constants import FileStorageConstants, FileStorageKeys -from utils.file_storage.helpers.streaming_writer import write_streaming from unstract.core.utilities import UnstractUtils from unstract.sdk1.file_storage import FileStorage from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper +from utils.file_storage.constants import FileStorageConstants, FileStorageKeys +from utils.file_storage.helpers.streaming_writer import write_streaming logger = logging.getLogger(__name__) diff --git a/backend/uv.lock b/backend/uv.lock index c70acebb2a..a68e8aee7a 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1681,7 +1681,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1697,9 +1697,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -2970,6 +2970,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] +[[package]] +name = "pytest-django" +version = "4.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156, upload-time = "2026-02-14T18:40:49.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, +] + [[package]] name = "python-crontab" version = "3.3.0" @@ -3721,6 +3733,7 @@ dev = [ ] test = [ { name = "pytest" }, + { name = "pytest-django" }, ] [package.metadata] @@ -3784,7 +3797,10 @@ dev = [ { name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" }, { name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" }, ] -test = [{ name = "pytest", specifier = ">=8.0.1" }] +test = [ + { name = "pytest", specifier = ">=8.0.1" }, + { name = "pytest-django", specifier = ">=4.12.0" }, +] [[package]] name = "unstract-connectors" @@ -3937,7 +3953,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py index 33f31f055a..445affba1d 100644 --- a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py +++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py @@ -1,19 +1,18 @@ import os from unittest.mock import Mock, patch +import pytest from django.test import TestCase +from unstract.connectors.databases.postgresql import PostgreSQL from workflow_manager.endpoint_v2.constants import DestinationKey from workflow_manager.endpoint_v2.destination import DestinationConnector -from unstract.connectors.databases.postgresql import PostgreSQL - class TestDestinationConnectorPostgreSQL(TestCase): """Integration test for insert_into_db method with real PostgreSQL connector.""" def setUp(self) -> None: """Set up test data and real PostgreSQL configuration.""" - # Real PostgreSQL connection settings for testing self.postgres_config = { "host": os.getenv("DB_HOST", "localhost"), @@ -21,7 +20,7 @@ def setUp(self) -> None: "database": os.getenv("DB_NAME", "test_unstract"), "user": os.getenv("DB_USER", "postgres"), "password": os.getenv("DB_PASSWORD", "password"), - "schema": "test", # Add schema to fix PostgreSQL issue + "schema": os.getenv("DB_SCHEMA", "public"), } # Test data that will be inserted into the database @@ -32,7 +31,9 @@ def setUp(self) -> None: "processing_time": 1.5, } self.input_file_path = "/path/to/test/file.pdf" - self.test_table_name = "OUTPUT_3" + # Lowercase: the connector quotes the name on CREATE (case-preserved) but + # lowercases it when reading information_schema back. + self.test_table_name = "output_3" # Create real PostgreSQL connector instance self.postgres_connector = PostgreSQL(settings=self.postgres_config) @@ -192,22 +193,27 @@ def test_insert_into_db_happy_path_postgresql(self) -> None: f"✅ Successfully inserted test data into PostgreSQL table: {self.test_table_name}" ) + @pytest.mark.xfail( + reason=( + "get_sql_values_for_query serializes data=None as the literal " + "string 'None', which is invalid JSON for the jsonb data column " + "(PR #2115 review). Remove this marker once None maps to SQL NULL." + ), + strict=True, + ) def test_insert_into_db_with_error_postgresql(self) -> None: """Test insertion with error parameter into real PostgreSQL database.""" - # Create mock objects mock_workflow = self.create_mock_workflow() mock_workflow_log = self.create_mock_workflow_log() mock_connector_instance = self.create_real_connector_instance() mock_endpoint = self.create_mock_endpoint(mock_connector_instance) - # Create destination connector destination_connector = self.create_destination_connector( mock_workflow, mock_workflow_log, mock_endpoint ) error_message = "Test processing error occurred" - # Mock the methods that get data with patch.object( destination_connector, "get_tool_execution_result", @@ -218,7 +224,6 @@ def test_insert_into_db_with_error_postgresql(self) -> None: "get_combined_metadata", return_value=self.test_metadata, ): - # Execute with error parameter destination_connector.insert_into_db( input_file_path=self.input_file_path, error=error_message ) @@ -226,10 +231,6 @@ def test_insert_into_db_with_error_postgresql(self) -> None: # Verify that all expected columns were created self.verify_table_columns(self.test_table_name) - print( - f"✅ Successfully inserted error data into PostgreSQL table: {self.test_table_name}" - ) - def test_postgresql_connector_connection(self) -> None: """Test that the PostgreSQL connector can establish a connection.""" # Test the real PostgreSQL connector directly diff --git a/backend/workflow_manager/workflow_v2/tests/test_workflow_author.py b/backend/workflow_manager/workflow_v2/tests/test_workflow_author.py new file mode 100644 index 0000000000..bc6b5eac26 --- /dev/null +++ b/backend/workflow_manager/workflow_v2/tests/test_workflow_author.py @@ -0,0 +1,95 @@ +"""Critical path ``workflow-author``: create a workflow and configure its +source + destination endpoints. + +This is the authoring contract that every execution path builds on. Creating a +workflow implicitly materialises its two endpoints — a workflow whose endpoints +are missing is unconfigurable and unexecutable, and nothing else in the product +recreates them. Fully synchronous: no workers, no execution. Needs a live DB +(integration tier). +""" + +from __future__ import annotations + +import secrets + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils +from workflow_manager.endpoint_v2.models import WorkflowEndpoint +from workflow_manager.workflow_v2.models.workflow import Workflow +from workflow_manager.workflow_v2.views import WorkflowViewSet + + +class WorkflowAuthorAPITest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-wf", display_name="Org WF", organization_id="org-wf" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="author@example.com", + email="author@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.factory = APIRequestFactory() + + def _create_workflow(self, name: str): + view = WorkflowViewSet.as_view({"post": "create"}) + request = self.factory.post( + "/api/v1/workflow/", {"workflow_name": name}, format="json" + ) + force_authenticate(request, user=self.user) + return view(request) + + def _patch_endpoint(self, endpoint: WorkflowEndpoint, payload: dict): + # Deferred: WorkflowEndpointSerializer resolves a queryset at class-body + # evaluation, so importing its view at module scope queries the DB during + # pytest collection — before any test has DB access. + from workflow_manager.endpoint_v2.views import WorkflowEndpointViewSet + + view = WorkflowEndpointViewSet.as_view({"patch": "partial_update"}) + request = self.factory.patch( + f"/api/v1/workflow/endpoint/{endpoint.id}/", payload, format="json" + ) + force_authenticate(request, user=self.user) + return view(request, pk=str(endpoint.id)) + + @pytest.mark.critical_path("workflow-author") + def test_create_workflow_materialises_configurable_endpoints(self) -> None: + response = self._create_workflow("wf-author") + assert response.status_code == status.HTTP_201_CREATED, response.data + + workflow = Workflow.objects.get(id=response.data["id"]) + assert workflow.organization_id == self.org.id + assert workflow.is_active + + endpoints = {e.endpoint_type: e for e in workflow.workflowendpoint_set.all()} + assert set(endpoints) == { + WorkflowEndpoint.EndpointType.SOURCE, + WorkflowEndpoint.EndpointType.DESTINATION, + } + # Endpoints land unconfigured — authoring, not execution, fills them in. + assert all(not e.connection_type for e in endpoints.values()) + + source = self._patch_endpoint( + endpoints[WorkflowEndpoint.EndpointType.SOURCE], + {"connection_type": WorkflowEndpoint.ConnectionType.API}, + ) + assert source.status_code == status.HTTP_200_OK, source.data + + destination = self._patch_endpoint( + endpoints[WorkflowEndpoint.EndpointType.DESTINATION], + {"connection_type": WorkflowEndpoint.ConnectionType.API}, + ) + assert destination.status_code == status.HTTP_200_OK, destination.data + + assert WorkflowEndpointUtils.is_api_workflow(workflow) diff --git a/docker/compose.debug.yaml b/docker/compose.debug.yaml index 66d247997b..3522902b76 100644 --- a/docker/compose.debug.yaml +++ b/docker/compose.debug.yaml @@ -7,7 +7,6 @@ # 5678 - backend # 5679 - runner # 5680 - platform-service -# 5681 - prompt-service # 5682 - worker-file-processing-v2 # 5683 - worker-callback-v2 # 5684 - worker-api-deployment-v2 @@ -65,21 +64,6 @@ services: --graceful-timeout 5 unstract.platform_service.run:app" ] - prompt-service: - ports: - - "5681:5681" - command: [ - "uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5681 .venv/bin/gunicorn - --bind 0.0.0.0:3003 - --workers 1 - --threads 2 - --worker-class gthread - --log-level debug - --timeout 900 - --access-logfile - - --graceful-timeout 5 unstract.prompt_service.run:app" - ] - ######################################################################################################### # V2 Workers with debugpy # Using --pool=threads for debugpy compatibility (prefork doesn't work well with debugpy) diff --git a/docker/docker-compose.build.yaml b/docker/docker-compose.build.yaml index 3969b1f98f..e5ccdc38ed 100644 --- a/docker/docker-compose.build.yaml +++ b/docker/docker-compose.build.yaml @@ -20,35 +20,25 @@ services: build: dockerfile: docker/dockerfiles/tool-sidecar.Dockerfile context: .. - tool-structure: - image: unstract/tool-structure:${VERSION} + platform-service: + image: unstract/platform-service:${VERSION} build: - dockerfile: tools/structure/Dockerfile + dockerfile: docker/dockerfiles/platform.Dockerfile context: .. - tool-text_extractor: - image: unstract/tool-text_extractor:${VERSION} + x2text-service: + image: unstract/x2text-service:${VERSION} build: - dockerfile: tools/text_extractor/Dockerfile + dockerfile: docker/dockerfiles/x2text.Dockerfile context: .. tool-classifier: image: unstract/tool-classifier:${VERSION} build: dockerfile: tools/classifier/Dockerfile context: .. - platform-service: - image: unstract/platform-service:${VERSION} - build: - dockerfile: docker/dockerfiles/platform.Dockerfile - context: .. - prompt-service: - image: unstract/prompt-service:${VERSION} - build: - dockerfile: docker/dockerfiles/prompt.Dockerfile - context: .. - x2text-service: - image: unstract/x2text-service:${VERSION} + tool-text_extractor: + image: unstract/tool-text-extractor:${VERSION} build: - dockerfile: docker/dockerfiles/x2text.Dockerfile + dockerfile: tools/text_extractor/Dockerfile context: .. # Unified worker image (replaces all individual worker images) worker-unified: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 019c819ad6..5bb0cfff7c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -33,7 +33,6 @@ services: - minio - minio-bootstrap - platform-service - - prompt-service - x2text-service volumes: - prompt_studio_data:/app/prompt-studio-data @@ -145,6 +144,10 @@ services: - reverse-proxy environment: - ENVIRONMENT=development + # Running platform version, surfaced on the profile page. Reuses the same + # ${VERSION} that tags the image, so it always matches what's deployed + # (and updates on RC->stable promotion, which only retags — doesn't rebuild). + - UNSTRACT_APPS_VERSION=${VERSION} labels: - traefik.enable=true - traefik.http.routers.frontend.rule=Host(`frontend.unstract.localhost`) && !PathPrefix(`/api/v1`) && !PathPrefix(`/deployment`) && !PathPrefix(`/public`) @@ -164,23 +167,6 @@ services: labels: - traefik.enable=false - prompt-service: - <<: *host_gateway - image: unstract/prompt-service:${VERSION} - container_name: unstract-prompt-service - restart: unless-stopped - depends_on: - - db - - minio - - minio-bootstrap - - rabbitmq - ports: - - "3003:3003" - env_file: - - ../prompt-service/.env - labels: - - traefik.enable=false - x2text-service: image: unstract/x2text-service:${VERSION} container_name: unstract-x2text-service diff --git a/docker/dockerfiles/prompt.Dockerfile b/docker/dockerfiles/prompt.Dockerfile deleted file mode 100644 index cca9161d81..0000000000 --- a/docker/dockerfiles/prompt.Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -# Use a specific version of Python slim image -FROM python:3.12-slim-trixie AS base - -LABEL maintainer="Zipstack Inc." \ - description="Prompt Service Container" - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONPATH=/unstract \ - BUILD_CONTEXT_PATH=prompt-service \ - BUILD_PACKAGES_PATH=unstract \ - TARGET_PLUGINS_PATH=src/unstract/prompt_service/plugins \ - APP_USER=unstract \ - APP_HOME=/app \ - # OpenTelemetry configuration (disabled by default, enable in docker-compose) - OTEL_TRACES_EXPORTER=none \ - OTEL_METRICS_EXPORTER=none \ - OTEL_LOGS_EXPORTER=none \ - OTEL_SERVICE_NAME=unstract_prompt - -# Install system dependencies, create user, and setup directories in one layer -RUN apt-get update \ - && apt-get --no-install-recommends install -y \ - build-essential \ - libmagic-dev \ - pkg-config \ - git \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ - && adduser -u 5678 --disabled-password --gecos "" ${APP_USER} \ - && mkdir -p ${APP_HOME} \ - && chown -R ${APP_USER}:${APP_USER} ${APP_HOME} - -# Install uv package manager -COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /uvx /bin/ - -# Create working directory -WORKDIR ${APP_HOME} - -# ----------------------------------------------- -# EXTERNAL DEPENDENCIES STAGE - This layer gets cached if external dependencies don't change -# ----------------------------------------------- -FROM base AS ext-dependencies - -# Copy dependency-related files -COPY --chown=${APP_USER}:${APP_USER} ${BUILD_CONTEXT_PATH}/pyproject.toml ${BUILD_CONTEXT_PATH}/uv.lock ${BUILD_CONTEXT_PATH}/README.md ./ - -# Copy local package dependencies -COPY --chown=${APP_USER}:${APP_USER} ${BUILD_PACKAGES_PATH}/sdk1 /unstract/sdk1 -COPY --chown=${APP_USER}:${APP_USER} ${BUILD_PACKAGES_PATH}/core /unstract/core -COPY --chown=${APP_USER}:${APP_USER} ${BUILD_PACKAGES_PATH}/flags /unstract/flags - -# Increase timeout for large packages (flipt-client is ~45MB) -ENV UV_HTTP_TIMEOUT=120 - -# Switch to non-root user -USER ${APP_USER} - -# Install external dependencies from pyproject.toml -RUN uv sync --group deploy --locked --no-install-project --no-dev - -# ----------------------------------------------- -# FINAL STAGE - Minimal image for production -# ----------------------------------------------- -FROM ext-dependencies AS production - -# Set shell options for better error handling -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# Copy application code (this layer changes most frequently) -COPY --chown=${APP_USER}:${APP_USER} ${BUILD_CONTEXT_PATH} ./ - -# Switch to non-root user -USER ${APP_USER} - -# Install just the application in editable mode -RUN uv sync --group deploy --locked - -# Install plugins after copying source code -RUN for dir in "${TARGET_PLUGINS_PATH}"/*/; do \ - dirpath=${dir%*/}; \ - dirname=${dirpath##*/}; \ - if [ "${dirname}" != "*" ]; then \ - echo "Installing plugin: ${dirname}..." && \ - uv pip install "${TARGET_PLUGINS_PATH}/${dirname}"; \ - fi; \ - done && \ - uv run opentelemetry-bootstrap -a requirements | uv pip install --requirement - && \ - # Use OpenTelemetry v1 - v2 breaks LiteLLM with instrumentation enabled - uv pip uninstall opentelemetry-instrumentation-openai-v2 && \ - uv pip install opentelemetry-instrumentation-openai - -# Create required directories -RUN mkdir -p prompt-studio-data - -EXPOSE 3003 - -ARG VERSION=dev -ENV UNSTRACT_APPS_VERSION=${VERSION} - -CMD ["./entrypoint.sh"] diff --git a/docker/dockerfiles/prompt.Dockerfile.dockerignore b/docker/dockerfiles/prompt.Dockerfile.dockerignore deleted file mode 100644 index f2cffbf5be..0000000000 --- a/docker/dockerfiles/prompt.Dockerfile.dockerignore +++ /dev/null @@ -1,67 +0,0 @@ -**/__pycache__ -**/.mypy_cache -**/.pytest_cache -**/.python-version -**/.pyc -**/.pyo -**/.venv -**/.classpath -**/.dockerignore -**/.env -**/.git -**/.gitignore -**/.gitkeep -**/.project -**/.settings -**/.toolstarget -**/.vs -**/.vscode -**/*.*proj.user -**/*.dbmdl -**/*.jfm -**/bin -**/charts -**/docker-compose* -**/compose* -**/Dockerfile* -**/build -**/dist -**/node_modules -**/npm-debug.log -**/obj -**/secrets.dev.yaml -**/values.dev.yaml -**/.db -**/.sqlite3 -**/.log -**/*-log.txt -**/*.drawio -**/.tmp -**/.swp -**/.swo -**/.bak -*.idea -*.vscode -*.git -!LICENSE -*.md -!README.md -# But include the agent prompt markdown files -!*_system_prompt.md -.jshintrc -.pre-commit-config.yaml -**/tests -test*.py -**/*.egg-info - -backend -frontend -platform-service -!prompt-service -tools -unstract -!unstract/core -!unstract/flags -!unstract/sdk1 -runner -x2text-service diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yaml index 02e1ede731..b0e98d8b1e 100644 --- a/docker/sample.compose.override.yaml +++ b/docker/sample.compose.override.yaml @@ -138,38 +138,6 @@ services: - action: rebuild path: ../platform-service/uv.lock - ######################################################################################################### - # Prompt Service (memory optimized: 1 worker, 2 threads) - prompt-service: - image: unstract/prompt-service:${VERSION} - build: - dockerfile: docker/dockerfiles/prompt.Dockerfile - context: .. - entrypoint: ["bash", "-c"] - command: [ - "uv run python -Xfrozen_modules=off .venv/bin/gunicorn - --bind 0.0.0.0:3003 - --workers 1 - --threads 2 - --worker-class gthread - --log-level debug - --timeout 900 - --access-logfile - - --graceful-timeout 5 unstract.prompt_service.run:app" - ] - develop: - watch: - - action: sync+restart - path: ../prompt-service/ - target: /app - ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/, node_modules/] - - action: sync+restart - path: ../unstract/ - target: /unstract - ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/] - - action: rebuild - path: ../prompt-service/uv.lock - ######################################################################################################### # X2Text Service x2text-service: diff --git a/docker/scripts/bump_sdk_v0_version.sh b/docker/scripts/bump_sdk_v0_version.sh index 7f13e22543..5bbc1c0f68 100755 --- a/docker/scripts/bump_sdk_v0_version.sh +++ b/docker/scripts/bump_sdk_v0_version.sh @@ -10,18 +10,15 @@ set -e UNSTRACT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" BACKEND_DIR="${UNSTRACT_ROOT}/backend" PLATFORM_SERVICE_DIR="${UNSTRACT_ROOT}/platform-service" -PROMPT_SERVICE_DIR="${UNSTRACT_ROOT}/prompt-service" FILESYSTEM_DIR="${UNSTRACT_ROOT}/unstract/filesystem" TOOL_REGISTRY_DIR="${UNSTRACT_ROOT}/unstract/tool-registry" CLASSIFIER_DIR="${UNSTRACT_ROOT}/tools/classifier" -STRUCTURE_DIR="${UNSTRACT_ROOT}/tools/structure" TEXT_EXTRACTOR_DIR="${UNSTRACT_ROOT}/tools/text_extractor" # Organized directory arrays -STRUCTURE_TOOL_DIRS=("$STRUCTURE_DIR") CUSTOM_TOOL_DIRS=("$CLASSIFIER_DIR" "$TEXT_EXTRACTOR_DIR") PACKAGE_DIRS=("$FILESYSTEM_DIR" "$TOOL_REGISTRY_DIR") -SERVICE_DIRS=("$BACKEND_DIR" "$PLATFORM_SERVICE_DIR" "$PROMPT_SERVICE_DIR") +SERVICE_DIRS=("$BACKEND_DIR" "$PLATFORM_SERVICE_DIR") ROOT_DIR="$UNSTRACT_ROOT" # Additional files having version @@ -152,66 +149,6 @@ check_file() { fi } -# Update structure tool version in sample.env -update_structure_tool_version() { - local file="$1" - local version_arg="$2" - local old_url_version - local old_tag_version - local new_version - - echo "Processing structure tool..." - - # Check if file exists - if [[ ! -f "$file" ]]; then - echo "Warning: $file not found, skipping structure tool update" - return - fi - - log "Checking structure tool version in $file" - check_file "$file" - - # NOTE: Currently we bump only patch version for tools - if [[ "$version_arg" != *"."* ]]; then - version_arg="patch" - echo "Set target version for structure tool as $version_arg" - fi - - # Extract current structure tool versions - old_url_version=$(grep 'STRUCTURE_TOOL_IMAGE_URL' "$file" | grep -o '[0-9.]\+"$' | tr -d '"') - old_tag_version=$(grep 'STRUCTURE_TOOL_IMAGE_TAG' "$file" | grep -o '[0-9.]\+"$' | tr -d '"') - - if [[ -z "$old_url_version" || -z "$old_tag_version" ]]; then - echo "Error: Could not find structure tool version in $file" - exit 1 - fi - - log "Found structure tool URL version: $old_url_version" - log "Found structure tool TAG version: $old_tag_version" - - # Handle special version keywords - if [[ "$version_arg" == "patch" || "$version_arg" == "minor" || "$version_arg" == "major" ]]; then - new_version=$(bump_version "$old_url_version" "$version_arg") - log "Auto-bumped structure tool version: $old_url_version -> $new_version ($version_arg)" - else - new_version="$version_arg" - fi - - if [[ "$old_url_version" == "$new_version" && "$old_tag_version" == "$new_version" ]]; then - echo "Structure tool version already at $new_version, skipping update" - return - fi - - if [[ "$DRY_RUN" == true ]]; then - echo "Would update structure tool URL version from $old_url_version to $new_version in $file" - echo "Would update structure tool TAG version from $old_tag_version to $new_version in $file" - else - echo "Updating structure tool versions from $old_url_version/$old_tag_version to $new_version in $file" - sed -i "s/STRUCTURE_TOOL_IMAGE_URL=\"docker:unstract\/tool-structure:$old_url_version\"/STRUCTURE_TOOL_IMAGE_URL=\"docker:unstract\/tool-structure:$new_version\"/g" "$file" - sed -i "s/STRUCTURE_TOOL_IMAGE_TAG=\"$old_tag_version\"/STRUCTURE_TOOL_IMAGE_TAG=\"$new_version\"/g" "$file" - fi -} - # Update custom tool version and its SDK dependency update_custom_tool_version() { local dir="$1" @@ -416,8 +353,6 @@ reset_directory_changes() { reset_file "$BACKEND_DIR/uv.lock" reset_file "$PLATFORM_SERVICE_DIR/pyproject.toml" reset_file "$PLATFORM_SERVICE_DIR/uv.lock" - reset_file "$PROMPT_SERVICE_DIR/pyproject.toml" - reset_file "$PROMPT_SERVICE_DIR/uv.lock" # Reset package directories reset_file "$FILESYSTEM_DIR/pyproject.toml" @@ -431,7 +366,7 @@ reset_directory_changes() { reset_file "$TEXT_EXTRACTOR_DIR/requirements.txt" reset_file "$TEXT_EXTRACTOR_DIR/src/config/properties.json" - # Reset structure tool directories + # Reset additional files reset_file "$SAMPLE_ENV_FILE" reset_file "$TOOL_REGISTRY_JSON_FILE" } @@ -463,30 +398,27 @@ process_directories() { echo "[-] Processing all directories and files for version bump..." - # 1. Process structure tool - update_structure_tool_version "$SAMPLE_ENV_FILE" "$TARGET_VERSION" - - # 2. Process custom tools + # 1. Process custom tools echo "[-] Processing custom tools..." for dir in "${CUSTOM_TOOL_DIRS[@]}"; do update_custom_tool_version "$dir" "$TARGET_VERSION" "$TOOL_REGISTRY_JSON_FILE" done - # 3. Process packages (similar to services) + # 2. Process packages (similar to services) echo "[-] Processing packages..." for dir in "${PACKAGE_DIRS[@]}"; do update_sdk_version "$dir/pyproject.toml" "$TARGET_VERSION" generate_uvlock "$dir" done - # 4. Process services + # 3. Process services echo "[-] Processing services..." for dir in "${SERVICE_DIRS[@]}"; do update_sdk_version "$dir/pyproject.toml" "$TARGET_VERSION" generate_uvlock "$dir" done - # 5. Process root directory + # 4. Process root directory echo "[-] Processing root directory..." update_sdk_version "$ROOT_DIR/pyproject.toml" "$TARGET_VERSION" generate_uvlock "$ROOT_DIR" diff --git a/docker/scripts/uv-lock-gen/README.md b/docker/scripts/uv-lock-gen/README.md index f5c565640e..2c37fa758f 100644 --- a/docker/scripts/uv-lock-gen/README.md +++ b/docker/scripts/uv-lock-gen/README.md @@ -2,11 +2,10 @@ Helps generate uv's lockfiles by running `uv sync` on all necessary packages and services. -It also detects **transitive dependency changes** — if a local path dependency's `pyproject.toml` changed (e.g. `unstract/sdk1`), all services that depend on it (e.g. `backend`, `prompt-service`) will have their lockfiles regenerated too. +It also detects **transitive dependency changes** — if a local path dependency's `pyproject.toml` changed (e.g. `unstract/sdk1`), all services that depend on it (e.g. `backend`, `workers`) will have their lockfiles regenerated too. - project root - `backend` -- `prompt-service` - `runner` - `unstract/core` - `unstract/flags` @@ -22,5 +21,5 @@ Can be run without any arguments to check for lockfile generation on all necessa Accepts a list of directories to generate for as command line arguments ```shell -./uv-lock.sh backend prompt-service +./uv-lock.sh backend platform-service ``` diff --git a/docker/scripts/uv-lock-gen/uv-lock.sh b/docker/scripts/uv-lock-gen/uv-lock.sh index 9c61bf0ed5..7180b9fd56 100755 --- a/docker/scripts/uv-lock-gen/uv-lock.sh +++ b/docker/scripts/uv-lock-gen/uv-lock.sh @@ -100,7 +100,6 @@ list_descendants () directories=( "." "backend" - "prompt-service" "platform-service" "runner" "x2text-service" diff --git a/frontend/generate-runtime-config.sh b/frontend/generate-runtime-config.sh index 12d5f14422..51dba1426e 100755 --- a/frontend/generate-runtime-config.sh +++ b/frontend/generate-runtime-config.sh @@ -5,12 +5,21 @@ # Generate the runtime-config.js file with the current environment variables # Note: Using VITE_ prefix for Vite compatibility, fallback to REACT_APP_ for backward compatibility + +# Escape backslashes and double quotes so values stay valid inside JS strings +js_escape() { + printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' +} + +APP_VERSION=$(js_escape "${UNSTRACT_APPS_VERSION:-}") + cat > /usr/share/nginx/html/config/runtime-config.js << EOF // This file is auto-generated at runtime. Do not modify manually. window.RUNTIME_CONFIG = { faviconPath: "${VITE_FAVICON_PATH:-${REACT_APP_FAVICON_PATH}}", logoUrl: "${VITE_CUSTOM_LOGO_URL:-${REACT_APP_CUSTOM_LOGO_URL}}", - enablePosthog: "${VITE_ENABLE_POSTHOG:-${REACT_APP_ENABLE_POSTHOG}}" + enablePosthog: "${VITE_ENABLE_POSTHOG:-${REACT_APP_ENABLE_POSTHOG}}", + version: "${APP_VERSION}" }; EOF diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 3acc55381f..21dc36d3a7 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -28,6 +28,12 @@ http { keepalive_timeout 65; gzip on; + # nginx gzips only text/html by default; add asset types (~5-6x smaller). + gzip_types text/css application/javascript application/json image/svg+xml; + gzip_min_length 1024; + gzip_vary on; + # GKE LB adds a Via header; default gzip_proxied off would skip every request. + gzip_proxied any; # Non-root user will not have access to default # /var/cache/nginx/ prefix. diff --git a/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.css b/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.css index b4a330b154..aa6d44c188 100644 --- a/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.css +++ b/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.css @@ -1,5 +1,25 @@ /* Styles for AddLlmProfile */ +.conn-modal-form-pad-left:has(> .add-llm-profile-body) { + padding: 8px 16px; +} + +.conn-modal-row:has(.add-llm-profile-body) { + height: auto !important; +} +.conn-modal-col:has(.add-llm-profile-body) { + overflow-y: visible; +} + +.add-llm-profile-footer { + position: sticky; + bottom: 0; + z-index: 1; + margin-bottom: 0; + padding-top: 12px; + border-top: 1px solid var(--ant-color-border-secondary, #f0f0f0); +} + .add-llm-profile-row { width: 100%; } diff --git a/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.jsx b/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.jsx index bac0e5325f..80cf854c2d 100644 --- a/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.jsx +++ b/frontend/src/components/custom-tools/add-llm-profile/AddLlmProfile.jsx @@ -453,7 +453,7 @@ function AddLlmProfile({ }; return ( -
+
- + {editLlmProfileId ? "Update" : "Add"} diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx index 7e30a99339..a80d0d145c 100644 --- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx +++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx @@ -1,4 +1,4 @@ -import "prismjs"; +// global Prism is installed eagerly at app entry (helpers/prismSetup.js) import "prismjs/components/prism-json"; import "prismjs/plugins/line-numbers/prism-line-numbers.css"; import "prismjs/plugins/line-numbers/prism-line-numbers.js"; diff --git a/frontend/src/components/custom-tools/combined-output/JsonView.jsx b/frontend/src/components/custom-tools/combined-output/JsonView.jsx index 36e11c4648..0ea8d754a2 100644 --- a/frontend/src/components/custom-tools/combined-output/JsonView.jsx +++ b/frontend/src/components/custom-tools/combined-output/JsonView.jsx @@ -1,4 +1,4 @@ -import { Tabs } from "antd"; +import { Tabs, Tooltip } from "antd"; import TabPane from "antd/es/tabs/TabPane"; import Prism from "prismjs"; import PropTypes from "prop-types"; @@ -70,7 +70,15 @@ function JsonView({ )} {adapterData.map((adapter) => ( {adapter?.llm_model || adapter?.profile_name}} + tab={ + + + {adapter?.profile_name || + adapter?.llm_model || + `Profile ${adapter?.profile_id ?? "?"}`} + + + } key={adapter?.profile_id} /> ))} diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx index ce8b81ec54..8207583111 100644 --- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx +++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx @@ -3,7 +3,7 @@ import { CloseCircleFilled, InfoCircleFilled, } from "@ant-design/icons"; -import { Button, Modal, Table, Tabs, Typography } from "antd"; +import { Button, Modal, Table, Tabs, Tooltip, Typography } from "antd"; import TabPane from "antd/es/tabs/TabPane"; import PropTypes from "prop-types"; import { useEffect, useState } from "react"; @@ -340,7 +340,15 @@ function OutputForDocModal({ Default} key={"0"}> {adapterData?.map((adapter, index) => ( {adapter?.llm_model || adapter?.profile_name}} + tab={ + + + {adapter?.profile_name || + adapter?.llm_model || + `Profile ${index + 1}`} + + + } key={(index + 1)?.toString()} > ))} diff --git a/frontend/src/components/error/LazyOutlet/LazyOutlet.css b/frontend/src/components/error/LazyOutlet/LazyOutlet.css new file mode 100644 index 0000000000..45e908db26 --- /dev/null +++ b/frontend/src/components/error/LazyOutlet/LazyOutlet.css @@ -0,0 +1,17 @@ +/* Center the route-loading fallback in the content area. */ +.lazyOutletFallback { + display: flex; + width: 100%; + min-height: 60vh; + align-items: center; + justify-content: center; +} + +/* GenericLoader's .center uses `width/height: inherit`, which collapses to a + narrow column inside the flex content area (the loader text ends up wrapping + one word per line next to the sidebar). Pin it to the full fallback width so + it renders centered at its natural size instead. */ +.lazyOutletFallback > .center { + width: 100%; + height: auto; +} diff --git a/frontend/src/components/error/LazyOutlet/LazyOutlet.jsx b/frontend/src/components/error/LazyOutlet/LazyOutlet.jsx index ce321c9c60..fb43d5a0eb 100644 --- a/frontend/src/components/error/LazyOutlet/LazyOutlet.jsx +++ b/frontend/src/components/error/LazyOutlet/LazyOutlet.jsx @@ -4,6 +4,7 @@ import { Outlet, useLocation } from "react-router-dom"; import { GenericLoader } from "../../generic-loader/GenericLoader.jsx"; import { ErrorBoundary } from "../../widgets/error-boundary/ErrorBoundary.jsx"; +import "./LazyOutlet.css"; // A failed dynamic import — almost always a stale hashed chunk after a // redeploy (the client's index.html references a filename the CDN no longer @@ -74,7 +75,15 @@ export function LazyOutlet() { onError={handleRouteError} fallbackComponent={} > - }> + {/* Wrap the loader so it fills the content area and stays centered — see + LazyOutlet.css for why GenericLoader needs this here. */} + + +
+ } + > diff --git a/frontend/src/components/profile/Profile.jsx b/frontend/src/components/profile/Profile.jsx index 899110c777..aedf755e44 100644 --- a/frontend/src/components/profile/Profile.jsx +++ b/frontend/src/components/profile/Profile.jsx @@ -11,6 +11,7 @@ import { useNavigate } from "react-router-dom"; import "./Profile.css"; import { OrganizationIcon } from "../../assets"; +import config from "../../config"; import { useAxiosPrivate } from "../../hooks/useAxiosPrivate"; import { useAlertStore } from "../../store/alert-store.js"; import { useSessionStore } from "../../store/session-store.js"; @@ -234,6 +235,18 @@ function Profile() { )} + {config.version && ( +
+ + Platform Version + +
+ + {config.version} + +
+
+ )} diff --git a/frontend/src/components/settings/platform/PlatformSettings.jsx b/frontend/src/components/settings/platform/PlatformSettings.jsx index e62b48a911..80893e5dae 100644 --- a/frontend/src/components/settings/platform/PlatformSettings.jsx +++ b/frontend/src/components/settings/platform/PlatformSettings.jsx @@ -10,6 +10,7 @@ import { Input, InputNumber, Row, + Switch, Tag, Typography, } from "antd"; @@ -28,6 +29,19 @@ import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx"; import usePostHogEvents from "../../../hooks/usePostHogEvents.js"; import { SettingsLayout } from "../settings-layout/SettingsLayout.jsx"; +// The "restrict LLM creation" control is enterprise/cloud-only: org admin +// roles and user management don't exist in OSS, so this setting is meaningless +// there. Detect the enterprise build by probing for an enterprise-only plugin +// (absent in OSS) and hide the control entirely otherwise. Mirrors the +// plugin-gating idiom used in SideNavBar. +let isEnterpriseBuild = false; +try { + await import("../../../plugins/store/unstract-subscription-plan-store"); + isEnterpriseBuild = true; +} catch { + // OSS build — enterprise plugins are not bundled. +} + const defaultKeys = [ { id: null, @@ -85,6 +99,14 @@ function PlatformSettings() { // UI shows minutes; wire format (and ConfigSpec.value) is seconds. const [batchIntervalMinutes, setBatchIntervalMinutes] = useState(null); const [isSavingInterval, setIsSavingInterval] = useState(false); + // Controlled-mode flag: only admins can create LLM adapters when true. + const [restrictLlmCreation, setRestrictLlmCreation] = useState(false); + const [isSavingRestriction, setIsSavingRestriction] = useState(false); + // Controlled-mode flag: only admins can create connectors when true. + const [restrictConnectorCreation, setRestrictConnectorCreation] = + useState(false); + const [isSavingConnectorRestriction, setIsSavingConnectorRestriction] = + useState(false); const { sessionDetails } = useSessionStore(); const { setAlertDetails } = useAlertStore(); const axiosPrivate = useAxiosPrivate(); @@ -119,6 +141,96 @@ function PlatformSettings() { }); }, [sessionDetails?.orgId]); + useEffect(() => { + // Enterprise/cloud-only + admin-only setting; skip the call in OSS and for + // non-admins (the endpoint is admin-gated and would 403) and before + // session hydration. + if ( + !isEnterpriseBuild || + !sessionDetails?.orgId || + !sessionDetails?.isAdmin + ) { + return; + } + axiosPrivate({ + method: "GET", + url: `/api/v1/unstract/${sessionDetails?.orgId}/organization/settings`, + }) + .then((res) => { + setRestrictLlmCreation( + Boolean(res?.data?.restrict_llm_adapter_creation), + ); + setRestrictConnectorCreation( + Boolean(res?.data?.restrict_connector_creation), + ); + }) + .catch((err) => { + console.warn("Failed to load organization settings", err); + }); + }, [sessionDetails?.orgId, sessionDetails?.isAdmin]); + + const handleToggleRestriction = (checked) => { + const previous = restrictLlmCreation; + setRestrictLlmCreation(checked); // optimistic + setIsSavingRestriction(true); + axiosPrivate({ + method: "PATCH", + url: `/api/v1/unstract/${sessionDetails?.orgId}/organization/settings`, + headers: { + "X-CSRFToken": sessionDetails?.csrfToken, + "Content-Type": "application/json", + }, + data: { restrict_llm_adapter_creation: checked }, + }) + .then((res) => { + setRestrictLlmCreation( + Boolean(res?.data?.restrict_llm_adapter_creation), + ); + setAlertDetails({ + type: "success", + content: "LLM adapter creation setting updated.", + }); + }) + .catch((err) => { + setRestrictLlmCreation(previous); // revert on failure + setAlertDetails(handleException(err, "Failed to update setting")); + }) + .finally(() => { + setIsSavingRestriction(false); + }); + }; + + const handleToggleConnectorRestriction = (checked) => { + const previous = restrictConnectorCreation; + setRestrictConnectorCreation(checked); // optimistic + setIsSavingConnectorRestriction(true); + axiosPrivate({ + method: "PATCH", + url: `/api/v1/unstract/${sessionDetails?.orgId}/organization/settings`, + headers: { + "X-CSRFToken": sessionDetails?.csrfToken, + "Content-Type": "application/json", + }, + data: { restrict_connector_creation: checked }, + }) + .then((res) => { + setRestrictConnectorCreation( + Boolean(res?.data?.restrict_connector_creation), + ); + setAlertDetails({ + type: "success", + content: "Connector creation setting updated.", + }); + }) + .catch((err) => { + setRestrictConnectorCreation(previous); // revert on failure + setAlertDetails(handleException(err, "Failed to update setting")); + }) + .finally(() => { + setIsSavingConnectorRestriction(false); + }); + }; + const handleSaveInterval = () => { if ( !batchIntervalMinutes || @@ -483,6 +595,58 @@ function PlatformSettings() { + {isEnterpriseBuild && sessionDetails?.isAdmin && ( +
+ + LLM Adapter Creation + + + Restrict creation of LLM connections to organization admins. + When enabled, non-admin users cannot create LLM adapters. + +
+
+ + + Only admins can create LLM adapters + +
+
+
+ )} + {isEnterpriseBuild && sessionDetails?.isAdmin && ( +
+ + Connector Creation + + + Restrict creation of connectors to organization admins. When + enabled, non-admin users cannot create connectors. + +
+
+ + + Only admins can create connectors + +
+
+
+ )} diff --git a/frontend/src/config.js b/frontend/src/config.js index c3c79d4171..1f1abc3bf8 100644 --- a/frontend/src/config.js +++ b/frontend/src/config.js @@ -17,6 +17,7 @@ const config = { import.meta.env.VITE_FAVICON_PATH || "/favicon.ico", logoUrl: runtimeConfig.logoUrl || import.meta.env.VITE_CUSTOM_LOGO_URL, + version: runtimeConfig.version || import.meta.env.VITE_VERSION, // Add more values as OR case, if needed for fallback. }; diff --git a/frontend/src/helpers/prismSetup.js b/frontend/src/helpers/prismSetup.js new file mode 100644 index 0000000000..6c20d8ce2e --- /dev/null +++ b/frontend/src/helpers/prismSetup.js @@ -0,0 +1,20 @@ +import Prism from "prismjs"; + +// The prismjs language/plugin add-ons used across the app (`prismjs/components/*`, +// `prismjs/plugins/*`) register their grammars onto a *bare global* `Prism` and +// carry no import edge back to prismjs core. Under Vite's code-split production +// build those add-ons get hoisted into a shared chunk (Combined Output *and* the +// manual-review ResultEditor both import `prism-json`), which can evaluate before +// any per-component setup runs — so they threw `ReferenceError: Prism is not +// defined`, blanking the Prompt Studio detail page and the HITL review page. +// +// This module is imported EAGERLY from the app entry (`index.jsx`) so the global +// is installed at bootstrap, before any lazy chunk — including the shared one +// holding the add-ons — can load. Assign unconditionally: the global must be THIS +// instance, the one the add-ons extend and that `Prism.highlightAll()` reads; a +// `!globalThis.Prism` guard could leave a different, pre-existing Prism in place +// and silently drop highlighting. The used `Prism` binding also keeps the bundler +// from tree-shaking core away. +globalThis.Prism = Prism; + +export default Prism; diff --git a/frontend/src/index.jsx b/frontend/src/index.jsx index 66079513dd..608edbf951 100644 --- a/frontend/src/index.jsx +++ b/frontend/src/index.jsx @@ -1,3 +1,6 @@ +// Install the global `Prism` at bootstrap, before any lazy chunk (incl. the +// shared one holding prismjs add-ons) can load. See helpers/prismSetup.js. +import "./helpers/prismSetup"; import posthog from "posthog-js"; import { PostHogProvider } from "posthog-js/react"; import React from "react"; diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsx index 4f307a0199..2111417bc2 100644 --- a/frontend/src/routes/Router.jsx +++ b/frontend/src/routes/Router.jsx @@ -59,6 +59,14 @@ const MarketplaceStripeConflictEntry = lazyPlugin( () => import("../plugins/marketplace"), "MarketplaceStripeConflictPage", ); +// LLMWhisperer buyers get their own entry point: the LLMWhisperer Tackle +// listings redirect here, and the page claims against the LLMWhisperer portal +// backend (a different endpoint than the Unstract claim) — routing them to +// /marketplace-landing would fire the Unstract claim instead. +const LlmWhispererMarketplaceLandingEntry = lazyPlugin( + () => import("../plugins/llm-whisperer/pages/MarketplaceLandingPage.jsx"), + "MarketplaceLandingPage", +); // Simple Prompt Studio pages. const SimplePromptStudioHelper = lazyPlugin( @@ -217,6 +225,12 @@ function Router() { element={} /> )} + {LlmWhispererMarketplaceLandingEntry && ( + } + /> + )} {CustomPlanCheckoutPage && ( { target: "esnext", outDir: "build", sourcemap: true, + // Single stylesheet: per-chunk CSS loads in navigation order, making + // equal-specificity cross-component rules resolve unpredictably. JS + // splitting is unaffected. + cssCodeSplit: false, // Chunk size warning limit chunkSizeWarningLimit: 1000, rollupOptions: { diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py index 85f759e209..fbc8cdd8bb 100644 --- a/platform-service/src/unstract/platform_service/controller/platform.py +++ b/platform-service/src/unstract/platform_service/controller/platform.py @@ -1,3 +1,4 @@ +import functools import json import uuid from datetime import datetime @@ -33,6 +34,9 @@ def get_token_from_auth_header(request: Request) -> Any: def authentication_middleware(func: Any) -> Any: + # Flask names endpoints after the view's __name__; without wraps every + # decorated route registers as "wrapper" and collides. + @functools.wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Any: token = get_token_from_auth_header(request) # Check if bearer token exists and validate it @@ -111,7 +115,7 @@ def validate_bearer_token(token: str | None) -> bool: return False -@platform_bp.route("/page-usage", methods=["POST"], endpoint="page_usage") +@platform_bp.route("/page-usage", methods=["POST"]) @authentication_middleware def page_usage() -> Any: """Usage endpoint.""" @@ -262,11 +266,7 @@ def usage() -> Any: return make_response(result, 500) -@platform_bp.route( - "/platform_details", - methods=["GET"], - endpoint="platform_details", -) +@platform_bp.route("/platform_details", methods=["GET"]) @authentication_middleware def platform_details() -> Any: """Fetch details associated with the platform. This uses the authenticated @@ -288,7 +288,7 @@ def platform_details() -> Any: return result, 200 -@platform_bp.route("/cache", methods=["POST", "GET", "DELETE"], endpoint="cache") +@platform_bp.route("/cache", methods=["POST", "GET", "DELETE"]) @authentication_middleware def cache() -> Any: """Cache endpoint. @@ -348,11 +348,7 @@ def cache() -> Any: return "OK", 200 -@platform_bp.route( - "/adapter_instance", - methods=["GET"], - endpoint="adapter_instance", -) +@platform_bp.route("/adapter_instance", methods=["GET"]) @authentication_middleware def adapter_instance() -> Any: """Fetch Adapter instance. @@ -399,11 +395,7 @@ def adapter_instance() -> Any: raise APIError(message=msg) -@platform_bp.route( - "/custom_tool_instance", - methods=["GET"], - endpoint="custom_tool_instance", -) +@platform_bp.route("/custom_tool_instance", methods=["GET"]) @authentication_middleware def custom_tool_instance() -> Any: """Fetching exported custom tool instance. @@ -438,11 +430,7 @@ def custom_tool_instance() -> Any: raise APIError(message=msg) from e -@platform_bp.route( - "/agentic_tool_instance", - methods=["GET"], - endpoint="agentic_tool_instance", -) +@platform_bp.route("/agentic_tool_instance", methods=["GET"]) @authentication_middleware def agentic_tool_instance() -> Any: """Fetching exported agentic tool instance. @@ -479,11 +467,7 @@ def agentic_tool_instance() -> Any: raise APIError(message=msg) from e -@platform_bp.route( - "/llm_profile_instance", - methods=["GET"], - endpoint="llm_profile_instance", -) +@platform_bp.route("/llm_profile_instance", methods=["GET"]) @authentication_middleware def llm_profile_instance() -> Any: """Fetching llm profile instance.""" diff --git a/platform-service/tests/conftest.py b/platform-service/tests/conftest.py new file mode 100644 index 0000000000..04b6803b34 --- /dev/null +++ b/platform-service/tests/conftest.py @@ -0,0 +1,6 @@ +import os + +# platform_service.env validates required settings at import time. +os.environ.setdefault("REDIS_HOST", "localhost") +os.environ.setdefault("ENCRYPTION_KEY", "test-key") +os.environ.setdefault("DB_SCHEMA", "unstract") diff --git a/platform-service/tests/test_auth_middleware.py b/platform-service/tests/test_auth_middleware.py index decf9aaa3b..28cb0d0e39 100644 --- a/platform-service/tests/test_auth_middleware.py +++ b/platform-service/tests/test_auth_middleware.py @@ -1,19 +1,58 @@ -import unittest +"""Tests for bearer token validation used by `authentication_middleware`.""" -from unstract.platform_service.main import ( - get_account_from_bearer_token, - validate_bearer_token, -) +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +import pytest +from flask import Flask +from unstract.platform_service.controller import platform -class TestAuthMiddleware(unittest.TestCase): - def test_auth_middleware(self) -> None: - try: - self.assertTrue(validate_bearer_token("test")) - self.assertEqual(get_account_from_bearer_token("test"), "mock_org") - except Exception as e: - self.fail(f"Authentication Test failed: {e}") +@pytest.fixture +def app_ctx() -> Iterator[None]: + """`validate_bearer_token` logs via `flask.current_app`.""" + with Flask(__name__).app_context(): + yield -if __name__ == "__main__": - unittest.main() + +def _mock_safe_cursor(monkeypatch: pytest.MonkeyPatch, result_row: Any) -> None: + class _Cursor: + def fetchone(self) -> Any: + return result_row + + @contextmanager + def _safe_cursor(query: str, params: tuple = ()) -> Iterator[_Cursor]: + yield _Cursor() + + monkeypatch.setattr(platform, "safe_cursor", _safe_cursor) + + +def test_valid_active_token(app_ctx: None, monkeypatch: pytest.MonkeyPatch) -> None: + _mock_safe_cursor(monkeypatch, ("id", "test-token", True)) + assert platform.validate_bearer_token("test-token") is True + + +def test_rejects_missing_token(app_ctx: None) -> None: + assert platform.validate_bearer_token(None) is False + + +def test_rejects_unknown_token(app_ctx: None, monkeypatch: pytest.MonkeyPatch) -> None: + _mock_safe_cursor(monkeypatch, None) + assert platform.validate_bearer_token("test-token") is False + + +def test_rejects_inactive_token(app_ctx: None, monkeypatch: pytest.MonkeyPatch) -> None: + _mock_safe_cursor(monkeypatch, ("id", "test-token", False)) + assert platform.validate_bearer_token("test-token") is False + + +def test_db_error_denies_access(app_ctx: None, monkeypatch: pytest.MonkeyPatch) -> None: + """Auth must fail closed when the token lookup blows up.""" + + # Raising on call suffices: `safe_cursor(...)` is invoked inside the try. + def _boom(query: str, params: tuple = ()) -> Any: + raise RuntimeError("db down") + + monkeypatch.setattr(platform, "safe_cursor", _boom) + assert platform.validate_bearer_token("test-token") is False diff --git a/platform-service/tests/test_route_endpoints.py b/platform-service/tests/test_route_endpoints.py new file mode 100644 index 0000000000..f0b1725258 --- /dev/null +++ b/platform-service/tests/test_route_endpoints.py @@ -0,0 +1,18 @@ +"""Guards that `authentication_middleware` stays transparent to Flask. + +Without `functools.wraps` every decorated route registers as "wrapper", so the +second such route raises at import time. +""" + +from flask import Flask +from unstract.platform_service.controller.platform import platform_bp + + +def test_routes_register_under_their_function_names() -> None: + # NOSONAR — app is never served; bearer-token routes carry no CSRF surface. + app = Flask(__name__) # NOSONAR + app.register_blueprint(platform_bp) + + endpoints = {rule.endpoint for rule in app.url_map.iter_rules()} + assert "platform.wrapper" not in endpoints + assert "platform.usage" in endpoints diff --git a/platform-service/uv.lock b/platform-service/uv.lock index b904d7bc73..53346c3a7b 100644 --- a/platform-service/uv.lock +++ b/platform-service/uv.lock @@ -1102,7 +1102,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1118,9 +1118,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -2726,7 +2726,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/prompt-service/.gitignore b/prompt-service/.gitignore deleted file mode 100644 index ae2f699ad0..0000000000 --- a/prompt-service/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Plugins -src/unstract/prompt_service/plugins diff --git a/prompt-service/.python-version b/prompt-service/.python-version deleted file mode 100644 index f3fe474aee..0000000000 --- a/prompt-service/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12.9 diff --git a/prompt-service/README.md b/prompt-service/README.md deleted file mode 100644 index a6f313e012..0000000000 --- a/prompt-service/README.md +++ /dev/null @@ -1 +0,0 @@ -#TODO : Add README.md diff --git a/prompt-service/entrypoint.sh b/prompt-service/entrypoint.sh deleted file mode 100755 index a2eef06dc0..0000000000 --- a/prompt-service/entrypoint.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -show_help() { - echo "Usage: ./entrypoint.sh [OPTIONS]" - echo "" - echo "Options:" - echo " --dev Run Gunicorn in development mode with --reload and reduced graceful timeout (5s)." - echo " --help, -h Show this help message and exit." -} - -# Parse arguments -dev=false - -while [[ "$#" -gt 0 ]]; do - case $1 in - --dev) dev=true ;; - --help|-h) show_help; exit 0 ;; - *) echo "Unknown argument: $1"; exit 1 ;; - esac - shift -done - -gunicorn_args=( - --bind 0.0.0.0:3003 - --workers 2 - --threads 2 - --worker-class gthread - --log-level debug - --timeout 900 - --access-logfile - -) - -if [ "$dev" = true ]; then - echo "Running in development mode" - gunicorn_args+=(--reload --graceful-timeout 5) -else - echo "Running in production mode" -fi - -# Start Gunicorn -.venv/bin/gunicorn "${gunicorn_args[@]}" unstract.prompt_service.run:app diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml deleted file mode 100644 index b617a02dc9..0000000000 --- a/prompt-service/pyproject.toml +++ /dev/null @@ -1,65 +0,0 @@ -[project] -name = "unstract-prompt-service" -version = "0.0.1" -description = "Unstract's prompt studio helper" -authors = [{ name = "Zipstack Inc.", email = "devsupport@zipstack.com" }] -requires-python = ">=3.12,<3.13" -readme = "README.md" -# license = {text = "MIT"} - -dependencies = [ - "peewee~=3.16", - "nltk~=3.9", - "flask~=3.1", - "llama-index>=0.14.13", - "python-dotenv==1.2.2", - "json-repair~=0.42.0", - "requests>=2.33.0,<3.0", - "redis>=5.0.3,<5.3", - "unstract-core", - "unstract-flags", - "unstract-sdk1[aws,gcs,azure]" -] - -[tool.uv.sources] -unstract-flags = { path = "../unstract/flags", editable = true } -unstract-core = { path = "../unstract/core", editable = true } -unstract-sdk1 = { path = "../unstract/sdk1", editable = true } - -[dependency-groups] -test = [ - "pytest~=9.0.3", - "pytest-asyncio>=0.23.0", - "pytest-mock~=3.14.0", - "pytest-md-report>=0.6.2", - "python-dotenv==1.2.2", - "flask-WTF~=1.1", -] -deploy = [ - "gunicorn~=23.0", - # OpenTelemetry for tracing and profiling - "opentelemetry-distro", - "opentelemetry-exporter-otlp", -] -dev = ["poethepoet>=0.33.1", "debugpy>=1.8.14"] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/unstract"] - -[tool.pytest.ini_options] -asyncio_mode = "strict" - -[tool.poe] -envfile = ".env" - -[tool.poe.tasks.prompt-service] -cmd = "./entrypoint.sh" -help = "Runs the Unstract prompt service (Gunicorn)" - -[tool.poe.tasks.prompt-service-flask] -cmd = "uv run flask --app src/unstract/prompt_service/run.py run --port 3003" -help = "Runs the Unstract prompt service (Flask)" diff --git a/prompt-service/sample.env b/prompt-service/sample.env deleted file mode 100644 index 01d021f9d1..0000000000 --- a/prompt-service/sample.env +++ /dev/null @@ -1,79 +0,0 @@ -# Backend DB -PG_BE_HOST=unstract-db -PG_BE_PORT=5432 -PG_BE_USERNAME=unstract_dev -PG_BE_PASSWORD=unstract_pass -PG_BE_DATABASE=unstract_db -DB_SCHEMA="unstract" - -# Redis -REDIS_HOST="unstract-redis" -REDIS_PORT=6379 -REDIS_PASSWORD="" -REDIS_USER=default - -# Redis Sentinel mode - set to True for HA deployments (K8s only) -# When True: point REDIS_HOST to Sentinel K8s service DNS, REDIS_PORT to 26379 -# REDIS_PASSWORD is reused for Sentinel auth. -REDIS_SENTINEL_MODE=False -REDIS_SENTINEL_MASTER_NAME=mymaster - -# Logging -LOG_LEVEL=INFO - - -### Env from `unstract-core` ### -# Celery for PublishLogs -CELERY_BROKER_BASE_URL="amqp://unstract-rabbitmq:5672//" -CELERY_BROKER_USER=admin -CELERY_BROKER_PASS=password -# Logs Expiry of 24 hours -LOGS_EXPIRATION_TIME_IN_SECOND=86400 - - -### Env from `unstract-flags` ### -# Feature Flags -EVALUATION_SERVER_IP=unstract-flipt -EVALUATION_SERVER_PORT=9000 -PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python -# Flipt Service -FLIPT_SERVICE_AVAILABLE=False - - -### Env from `unstract-sdk` ### -# Platform Service -PLATFORM_SERVICE_HOST=http://unstract-platform-service -PLATFORM_SERVICE_PORT=3001 - -# X2Text Service -X2TEXT_HOST=http://unstract-x2text-service -X2TEXT_PORT=3004 - -# Remote storage related envs -PERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}' -TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}' -REMOTE_PROMPT_STUDIO_FILE_PATH="unstract/prompt-studio-data/" - -# Timeout for LLMW (v2) extraction -ADAPTER_LLMW_WAIT_TIMEOUT=900 # 15 mins - -# Control async extraction of LLMWhisperer (v1) -# Time in seconds to wait before polling LLMWhisperer's status API -ADAPTER_LLMW_POLL_INTERVAL=30 -# Total number of times to poll the status API. -# 500 mins to allow 1500 (max pages limit) * 20 (approx time in sec to process a page) -ADAPTER_LLMW_MAX_POLLS=1000 -# Number of times to retry the /whisper-status API before failing the extraction -ADAPTER_LLMW_STATUS_RETRIES=5 -# Retry backoff for LLMWhisperer client (v2) -# Max retry attempts for transient HTTP errors (429, 5xx). Set 0 to disable. -ADAPTER_LLMW_MAX_RETRIES=3 -# Min backoff wait in seconds between retries -ADAPTER_LLMW_RETRY_MIN_WAIT=1.0 -# Max backoff wait in seconds between retries -ADAPTER_LLMW_RETRY_MAX_WAIT=60.0 - -### Env for Rentroll Service ### -# Rentroll Service -RENTROLL_SERVICE_HOST=http://unstract-rentroll-service -RENTROLL_SERVICE_PORT=5003 diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py deleted file mode 100644 index 5edfb829ed..0000000000 --- a/prompt-service/src/unstract/prompt_service/config.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging -import warnings -from os import environ as env -from pathlib import Path - -from dotenv import load_dotenv -from flask import Flask - -from unstract.core.flask import ( - PluginManager, - register_error_handlers, - register_request_id_middleware, -) -from unstract.core.flask.logging import setup_logging -from unstract.prompt_service.controllers import api -from unstract.sdk1.constants import LogLevel - -load_dotenv() - - -def create_app() -> Flask: - """Creates and configures the Flask application.""" - log_level = env.get("LOG_LEVEL", LogLevel.INFO.value).upper() - setup_logging(log_level) - - # Suppress OpenTelemetry EventLogger LogRecord deprecation warning - # This is a bug in OpenTelemetry SDK 1.37.0 where EventLogger.emit() still uses - # deprecated trace_id/span_id/trace_flags parameters instead of context parameter. - # See: https://github.com/open-telemetry/opentelemetry-python/issues/4328 - # TODO: Remove this suppression once OpenTelemetry fixes EventLogger.emit() - warnings.filterwarnings( - "ignore", - message="LogRecord init with.*trace_id.*span_id.*trace_flags.*deprecated", - category=DeprecationWarning, - module="opentelemetry.sdk._events", - ) - - log_level = getattr(logging, log_level, logging.INFO) - app = Flask("prompt-service") - app.logger.setLevel(log_level) - app.logger.info("Initializing Flask application...") - - # Load plugins - plugins_dir = Path(__file__).parent / "plugins" - plugins_pkg = "unstract.prompt_service.plugins" - manager = PluginManager(app, plugins_dir, plugins_pkg) - manager.load_plugins() - - register_request_id_middleware(app) - register_error_handlers(app) - app.register_blueprint(api) - - app.logger.info("Flask app created successfully.") - return app diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py deleted file mode 100644 index 9eddab8423..0000000000 --- a/prompt-service/src/unstract/prompt_service/constants.py +++ /dev/null @@ -1,203 +0,0 @@ -from enum import Enum - - -class PromptServiceConstants: - """Constants used in the prompt service.""" - - WORD = "word" - SYNONYMS = "synonyms" - OUTPUTS = "outputs" - TOOL_ID = "tool_id" - RUN_ID = "run_id" - EXECUTION_ID = "execution_id" - FILE_NAME = "file_name" - FILE_HASH = "file_hash" - NAME = "name" - ACTIVE = "active" - PROMPT = "prompt" - CHUNK_SIZE = "chunk-size" - PROMPTX = "promptx" - VECTOR_DB = "vector-db" - EMBEDDING = "embedding" - X2TEXT_ADAPTER = "x2text_adapter" - CHUNK_OVERLAP = "chunk-overlap" - LLM = "llm" - IS_ASSERT = "is_assert" - ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt" - RETRIEVAL_STRATEGY = "retrieval-strategy" - TYPE = "type" - NUMBER = "number" - EMAIL = "email" - DATE = "date" - BOOLEAN = "boolean" - JSON = "json" - PREAMBLE = "preamble" - SIMILARITY_TOP_K = "similarity-top-k" - PROMPT_TOKENS = "prompt_tokens" - COMPLETION_TOKENS = "completion_tokens" - TOTAL_TOKENS = "total_tokens" - RESPONSE = "response" - POSTAMBLE = "postamble" - GRAMMAR = "grammar" - PLATFORM_SERVICE_API_KEY = "PLATFORM_SERVICE_API_KEY" - EMBEDDING_SUFFIX = "embedding_suffix" - EVAL_SETTINGS = "eval_settings" - EVAL_SETTINGS_EVALUATE = "evaluate" - EVAL_SETTINGS_MONITOR_LLM = "monitor_llm" - EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed" - TOOL_SETTINGS = "tool_settings" - LOG_EVENTS_ID = "log_events_id" - CHALLENGE_LLM = "challenge_llm" - CHALLENGE = "challenge" - ENABLE_CHALLENGE = "enable_challenge" - EXTRACTION = "extraction" - SUMMARIZE = "summarize" - SINGLE_PASS_EXTRACTION = "single-pass-extraction" - SIMPLE_PROMPT_STUDIO = "simple-prompt-studio" - LLM_USAGE_REASON = "llm_usage_reason" - METADATA = "metadata" - OUTPUT = "output" - CONTEXT = "context" - INCLUDE_METADATA = "include_metadata" - TABLE = "table" - TABLE_SETTINGS = "table_settings" - EPILOGUE = "epilogue" - PLATFORM_POSTAMBLE = "platform_postamble" - WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble" - HIGHLIGHT_DATA_PLUGIN = "highlight-data" - SUMMARIZE_AS_SOURCE = "summarize_as_source" - VARIABLE_MAP = "variable_map" - RECORD = "record" - CUSTOM_DATA = "custom_data" - TEXT = "text" - ENABLE_HIGHLIGHT = "enable_highlight" - ENABLE_WORD_CONFIDENCE = "enable_word_confidence" - FILE_PATH = "file_path" - HIGHLIGHT_DATA = "highlight_data" - CONFIDENCE_DATA = "confidence_data" - WORD_CONFIDENCE_DATA = "word_confidence_data" - REQUIRED_FIELDS = "required_fields" - REQUIRED = "required" - EXECUTION_SOURCE = "execution_source" - METRICS = "metrics" - CAPTURE_METRICS = "capture_metrics" - LINE_ITEM = "line-item" - LINE_NUMBERS = "line_numbers" - WHISPER_HASH = "whisper_hash" - PAID_FEATURE_MSG = ( - "It is a cloud / enterprise feature. If you have purchased a plan and still " - "face this issue, please contact support" - ) - NO_CONTEXT_ERROR = ( - "Couldn't fetch context from vector DB. " - "This happens usually due to a delay by the Vector DB " - "provider to confirm writes to DB. " - "Please try again after some time" - ) - COMBINED_PROMPT = "combined_prompt" - TOOL = "tool" - JSON_POSTAMBLE = "JSON_POSTAMBLE" - DEFAULT_JSON_POSTAMBLE = "Wrap the final JSON result inbetween §§§ like below example:\n§§§\n\n§§§" - DOCUMENT_TYPE = "document_type" - # Webhook postprocessing settings - ENABLE_POSTPROCESSING_WEBHOOK = "enable_postprocessing_webhook" - POSTPROCESSING_WEBHOOK_URL = "postprocessing_webhook_url" - - -class RunLevel(Enum): - """Different stages of prompt execution. - - Comprises of prompt run and response evaluation stages. - """ - - RUN = "RUN" - EVAL = "EVAL" - CHALLENGE = "CHALLENGE" - TABLE_EXTRACTION = "TABLE_EXTRACTION" - - -class DBTableV2: - """Database tables.""" - - ORGANIZATION = "organization" - ADAPTER_INSTANCE = "adapter_instance" - PROMPT_STUDIO_REGISTRY = "prompt_studio_registry" - PLATFORM_KEY = "platform_key" - TOKEN_USAGE = "usage" - - -class FileStorageKeys: - """File storage keys.""" - - PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE" - TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE" - - -class FileStorageType(Enum): - """File storage type.""" - - PERMANENT = "permanent" - TEMPORARY = "temporary" - - -class ExecutionSource(Enum): - """Execution source.""" - - IDE = "ide" - TOOL = "tool" - - -class VariableType(str, Enum): - """Type of variable.""" - - STATIC = "STATIC" - DYNAMIC = "DYNAMIC" - CUSTOM_DATA = "CUSTOM_DATA" - - -class RetrievalStrategy(str, Enum): - """Available retrieval strategies for prompt service.""" - - SIMPLE = "simple" - SUBQUESTION = "subquestion" - FUSION = "fusion" - RECURSIVE = "recursive" - ROUTER = "router" - KEYWORD_TABLE = "keyword_table" - AUTOMERGING = "automerging" - - -class VariableConstants: - """Constants for variable extraction.""" - - VARIABLE_REGEX = "{{(.+?)}}" - DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]" - DYNAMIC_VARIABLE_URL_REGEX = ( - r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»" - "'']))" - ) # noqa: E501 - CUSTOM_DATA_VARIABLE_REGEX = r"custom_data\.([a-zA-Z0-9_\.]+)" - - -class IndexingConstants: - TOOL_ID = "tool_id" - EMBEDDING_INSTANCE_ID = "embedding_instance_id" - VECTOR_DB_INSTANCE_ID = "vector_db_instance_id" - X2TEXT_INSTANCE_ID = "x2text_instance_id" - FILE_PATH = "file_path" - CHUNK_SIZE = "chunk_size" - CHUNK_OVERLAP = "chunk_overlap" - REINDEX = "reindex" - FILE_HASH = "file_hash" - OUTPUT_FILE_PATH = "output_file_path" - ENABLE_HIGHLIGHT = "enable_highlight" - ENABLE_WORD_CONFIDENCE = "enable_word_confidence" - USAGE_KWARGS = "usage_kwargs" - PROCESS_TEXT = "process_text" - EXTRACTED_TEXT = "extracted_text" - TAGS = "tags" - EXECUTION_SOURCE = "execution_source" - DOC_ID = "doc_id" - TOOL_EXECUTION_METATADA = "tool_execution_metadata" - EXECUTION_DATA_DIR = "execution_data_dir" - METADATA_FILE = "METADATA.json" diff --git a/prompt-service/src/unstract/prompt_service/controllers/__init__.py b/prompt-service/src/unstract/prompt_service/controllers/__init__.py deleted file mode 100644 index e6b149de21..0000000000 --- a/prompt-service/src/unstract/prompt_service/controllers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from flask import Blueprint - -from .answer_prompt import answer_prompt_bp -from .extraction import extraction_bp -from .health import health_bp -from .indexing import indexing_bp - -api = Blueprint("api", __name__) - -# Register blueprint to the API Blueprint -api.register_blueprint(health_bp) -api.register_blueprint(answer_prompt_bp) -api.register_blueprint(indexing_bp) -api.register_blueprint(extraction_bp) diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py deleted file mode 100644 index fdf0deaae1..0000000000 --- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py +++ /dev/null @@ -1,709 +0,0 @@ -"""Published API Controller""" - -from typing import Any - -from flask import Blueprint, request -from flask import current_app as app - -from unstract.core.flask import PluginManager -from unstract.core.flask.exceptions import APIError -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.constants import RetrievalStrategy, RunLevel -from unstract.prompt_service.exceptions import BadRequest -from unstract.prompt_service.helpers.auth import AuthHelper -from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool -from unstract.prompt_service.helpers.usage import UsageHelper -from unstract.prompt_service.services.answer_prompt import AnswerPromptService -from unstract.prompt_service.services.rentrolls_extractor.interface import ( - RentRollExtractor, -) -from unstract.prompt_service.services.retrieval import RetrievalService -from unstract.prompt_service.services.variable_replacement import ( - VariableReplacementService, -) -from unstract.prompt_service.utils.file_utils import FileUtils -from unstract.prompt_service.utils.log import publish_log -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.embedding import EmbeddingCompat -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.index import Index -from unstract.sdk1.llm import LLM -from unstract.sdk1.platform import PlatformHelper as ToolAdapter -from unstract.sdk1.vector_db import VectorDB - -answer_prompt_bp = Blueprint("answer-prompt", __name__) - - -@AuthHelper.auth_required -@answer_prompt_bp.route("/answer-prompt", methods=["POST"]) -def prompt_processor() -> Any: - platform_key = AuthHelper.get_token_from_auth_header(request) - payload: dict[Any, Any] = request.json - if not payload: - raise BadRequest - tool_settings = payload.get(PSKeys.TOOL_SETTINGS, {}) - enable_challenge = tool_settings.get(PSKeys.ENABLE_CHALLENGE, False) - # Rename "outputs" to "prompts" in payload - prompts = payload.get(PSKeys.OUTPUTS, []) - tool_id: str = payload.get(PSKeys.TOOL_ID, "") - run_id: str = payload.get(PSKeys.RUN_ID, "") - execution_id: str = payload.get(PSKeys.EXECUTION_ID, "") - file_hash = payload.get(PSKeys.FILE_HASH) - file_path = payload.get(PSKeys.FILE_PATH) - doc_name = str(payload.get(PSKeys.FILE_NAME, "")) - log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "") - custom_data: dict[str, Any] = payload.get(PSKeys.CUSTOM_DATA, {}) - structured_output: dict[str, Any] = {} - metadata: dict[str, Any] = { - PSKeys.RUN_ID: run_id, - PSKeys.FILE_NAME: doc_name, - PSKeys.CONTEXT: {}, - PSKeys.REQUIRED_FIELDS: {}, - } - metrics: dict = {} - variable_names: list[str] = [] - # Identifier for source of invocation - execution_source = payload.get(PSKeys.EXECUTION_SOURCE, "") - context_retrieval_metrics = {} - publish_log( - log_events_id, - {"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name}, - LogLevel.DEBUG, - RunLevel.RUN, - f"Preparing to execute '{len(prompts)}' prompt(s)", - ) - # Rename "output" to "prompt" - for output in prompts: # type:ignore - variable_names.append(output[PSKeys.NAME]) - metadata[PSKeys.REQUIRED_FIELDS][output[PSKeys.NAME]] = output.get( - PSKeys.REQUIRED, None - ) - - for output in prompts: # type:ignore - prompt_name = output[PSKeys.NAME] - prompt_text = output[PSKeys.PROMPT] - chunk_size = output[PSKeys.CHUNK_SIZE] - app.logger.info(f"[{tool_id}] chunk size: {chunk_size}") - util = PromptServiceBaseTool(platform_key=platform_key) - index = Index(tool=util, run_id=run_id, capture_metrics=True) - if VariableReplacementService.is_variables_present(prompt_text=prompt_text): - # Determine if this is from IDE (Prompt Studio) or API deployment - is_ide = execution_source == "ide" - prompt_text = VariableReplacementService.replace_variables_in_prompt( - prompt=output, - structured_output=structured_output, - log_events_id=log_events_id, - tool_id=tool_id, - prompt_name=prompt_name, - doc_name=doc_name, - custom_data=custom_data, - is_ide=is_ide, - ) - - app.logger.info(f"[{tool_id}] Executing prompt: '{prompt_name}'") - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.RUN, - "Executing prompt", - ) - - # Finding and replacing the variables in the prompt - # The variables are in the form %variable_name% - - output[PSKeys.PROMPTX] = AnswerPromptService.extract_variable( - structured_output, variable_names, output, prompt_text - ) - - doc_id = index.generate_index_key( - file_hash=file_hash, - vector_db=output[PSKeys.VECTOR_DB], - embedding=output[PSKeys.EMBEDDING], - x2text=output[PSKeys.X2TEXT_ADAPTER], - chunk_size=str(output[PSKeys.CHUNK_SIZE]), - chunk_overlap=str(output[PSKeys.CHUNK_OVERLAP]), - ) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.RUN, - "Retrieved document ID", - ) - - try: - usage_kwargs = {"run_id": run_id, "execution_id": execution_id} - llm = LLM( - adapter_instance_id=output[PSKeys.LLM], - tool=util, - usage_kwargs={ - **usage_kwargs, - PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION, - }, - capture_metrics=True, - ) - - # Only create embedding and vector_db if chunk_size > 0 - # When chunk_size is 0, we read the complete file without embeddings - embedding = None - vector_db = None - if chunk_size > 0: - embedding = EmbeddingCompat( - adapter_instance_id=output[PSKeys.EMBEDDING], - tool=util, - kwargs={ - **usage_kwargs, - }, - ) - - vector_db = VectorDB( - tool=util, - adapter_instance_id=output[PSKeys.VECTOR_DB], - embedding=embedding, - ) - except SdkError as e: - msg = f"Couldn't fetch adapter. {e}" - app.logger.error(msg) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.ERROR, - RunLevel.RUN, - "Unable to obtain LLM / embedding / vectorDB", - ) - # Preserve the status code from the SDK error - status_code = getattr(e, "status_code", None) or 500 - raise APIError(message=msg, code=status_code) from e - - if output[PSKeys.TYPE] == PSKeys.TABLE: - adapter_parent_data = ToolAdapter.get_adapter_config(util, output[PSKeys.LLM]) - llm_config = adapter_parent_data.get("adapter_metadata") - adapter_id = adapter_parent_data.get("adapter_id") - adapter_prefix = adapter_id.split("|")[0] - llm_provider = llm.kwargs.get("provider") - llm_adapter_config = {"adapter_id": adapter_prefix} - llm_adapter_config["provider"] = llm_provider - if adapter_prefix == "azureopenai": - llm_adapter_config["model"] = llm_config.get("model") - llm_adapter_config["api_key"] = llm_config.get("api_key") - llm_adapter_config["api_base"] = llm_config.get("azure_endpoint") - llm_adapter_config["max_retries"] = llm_config.get("max_retries") - llm_adapter_config["timeout"] = llm_config.get("timeout") - llm_adapter_config["deployment"] = llm_config.get("deployment_name") - llm_adapter_config["api_version"] = llm_config.get("api_version") - if adapter_prefix == "openai": - llm_adapter_config["model"] = llm_config.get("model") - llm_adapter_config["api_key"] = llm_config.get("api_key") - llm_adapter_config["api_base"] = llm_config.get("api_base") - llm_adapter_config["max_retries"] = llm_config.get("max_retries") - llm_adapter_config["timeout"] = llm_config.get("timeout") - if adapter_prefix == "anthropic": - llm_adapter_config["model"] = llm_config.get("model") - llm_adapter_config["api_key"] = llm_config.get("api_key") - llm_adapter_config["max_retries"] = llm_config.get("max_retries") - llm_adapter_config["timeout"] = llm_config.get("timeout") - if adapter_prefix == "bedrock": - llm_adapter_config["model"] = llm_config.get("model") - llm_adapter_config["aws_access_key_id"] = llm_config.get( - "aws_access_key_id" - ) - llm_adapter_config["aws_secret_access_key"] = llm_config.get( - "aws_secret_access_key" - ) - llm_adapter_config["max_retries"] = llm_config.get("max_retries") - llm_adapter_config["budget_tokens"] = llm_config.get("budget_tokens") - llm_adapter_config["max_tokens"] = llm_config.get("max_tokens") - llm_adapter_config["region_name"] = llm_config.get("region_name") - llm_adapter_config["timeout"] = llm_config.get("timeout") - table_settings = output[PSKeys.TABLE_SETTINGS] - document_type: str = table_settings.get(PSKeys.DOCUMENT_TYPE) - app.logger.info("Document type: %s", document_type) - if document_type == "rent_rolls": - # Create RentRollExtractor instance and process the extraction - extractor = RentRollExtractor() - fs_instance = FileUtils.get_fs_instance(execution_source=execution_source) - extracted_data = fs_instance.read(path=file_path, mode="r") - app.logger.info("Starting rent roll extraction run...") - try: - extraction_result = extractor.process( - extractor_settings=output, - extracted_data=extracted_data, - llm_config=llm_adapter_config, - schema=prompt_text, - ) - - # Update structured output with the extraction result - structured_output[output[PSKeys.NAME]] = extraction_result["data"] - response = { - PSKeys.OUTPUT: structured_output, - PSKeys.METADATA: extraction_result["metrics"], - PSKeys.METRICS: extraction_result["metrics"], - } - - # Track token usage by sending to the audit service - try: - from unstract.sdk1.utils.token_counter import TokenCounter - - # Get metrics from the extraction result - metrics = extraction_result.get("metrics", {}) - - # Create token counter adapter from metrics - token_usage = metrics.get("token_usage") or {} - token_counter = TokenCounter( - input_tokens=token_usage.get("prompt_tokens"), - output_tokens=token_usage.get("completion_tokens"), - ) - - # Extract model name from llm config - # Extract model name from llm config - model_info = metrics.get("model_info") or {} - model_name = model_info.get("model_name") - provider = model_info.get("provider") - - # Prepare usage data for audit - kwargs = { - "workflow_id": "", # Not applicable for rent rolls - "execution_id": "", # Not applicable for rent rolls - "adapter_instance_id": adapter_id, - "run_id": str(run_id), - "provider": provider, - "llm_usage_reason": "extraction", - } - - # Push usage data to audit service - UsageHelper.push_usage_data( - event_type="extraction", - kwargs=kwargs, - platform_api_key=platform_key, - token_counter=token_counter, - model_name=model_name, - ) - app.logger.info( - "Successfully pushed token usage data to audit service" - ) - except Exception as e: - app.logger.error(f"Failed to track token usage: {str(e)}") - app.logger.info("Rent roll extraction completed successfully") - return response - except Exception as e: - app.logger.error(f"Failed to process rent roll: {str(e)}") - raise e - if document_type != "rent_rolls": - try: - structured_output = AnswerPromptService.extract_table( - output=output, - structured_output=structured_output, - llm=llm, - execution_source=execution_source, - prompt=prompt_text, - ) - metadata = UsageHelper.query_usage_metadata( - token=platform_key, metadata=metadata - ) - response = { - PSKeys.METADATA: metadata, - PSKeys.OUTPUT: structured_output, - PSKeys.METRICS: metrics, - } - return response - except APIError as api_error: - app.logger.error( - "Failed to extract table for the prompt %s: %s", - output[PSKeys.NAME], - str(api_error), - ) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.ERROR, - RunLevel.TABLE_EXTRACTION, - "Error while extracting table for the prompt", - ) - raise api_error - elif output[PSKeys.TYPE] == PSKeys.LINE_ITEM: - try: - structured_output = AnswerPromptService.extract_line_item( - tool_settings=tool_settings, - output=output, - structured_output=structured_output, - llm=llm, - file_path=file_path, - metadata=metadata, - execution_source=execution_source, - ) - continue - except APIError as e: - app.logger.error( - "Failed to extract line-item for the prompt %s: %s", - output[PSKeys.NAME], - str(e), - ) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.ERROR, - RunLevel.RUN, - "Error while extracting line-item for the prompt", - ) - raise e - - try: - answer = "NA" - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.INFO, - RunLevel.RUN, - "Retrieving context from adapter", - ) - - retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY) - - # Get all valid retrieval strategies - valid_strategies = {strategy.value for strategy in RetrievalStrategy} - - if retrieval_strategy in valid_strategies: - app.logger.info(f"[{tool_id}] Performing retrieval for : {file_path}") - answer, context = RetrievalService.perform_retrieval( - tool_settings=tool_settings, - output=output, - doc_id=doc_id, - llm=llm, - vector_db=vector_db, # This will be None when chunk_size is 0 - retrieval_type=retrieval_strategy, - metadata=metadata, - chunk_size=chunk_size, - execution_source=execution_source, - file_path=file_path, - context_retrieval_metrics=context_retrieval_metrics, - ) - metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = context - else: - app.logger.info( - "Invalid retrieval strategy passed: %s", - retrieval_strategy, - ) - - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.RUN, - "Retrieved context from adapter", - ) - - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.INFO, - RunLevel.RUN, - f"Processing prompt type: {output[PSKeys.TYPE]}", - ) - - if output[PSKeys.TYPE] == PSKeys.NUMBER: - if answer.lower() == "na": - structured_output[output[PSKeys.NAME]] = None - else: - # Extract these prompts as constants after pkging - prompt = f"Extract the number from the following \ - text:\n{answer}\n\nOutput just the number. \ - If the number is expressed in millions \ - or thousands, expand the number to its numeric value \ - The number should be directly assignable\ - to a numeric variable.\ - It should not have any commas, \ - percentages or other grouping \ - characters. No explanation is required.\ - If you cannot extract the number, output 0." - answer = AnswerPromptService.run_completion( - llm=llm, - prompt=prompt, - ) - try: - structured_output[output[PSKeys.NAME]] = float(answer) - except Exception as e: - app.logger.info( - f"Error parsing response (to numeric, float): {e}", - LogLevel.ERROR, - ) - structured_output[output[PSKeys.NAME]] = None - elif output[PSKeys.TYPE] == PSKeys.EMAIL: - if answer.lower() == "na": - structured_output[output[PSKeys.NAME]] = None - else: - prompt = f'Extract the email from the following text:\n{answer}\n\nOutput just the email. \ - The email should be directly assignable to a string variable. \ - No explanation is required. If you cannot extract the email, output "NA".' # noqa - answer = AnswerPromptService.run_completion( - llm=llm, - prompt=prompt, - ) - structured_output[output[PSKeys.NAME]] = answer - elif output[PSKeys.TYPE] == PSKeys.DATE: - if answer.lower() == "na": - structured_output[output[PSKeys.NAME]] = None - else: - prompt = f'Extract the date from the following text:\n{answer}\n\nOutput just the date.\ - The date should be in ISO date time format. No explanation is required. \ - The date should be directly assignable to a date variable. \ - If you cannot convert the string into a date, output "NA".' # noqa - answer = AnswerPromptService.run_completion( - llm=llm, - prompt=prompt, - ) - structured_output[output[PSKeys.NAME]] = answer - - elif output[PSKeys.TYPE] == PSKeys.BOOLEAN: - if answer.lower() == "na": - structured_output[output[PSKeys.NAME]] = None - else: - prompt = f'Extract yes/no from the following text:\n{answer}\n\n\ - Output in single word.\ - If the context is trying to convey that the answer is true, \ - then return "yes", else return "no".' - answer = AnswerPromptService.run_completion( - llm=llm, - prompt=prompt, - ) - if answer.lower() == "yes": - structured_output[output[PSKeys.NAME]] = True - else: - structured_output[output[PSKeys.NAME]] = False - elif output[PSKeys.TYPE] == PSKeys.JSON: - AnswerPromptService.handle_json( - answer=answer, - structured_output=structured_output, - output=output, - log_events_id=log_events_id, - tool_id=tool_id, - doc_name=doc_name, - llm=llm, - enable_highlight=tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False), - enable_word_confidence=tool_settings.get( - PSKeys.ENABLE_WORD_CONFIDENCE, False - ), - execution_source=execution_source, - metadata=metadata, - file_path=file_path, - ) - else: - structured_output[output[PSKeys.NAME]] = answer - - # If there is a trailing '\n' remove it - if isinstance(structured_output[output[PSKeys.NAME]], str): - structured_output[output[PSKeys.NAME]] = structured_output[ - output[PSKeys.NAME] - ].rstrip("\n") - - # Challenge condition - if enable_challenge: - challenge_plugin: dict[str, Any] = PluginManager().get_plugin( - PSKeys.CHALLENGE - ) - try: - if challenge_plugin: - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.INFO, - RunLevel.CHALLENGE, - "Challenging response", - ) - challenge_llm = LLM( - adapter_instance_id=tool_settings[PSKeys.CHALLENGE_LLM], - tool=util, - usage_kwargs={ - **usage_kwargs, - PSKeys.LLM_USAGE_REASON: PSKeys.CHALLENGE, - }, - capture_metrics=True, - ) - challenge = challenge_plugin["entrypoint_cls"]( - llm=llm, - challenge_llm=challenge_llm, - run_id=run_id, - context="\n".join(context), - tool_settings=tool_settings, - output=output, - structured_output=structured_output, - platform_key=platform_key, - metadata=metadata, - ) - # Will inline replace the structured output passed. - challenge.run() - else: - app.logger.info( - "No challenge plugin found to evaluate prompt: %s", - output[PSKeys.NAME], - ) - except challenge_plugin["exception_cls"] as e: - app.logger.error( - "Failed to challenge prompt %s: %s", - output[PSKeys.NAME], - str(e), - ) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.ERROR, - RunLevel.CHALLENGE, - "Error while challenging response", - ) - - # - # Evaluate the prompt. - # - if ( - PSKeys.EVAL_SETTINGS in output - and output[PSKeys.EVAL_SETTINGS][PSKeys.EVAL_SETTINGS_EVALUATE] - ): - eval_plugin: dict[str, Any] = PluginManager().get_plugin("evaluation") - if eval_plugin: - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.INFO, - RunLevel.EVAL, - "Evaluating response", - ) - try: - evaluator = eval_plugin["entrypoint_cls"]( - "", - "".join(context), - "", - "", - output, - structured_output, - platform_key, - ) - # Will inline replace the structured output passed. - evaluator.run() - except eval_plugin["exception_cls"] as e: - app.logger.error( - f"Failed to evaluate prompt {output[PSKeys.NAME]}: {str(e)}" - ) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.ERROR, - RunLevel.EVAL, - "Error while evaluation", - ) - else: - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.EVAL, - "Evaluation completed", - ) - else: - app.logger.info( - f"No eval plugin found to evaluate prompt: {output[PSKeys.NAME]}" # noqa: E501 - ) - finally: - challenge_metrics = ( - {f"{challenge_llm.get_usage_reason()}_llm": challenge_llm.get_metrics()} - if enable_challenge and challenge_llm - else {} - ) - metrics.setdefault(prompt_name, {}).update( - { - "context_retrieval": context_retrieval_metrics.get(prompt_name, {}), - f"{llm.get_usage_reason()}_llm": llm.get_metrics(), - **challenge_metrics, - } - ) - # Only close vector_db if it was created (chunk_size > 0) - if vector_db: - vector_db.close() - publish_log( - log_events_id, - {"tool_id": tool_id, "doc_name": doc_name}, - LogLevel.INFO, - RunLevel.RUN, - "Sanitizing null values", - ) - for k, v in structured_output.items(): - if isinstance(v, str) and v.lower() == "na": - structured_output[k] = None - elif isinstance(v, list): - for i in range(len(v)): - if isinstance(v[i], str) and v[i].lower() == "na": - v[i] = None - elif isinstance(v[i], dict): - for k1, v1 in v[i].items(): - if isinstance(v1, str) and v1.lower() == "na": - v[i][k1] = None - elif isinstance(v, dict): - for k1, v1 in v.items(): - if isinstance(v1, str) and v1.lower() == "na": - v[k1] = None - - publish_log( - log_events_id, - {"tool_id": tool_id, "doc_name": doc_name}, - LogLevel.INFO, - RunLevel.RUN, - "Execution complete", - ) - metadata = UsageHelper.query_usage_metadata(token=platform_key, metadata=metadata) - response = { - PSKeys.METADATA: metadata, - PSKeys.OUTPUT: structured_output, - PSKeys.METRICS: metrics, - } - return response diff --git a/prompt-service/src/unstract/prompt_service/controllers/extraction.py b/prompt-service/src/unstract/prompt_service/controllers/extraction.py deleted file mode 100644 index 516894f429..0000000000 --- a/prompt-service/src/unstract/prompt_service/controllers/extraction.py +++ /dev/null @@ -1,53 +0,0 @@ -from typing import Any - -from flask import Blueprint, request - -from unstract.prompt_service.constants import IndexingConstants as IKeys -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.helpers.auth import AuthHelper -from unstract.prompt_service.services.extraction import ExtractionService -from unstract.prompt_service.utils.request import validate_request_payload - -extraction_bp = Blueprint("extract", __name__) - -REQUIRED_FIELDS = [ - "x2text_instance_id", - "file_path", - "execution_source", - "run_id", -] - - -@AuthHelper.auth_required -@extraction_bp.route("/extract", methods=["POST"]) -def extract() -> Any: - platform_key = AuthHelper.get_token_from_auth_header(request) - payload: dict[Any, Any] = request.json - validate_request_payload(payload, REQUIRED_FIELDS) - - x2text_instance_id: str = payload.get(IKeys.X2TEXT_INSTANCE_ID, "") - file_path: str = payload.get(IKeys.FILE_PATH, "") - output_file_path: str | None = payload.get(IKeys.OUTPUT_FILE_PATH, "") - enable_highlight: bool = payload.get(IKeys.ENABLE_HIGHLIGHT, False) - usage_kwargs: dict[Any, Any] = payload.get(IKeys.USAGE_KWARGS, {}) - run_id: str = payload.get(PSKeys.RUN_ID, "") - execution_source = payload.get(IKeys.EXECUTION_SOURCE, None) - tags: str = payload.get(IKeys.TAGS, "") - tool_exec_metadata = payload.get(IKeys.TOOL_EXECUTION_METATADA, {}) - execution_run_data_folder = payload.get(IKeys.EXECUTION_DATA_DIR, "") - - extracted_text = ExtractionService.perform_extraction( - file_path=file_path, - x2text_instance_id=x2text_instance_id, - output_file_path=output_file_path, - enable_highlight=enable_highlight, - usage_kwargs=usage_kwargs, - run_id=run_id, - execution_source=execution_source, - platform_key=platform_key, - tags=tags, - tool_exec_metadata=tool_exec_metadata, - execution_run_data_folder=execution_run_data_folder, - ) - response = {IKeys.EXTRACTED_TEXT: extracted_text} - return response diff --git a/prompt-service/src/unstract/prompt_service/controllers/health.py b/prompt-service/src/unstract/prompt_service/controllers/health.py deleted file mode 100644 index 59cc428210..0000000000 --- a/prompt-service/src/unstract/prompt_service/controllers/health.py +++ /dev/null @@ -1,8 +0,0 @@ -from flask import Blueprint - -health_bp = Blueprint("health", __name__) - - -@health_bp.route("/health", methods=["GET"]) -def health_check() -> str: - return "OK" diff --git a/prompt-service/src/unstract/prompt_service/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.py deleted file mode 100644 index f9416a9c92..0000000000 --- a/prompt-service/src/unstract/prompt_service/controllers/indexing.py +++ /dev/null @@ -1,102 +0,0 @@ -import logging -from typing import Any - -from flask import Blueprint, request - -from unstract.prompt_service.constants import IndexingConstants as IKeys -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.dto import ( - ChunkingConfig, - FileInfo, - InstanceIdentifiers, - ProcessingOptions, -) -from unstract.prompt_service.helpers.auth import AuthHelper -from unstract.prompt_service.services.indexing import IndexingService -from unstract.prompt_service.utils.request import validate_request_payload - -indexing_bp = Blueprint("index", __name__) -logger = logging.getLogger(__name__) - -REQUIRED_FIELDS = [ - "tool_id", - "extracted_text", - "embedding_instance_id", - "vector_db_instance_id", - "x2text_instance_id", - "file_path", - "chunk_size", - "chunk_overlap", - "execution_source", - "run_id", -] - - -@AuthHelper.auth_required -@indexing_bp.route("/index", methods=["POST"]) -def index() -> Any: - """Endpoint for indexing documents into the vector database. - - This API accepts a JSON payload containing document details, processes the - document, and stores it in the vector database for retrieval. - - Raises: - BadRequest: If the request payload is missing or invalid. - - Returns: - str: doc_id - """ - platform_key = AuthHelper.get_token_from_auth_header(request) - payload: dict[Any, Any] = request.json - validate_request_payload(payload, REQUIRED_FIELDS) - - tool_id: str = payload.get(IKeys.TOOL_ID, "") - embedding_instance_id: str = payload.get(IKeys.EMBEDDING_INSTANCE_ID, "") - vector_db_instance_id: str = payload.get(IKeys.VECTOR_DB_INSTANCE_ID, "") - x2text_instance_id: str = payload.get(IKeys.X2TEXT_INSTANCE_ID, "") - file_path: str = payload.get(IKeys.FILE_PATH, "") - file_hash: str | None = payload.get(IKeys.FILE_HASH) - chunk_size: int = payload.get(IKeys.CHUNK_SIZE, 512) # Default chunk size - chunk_overlap: int = payload.get(IKeys.CHUNK_OVERLAP, 128) # Default chunk overlap - reindex: bool = payload.get(IKeys.REINDEX, False) - enable_highlight: bool = payload.get(IKeys.ENABLE_HIGHLIGHT, False) - enable_word_confidence: bool = payload.get(IKeys.ENABLE_WORD_CONFIDENCE, False) - usage_kwargs: dict[Any, Any] = payload.get(IKeys.USAGE_KWARGS, {}) - extracted_text: str = payload.get(IKeys.EXTRACTED_TEXT, "") - tags: list[str] = payload.get(IKeys.TAGS, None) - execution_source = payload.get(IKeys.EXECUTION_SOURCE, None) - run_id: str = payload.get(PSKeys.RUN_ID, "") - - instance_identifiers = InstanceIdentifiers( - embedding_instance_id=embedding_instance_id, - vector_db_instance_id=vector_db_instance_id, - x2text_instance_id=x2text_instance_id, - tool_id=tool_id, - tags=tags, - llm_instance_id=None, - ) - - file_info = FileInfo(file_path=file_path, file_hash=file_hash) - - chunking_config = ChunkingConfig(chunk_size=chunk_size, chunk_overlap=chunk_overlap) - - processing_options = ProcessingOptions( - reindex=reindex, - enable_highlight=enable_highlight, - enable_word_confidence=enable_word_confidence, - usage_kwargs=usage_kwargs, - ) - doc_id = IndexingService.index( - chunking_config=chunking_config, - execution_source=execution_source, - run_id=run_id, - file_info=file_info, - instance_identifiers=instance_identifiers, - platform_key=platform_key, - processing_options=processing_options, - extracted_text=extracted_text, - ) - response = { - IKeys.DOC_ID: doc_id, - } - return response diff --git a/prompt-service/src/unstract/prompt_service/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.py deleted file mode 100644 index 5f1703b320..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/index_v2.py +++ /dev/null @@ -1,240 +0,0 @@ -import json -import logging -from typing import Any - -import openai -from llama_index.core import Document -from llama_index.core.vector_stores import ( - FilterOperator, - MetadataFilter, - MetadataFilters, - VectorStoreQuery, - VectorStoreQueryResult, -) - -from unstract.prompt_service.dto import ( - ChunkingConfig, - FileInfo, - InstanceIdentifiers, - ProcessingOptions, -) -from unstract.sdk1.adapters.vectordb.no_op.src.no_op_custom_vectordb import ( - NoOpCustomVectorDB, -) -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.embedding import Embedding -from unstract.sdk1.exceptions import SdkError, parse_litellm_err -from unstract.sdk1.file_storage.impl import FileStorage -from unstract.sdk1.file_storage.provider import FileStorageProvider -from unstract.sdk1.platform import PlatformHelper as ToolAdapter -from unstract.sdk1.tool.stream import StreamMixin -from unstract.sdk1.utils.common import Utils, capture_metrics -from unstract.sdk1.utils.tool import ToolUtils -from unstract.sdk1.vector_db import VectorDB - -logger = logging.getLogger(__name__) - - -class Index: - def __init__( - self, - tool: StreamMixin, - instance_identifiers: InstanceIdentifiers, - chunking_config: ChunkingConfig, - processing_options: ProcessingOptions, - run_id: str | None = None, - capture_metrics: bool = False, - ): - self.tool = tool - self._run_id = run_id - self._capture_metrics = capture_metrics - self.instance_identifiers = instance_identifiers - self.chunking_config = chunking_config - self.processing_options = processing_options - self._metrics = {} - - def generate_index_key( - self, - file_info: FileInfo, - fs: FileStorage = FileStorage(provider=FileStorageProvider.LOCAL), - ) -> str: - """Generates a unique index key based on the provided configuration, - file information, instance identifiers, and processing options. - - Args: - chunking_config : ChunkingConfig - file_info (FileInfo): Contains file-related - information such as path and hash. - instance_identifiers (InstanceIdentifiers): Identifiers for - embedding, vector DB, tool, etc. - processing_options (ProcessingOptions): Options related to reindexing, - highlighting, and processing text. - fs (FileStorage, optional): File storage for remote storage. - - Returns: - str: A unique index key used for indexing the document. - """ - if not file_info.file_path and not file_info.file_hash: - raise ValueError("One of `file_path` or `file_hash` need to be provided") - - if not file_info.file_hash: - file_hash = fs.get_hash_from_file(path=file_info.file_path) - - # Whole adapter config is used currently even though it contains some keys - # which might not be relevant to indexing. This is easier for now than - # marking certain keys of the adapter config as necessary. - vector_db_config = ToolAdapter.get_adapter_config( - self.tool, self.instance_identifiers.vector_db_instance_id - ) - embedding_config = ToolAdapter.get_adapter_config( - self.tool, self.instance_identifiers.embedding_instance_id - ) - x2text_config = ToolAdapter.get_adapter_config( - self.tool, self.instance_identifiers.x2text_instance_id - ) - Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config) - index_key = { - "file_hash": file_hash, - "vector_db_config": vector_db_config, - "embedding_config": embedding_config, - "x2text_config": x2text_config, - # Typed and hashed as strings since the final hash is persisted - # and this is required to be backward compatible - "chunk_size": str(self.chunking_config.chunk_size), - "chunk_overlap": str(self.chunking_config.chunk_overlap), - } - # JSON keys are sorted to ensure that the same key gets hashed even in - # case where the fields are reordered. - hashed_index_key = ToolUtils.hash_str(json.dumps(index_key, sort_keys=True)) - return hashed_index_key - - @capture_metrics - def is_document_indexed( - self, - doc_id: str, - embedding: Embedding, - vector_db: VectorDB, - ) -> bool: - """Checks if nodes are already present in the vector database for a - given doc_id. - - Returns: - str: The document ID. - """ - # Checking if document is already indexed against doc_id - doc_id_eq_filter = MetadataFilter.from_dict( - {"key": "doc_id", "operator": FilterOperator.EQ, "value": doc_id} - ) - filters = MetadataFilters(filters=[doc_id_eq_filter]) - q = VectorStoreQuery( - query_embedding=embedding.get_query_embedding(" "), - doc_ids=[doc_id], - filters=filters, - ) - - doc_id_found = False - try: - n: VectorStoreQueryResult = vector_db.query(query=q) - if len(n.nodes) > 0: - doc_id_found = True - self.tool.stream_log(f"Found {len(n.nodes)} nodes for {doc_id}") - else: - self.tool.stream_log(f"No nodes found for {doc_id}") - except Exception as e: - logger.warning( - f"Error querying {self.instance_identifiers.vector_db_instance_id}:" - f" {str(e)}, proceeding to index", - exc_info=True, - ) - - if doc_id_found and not self.processing_options.reindex: - self.tool.stream_log(f"File was indexed already under {doc_id}") - return doc_id_found - - return doc_id_found - - @capture_metrics - def perform_indexing( - self, - vector_db: VectorDB, - doc_id: str, - extracted_text: str, - doc_id_found: bool, - ): - if isinstance( - vector_db.get_vector_db( - adapter_instance_id=self.instance_identifiers.vector_db_instance_id, - embedding_dimension=1, - ), - (NoOpCustomVectorDB), - ): - return doc_id - - self.tool.stream_log("Indexing file...") - full_text = [ - { - "section": "full", - "text_contents": str(extracted_text), - } - ] - # Convert raw text to llama index usage Document - documents = self._prepare_documents(doc_id, full_text) - if self.processing_options.reindex and doc_id_found: - self.delete_nodes(vector_db, doc_id) - self._trigger_indexing(vector_db, documents) - return doc_id - - def _trigger_indexing(self, vector_db, documents): - self.tool.stream_log("Adding nodes to vector db...") - try: - vector_db.index_document( - documents, - chunk_size=self.chunking_config.chunk_size, - chunk_overlap=self.chunking_config.chunk_overlap, - show_progress=True, - ) - self.tool.stream_log("File has been indexed successfully") - # Handle embedding errors from litellm - except openai.OpenAIError as e: - e = parse_litellm_err(e) - raise e - except Exception as e: - self.tool.stream_log( - f"Error adding nodes to vector db: {e}", - level=LogLevel.ERROR, - ) - raise e - - def delete_nodes(self, vector_db: VectorDB, doc_id: str): - try: - vector_db.delete(ref_doc_id=doc_id) - self.tool.stream_log(f"Deleted nodes for {doc_id}") - except Exception as e: - self.tool.stream_log( - f"Error deleting nodes for {doc_id}: {e}", - level=LogLevel.ERROR, - ) - raise SdkError(f"Error deleting nodes for {doc_id}: {e}") from e - - def _prepare_documents(self, doc_id: str, full_text: Any) -> list: - documents = [] - try: - for item in full_text: - text = item["text_contents"] - document = Document( - text=text, - doc_id=doc_id, - metadata={"section": item["section"]}, - ) - document.id_ = doc_id - documents.append(document) - self.tool.stream_log(f"Number of documents: {len(documents)}") - return documents - except Exception as e: - self.tool.stream_log( - f"Error while processing documents {doc_id}: {e}", - level=LogLevel.ERROR, - ) - raise SdkError( - f"Error while processing documents for indexing {doc_id}: {e}" - ) from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py b/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py deleted file mode 100644 index 3df2a3776e..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/automerging.py +++ /dev/null @@ -1,85 +0,0 @@ -import logging - -from llama_index.core import VectorStoreIndex -from llama_index.core.retrievers import AutoMergingRetriever as LlamaAutoMergingRetriever -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class AutomergingRetriever(BaseRetriever): - """Automerging retrieval using LlamaIndex's native AutoMergingRetriever. - - This retriever merges smaller chunks into larger ones when the smaller chunks - don't contain enough information, providing better context for answers. - """ - - def retrieve(self) -> set[str]: - """Retrieve text chunks using LlamaIndex's native AutoMergingRetriever. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - logger.info( - f"Retrieving chunks for {self.doc_id} using LlamaIndex AutoMergingRetriever." - ) - - # Get the vector store index - vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index() - - # Create base vector retriever with metadata filters - base_retriever = vector_store_index.as_retriever( - similarity_top_k=self.top_k, - filters=MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ), - ) - - # Try to use native AutoMergingRetriever - try: - # Create AutoMergingRetriever with the base retriever - auto_merging_retriever = LlamaAutoMergingRetriever( - base_retriever, - storage_context=self.vector_db.get_storage_context() - if hasattr(self.vector_db, "get_storage_context") - else None, - verbose=False, - ) - - # Retrieve nodes using auto-merging - nodes = auto_merging_retriever.retrieve(self.prompt) - - except Exception as e: - logger.error(f"AutoMergingRetriever failed : {e}") - raise RetrievalError(f"AutoMergingRetriever failed: {str(e)}") from e - - # Extract unique text chunks - chunks: set[str] = set() - for node in nodes: - if node.score > 0: - chunks.add(node.get_content()) - else: - logger.info( - f"Node score is less than 0. " - f"Ignored: {node.node_id} with score {node.score}" - ) - - logger.info( - f"Successfully retrieved {len(chunks)} chunks using AutoMergingRetriever." - ) - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during auto-merging retrieval for {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error( - f"Unexpected error during auto-merging retrieval for {self.doc_id}: {e}" - ) - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py b/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py deleted file mode 100644 index 288223c00a..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py +++ /dev/null @@ -1,61 +0,0 @@ -from unstract.prompt_service.core.retrievers.retriever_llm import RetrieverLLM -from unstract.sdk1.llm import LLM -from unstract.sdk1.vector_db import VectorDB - - -class BaseRetriever: - def __init__( - self, - vector_db: VectorDB, - prompt: str, - doc_id: str, - top_k: int, - llm: LLM | None = None, - ): - """Initialize the Retrieval class. - - Args: - vector_db (VectorDB): The vector database instance. - prompt (str): The query prompt. - doc_id (str): Document identifier for query context. - top_k (int): Number of top results to retrieve. - """ - self.vector_db = vector_db - self.prompt = prompt - self.doc_id = doc_id - self.top_k = top_k - self._llm: LLM | None = llm - self._retriever_llm: RetrieverLLM | None = None - - @property - def llm(self) -> RetrieverLLM | None: - """Return a llama-index compatible LLM, lazily created on first access. - - Avoids the cost of RetrieverLLM construction (adapter init, - CallbackManager setup) for retrievers that never use the LLM - (Simple, Automerging, Recursive). - """ - if self._llm is None: - return None - if self._retriever_llm is None: - self._retriever_llm = RetrieverLLM(llm=self._llm) - return self._retriever_llm - - def require_llm(self) -> RetrieverLLM: - """Return the llama-index LLM or raise if not configured. - - Call this in retrievers that need an LLM (KeywordTable, Fusion, - Subquestion) to fail early with a clear message instead of - letting llama-index silently fall back to its default OpenAI LLM. - """ - llm = self.llm - if llm is None: - raise ValueError( - f"{type(self).__name__} requires an LLM. " - "Pass llm= when constructing the retriever." - ) - return llm - - @staticmethod - def retrieve() -> set[str]: - return set() diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py deleted file mode 100644 index 5fd7870f01..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/fusion.py +++ /dev/null @@ -1,95 +0,0 @@ -import logging - -from llama_index.core import VectorStoreIndex -from llama_index.core.retrievers import QueryFusionRetriever -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class FusionRetriever(BaseRetriever): - """Fusion retrieval class using LlamaIndex's native QueryFusionRetriever. - - This technique generates multiple query variations and combines results - using reciprocal rank fusion for improved relevance. - """ - - def retrieve(self) -> set[str]: - """Retrieve text chunks using LlamaIndex's QueryFusionRetriever. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - llm = self.require_llm() - logger.info( - f"Retrieving chunks for {self.doc_id} using LlamaIndex QueryFusionRetriever." - ) - - # Get the vector store index - vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index() - - # Create multiple retrievers with different parameters for true fusion - filters = MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ) - - # Retriever 1: Standard similarity search - retriever_1 = vector_store_index.as_retriever( - similarity_top_k=self.top_k, - filters=filters, - ) - - # Retriever 2: Broader search with more candidates - retriever_2 = vector_store_index.as_retriever( - similarity_top_k=self.top_k * 2, - filters=filters, - ) - - # Retriever 3: Focused search with fewer candidates - retriever_3 = vector_store_index.as_retriever( - similarity_top_k=max(1, self.top_k // 2), - filters=filters, - ) - - # Create LlamaIndex QueryFusionRetriever with multiple retrievers - fusion_retriever = QueryFusionRetriever( - [retriever_1, retriever_2, retriever_3], # Multiple retrievers for fusion - similarity_top_k=self.top_k, - num_queries=4, # Generate multiple query variations - mode="simple", # Use simple fusion mode (reciprocal rank fusion) - use_async=False, - verbose=True, - llm=llm, - ) - - # Retrieve nodes using fusion technique - nodes = fusion_retriever.retrieve(self.prompt) - - # Extract unique text chunks - chunks: set[str] = set() - for node in nodes: - if node.score > 0: - chunks.add(node.get_content()) - else: - logger.info( - f"Node score is less than 0. " - f"Ignored: {node.node_id} with score {node.score}" - ) - - logger.info(f"Successfully retrieved {len(chunks)} chunks using fusion.") - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during fusion retrieval for {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error( - f"Unexpected error during fusion retrieval for {self.doc_id}: {e}" - ) - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py b/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py deleted file mode 100644 index d17e507b50..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/keyword_table.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging - -from llama_index.core import VectorStoreIndex -from llama_index.core.indices.keyword_table import KeywordTableIndex -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class KeywordTableRetriever(BaseRetriever): - """Keyword table retrieval using LlamaIndex's native KeywordTableIndex.""" - - def retrieve(self) -> set[str]: - """Retrieve text chunks using LlamaIndex's native KeywordTableIndex. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - llm = self.require_llm() - logger.info( - f"Retrieving chunks for {self.doc_id} using LlamaIndex KeywordTableIndex." - ) - - # Get documents from vector index for keyword indexing - vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index() - - # Get all nodes for the document - all_retriever = vector_store_index.as_retriever( - similarity_top_k=1000, # Get all nodes - filters=MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ), - ) - - # Retrieve all nodes to build keyword index - all_nodes = all_retriever.retrieve(" ") - - if not all_nodes: - logger.warning(f"No nodes found for doc_id: {self.doc_id}") - return set() - - # Create KeywordTableIndex from nodes using our provided LLM - keyword_index = KeywordTableIndex( - nodes=[node.node for node in all_nodes], - show_progress=True, - llm=llm, - ) - - # Create retriever from keyword index - keyword_retriever = keyword_index.as_retriever( - similarity_top_k=self.top_k, - ) - - # Retrieve nodes using keyword matching - nodes = keyword_retriever.retrieve(self.prompt) - - # Extract unique text chunks - chunks: set[str] = set() - for node in nodes: - chunks.add(node.get_content()) - - logger.info( - f"Successfully retrieved {len(chunks)} chunks using KeywordTableIndex." - ) - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during keyword retrieval for {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error( - f"Unexpected error during keyword retrieval for {self.doc_id}: {e}" - ) - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py b/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py deleted file mode 100644 index dcec3d6176..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/recursive.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging - -from llama_index.core import VectorStoreIndex -from llama_index.core.retrievers import RecursiveRetriever -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class RecursiveRetrieval(BaseRetriever): - """Recursive retrieval using LlamaIndex's native RecursiveRetriever. - - This retriever performs recursive retrieval by breaking down queries - and refining results through multiple retrieval steps. - """ - - def retrieve(self) -> set[str]: - """Retrieve text chunks using LlamaIndex's native RecursiveRetriever. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - logger.info( - f"Retrieving chunks for {self.doc_id} using LlamaIndex RecursiveRetriever." - ) - - # Get the vector store index - vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index() - - # Create base retriever with metadata filters - base_retriever = vector_store_index.as_retriever( - similarity_top_k=self.top_k, - filters=MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ), - ) - - # Create RecursiveRetriever - recursive_retriever = RecursiveRetriever( - "vector", # root retriever key - retriever_dict={"vector": base_retriever}, - verbose=True, - ) - - # Retrieve nodes using RecursiveRetriever - nodes = recursive_retriever.retrieve(self.prompt) - - # Extract unique text chunks - chunks: set[str] = set() - for node in nodes: - if node.score > 0: - chunks.add(node.get_content()) - else: - logger.info( - f"Node score is less than 0. " - f"Ignored: {node.node_id} with score {node.score}" - ) - - logger.info( - f"Successfully retrieved {len(chunks)} chunks using RecursiveRetriever." - ) - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during recursive retrieval for {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error( - f"Unexpected error during recursive retrieval for {self.doc_id}: {e}" - ) - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py b/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py deleted file mode 100644 index c2038f9181..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/retriever_llm.py +++ /dev/null @@ -1,126 +0,0 @@ -from collections.abc import Sequence -from typing import Any - -from llama_index.core.base.llms.types import ( - ChatMessage, - ChatResponse, - ChatResponseAsyncGen, - ChatResponseGen, - CompletionResponse, - CompletionResponseAsyncGen, - CompletionResponseGen, - LLMMetadata, - MessageRole, -) -from llama_index.core.llms.llm import LLM as LlamaIndexBaseLLM # noqa: N811 -from pydantic import PrivateAttr - -from unstract.sdk1.llm import LLM, LLMCompat - - -class RetrieverLLM(LlamaIndexBaseLLM): - """Bridges SDK1's LLMCompat with llama-index's LLM for retriever use. - - Llama-index's ``resolve_llm()`` asserts ``isinstance(llm, LLM)`` - where ``LLM`` is ``llama_index.core.llms.llm.LLM``. Since SDK1's - ``LLMCompat`` is a plain class without llama-index inheritance, - it fails this check. - - ``RetrieverLLM`` inherits from llama-index's ``LLM`` base class - (passing the isinstance check) and delegates all LLM calls to an - internal ``LLMCompat`` instance. - """ - - _compat: LLMCompat = PrivateAttr() - - def __init__(self, llm: LLM, **kwargs: Any) -> None: # noqa: ANN401 - """Initialize with an SDK1 LLM instance.""" - super().__init__(**kwargs) - self._compat = LLMCompat.from_llm(llm) - - @property - def metadata(self) -> LLMMetadata: - return LLMMetadata( - is_chat_model=True, - model_name=self._compat.get_model_name(), - ) - - # ── Sync ───────────────────────────────────────────────────────────────── - - def chat( - self, - messages: Sequence[ChatMessage], - **kwargs: Any, # noqa: ANN401 - ) -> ChatResponse: - result = self._compat.chat(messages, **kwargs) - return ChatResponse( - message=ChatMessage( - role=MessageRole.ASSISTANT, - content=result.message.content, - ), - raw=result.raw, - ) - - def complete( - self, - prompt: str, - formatted: bool = False, - **kwargs: Any, # noqa: ANN401 - ) -> CompletionResponse: - result = self._compat.complete(prompt, formatted=formatted, **kwargs) - return CompletionResponse(text=result.text, raw=result.raw) - - def stream_chat( - self, - messages: Sequence[ChatMessage], - **kwargs: Any, # noqa: ANN401 - ) -> ChatResponseGen: - raise NotImplementedError("stream_chat is not supported.") - - def stream_complete( - self, - prompt: str, - formatted: bool = False, - **kwargs: Any, # noqa: ANN401 - ) -> CompletionResponseGen: - raise NotImplementedError("stream_complete is not supported.") - - # ── Async ──────────────────────────────────────────────────────────────── - - async def achat( - self, - messages: Sequence[ChatMessage], - **kwargs: Any, # noqa: ANN401 - ) -> ChatResponse: - result = await self._compat.achat(messages, **kwargs) - return ChatResponse( - message=ChatMessage( - role=MessageRole.ASSISTANT, - content=result.message.content, - ), - raw=result.raw, - ) - - async def acomplete( - self, - prompt: str, - formatted: bool = False, - **kwargs: Any, # noqa: ANN401 - ) -> CompletionResponse: - result = await self._compat.acomplete(prompt, formatted=formatted, **kwargs) - return CompletionResponse(text=result.text, raw=result.raw) - - async def astream_chat( - self, - messages: Sequence[ChatMessage], - **kwargs: Any, # noqa: ANN401 - ) -> ChatResponseAsyncGen: - raise NotImplementedError("astream_chat is not supported.") - - async def astream_complete( - self, - prompt: str, - formatted: bool = False, - **kwargs: Any, # noqa: ANN401 - ) -> CompletionResponseAsyncGen: - raise NotImplementedError("astream_complete is not supported.") diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/router.py b/prompt-service/src/unstract/prompt_service/core/retrievers/router.py deleted file mode 100644 index b5bc4efe1d..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/router.py +++ /dev/null @@ -1,157 +0,0 @@ -import logging - -from llama_index.core import VectorStoreIndex -from llama_index.core.query_engine import RouterQueryEngine -from llama_index.core.selectors import LLMSingleSelector -from llama_index.core.tools import QueryEngineTool, ToolMetadata -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class RouterRetriever(BaseRetriever): - """Router retrieval class using LlamaIndex's native RouterQueryEngine. - - This technique intelligently routes queries to different retrieval strategies - based on query analysis. - """ - - def _create_metadata_filters(self): - """Create metadata filters for doc_id.""" - return MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ) - - def _create_base_query_engine(self, vector_store_index, filters): - """Create the base vector query engine.""" - return vector_store_index.as_query_engine( - similarity_top_k=self.top_k, - filters=filters, - llm=self.llm, - ) - - def _add_keyword_search_tool(self, query_engine_tools, vector_store_index, filters): - """Add keyword search tool to query engine tools list.""" - try: - keyword_query_engine = vector_store_index.as_query_engine( - similarity_top_k=self.top_k * 2, - filters=filters, - llm=self.llm, - ) - query_engine_tools.append( - QueryEngineTool( - query_engine=keyword_query_engine, - metadata=ToolMetadata( - name="keyword_search", - description=( - "Best for finding specific terms, names, numbers, dates, " - "or exact phrases. Use when looking for precise matches." - ), - ), - ) - ) - except Exception as e: - logger.debug(f"Could not create keyword search engine: {e}") - - def _add_broad_search_tool(self, query_engine_tools, vector_store_index, filters): - """Add broad search tool to query engine tools list.""" - try: - broad_query_engine = vector_store_index.as_query_engine( - similarity_top_k=self.top_k * 3, - filters=filters, - llm=self.llm, - ) - query_engine_tools.append( - QueryEngineTool( - query_engine=broad_query_engine, - metadata=ToolMetadata( - name="broad_search", - description=( - "Useful for general questions, exploratory queries, " - "or when you need comprehensive information on a topic." - ), - ), - ) - ) - except Exception as e: - logger.debug(f"Could not create broad search engine: {e}") - - def _extract_chunks_from_response(self, response): - """Extract chunks from router query response.""" - chunks: set[str] = set() - if hasattr(response, "source_nodes"): - for node in response.source_nodes: - if node.score > 0: - chunks.add(node.get_content()) - else: - logger.info( - f"Node score is less than 0. " - f"Ignored: {node.node_id} with score {node.score}" - ) - return chunks - - def retrieve(self) -> set[str]: - """Retrieve text chunks using LlamaIndex's RouterQueryEngine. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - logger.info( - f"Retrieving chunks for {self.doc_id} using LlamaIndex RouterQueryEngine." - ) - - vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index() - filters = self._create_metadata_filters() - vector_query_engine = self._create_base_query_engine( - vector_store_index, filters - ) - - if not self.llm: - return set() - - # Create base query engine tools - query_engine_tools = [ - QueryEngineTool( - query_engine=vector_query_engine, - metadata=ToolMetadata( - name="vector_search", - description=( - "Useful for semantic similarity search, conceptual questions, " - "and finding information based on meaning and context." - ), - ), - ), - ] - - # Add additional search strategies - self._add_keyword_search_tool(query_engine_tools, vector_store_index, filters) - self._add_broad_search_tool(query_engine_tools, vector_store_index, filters) - - # Create and execute router query - router_query_engine = RouterQueryEngine.from_defaults( - selector=LLMSingleSelector.from_defaults(llm=self.llm), - query_engine_tools=query_engine_tools, - verbose=True, - llm=self.llm, - ) - - response = router_query_engine.query(self.prompt) - chunks = self._extract_chunks_from_response(response) - - logger.info(f"Successfully retrieved {len(chunks)} chunks using router.") - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during router retrieval for {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error( - f"Unexpected error during router retrieval for {self.doc_id}: {e}" - ) - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py deleted file mode 100644 index 15f25db1bb..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py +++ /dev/null @@ -1,51 +0,0 @@ -import time - -from flask import current_app as app -from llama_index.core import VectorStoreIndex -from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever - - -class SimpleRetriever(BaseRetriever): - def retrieve(self) -> set[str]: - context = self._simple_retrieval() - if not context: - # UN-1288 For Pinecone, we are seeing an inconsistent case where - # query with doc_id fails even though indexing just happened. - # This causes the following retrieve to return no text. - # To rule out any lag on the Pinecone vector DB write, - # the following sleep is added - # Note: This will not fix the issue. Since this issue is inconsistent - # and not reproducible easily, this is just a safety net. - app.logger.info( - f"[doc_id: {self.doc_id}] Could not retrieve context, " - "retrying after 2 secs to handle issues due to lag" - ) - time.sleep(2) - context = self._simple_retrieval() - return context - - def _simple_retrieval(self): - vector_query_engine: VectorStoreIndex = self.vector_db.get_vector_store_index() - retriever = vector_query_engine.as_retriever( - similarity_top_k=self.top_k, - filters=MetadataFilters( - filters=[ - ExactMatchFilter(key="doc_id", value=self.doc_id), - ], - ), - ) - nodes = retriever.retrieve(self.prompt) - context: set[str] = set() - for node in nodes: - # May have to fine-tune this value for node score or keep it - # configurable at the adapter level - if node.score > 0: - context.add(node.get_content()) - else: - app.logger.info( - "Node score is less than 0. " - f"Ignored: {node.node_id} with score {node.score}" - ) - return context diff --git a/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py deleted file mode 100644 index 6930bc131b..0000000000 --- a/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging - -from llama_index.core.query_engine import SubQuestionQueryEngine -from llama_index.core.question_gen.llm_generators import LLMQuestionGenerator -from llama_index.core.schema import QueryBundle -from llama_index.core.tools import QueryEngineTool, ToolMetadata - -from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever -from unstract.prompt_service.exceptions import RetrievalError - -logger = logging.getLogger(__name__) - - -class SubquestionRetriever(BaseRetriever): - """SubquestionRetrieval class for querying VectorDB using LlamaIndex's - SubQuestionQueryEngine. - """ - - def retrieve(self) -> set[str]: - """Retrieve text chunks from the VectorDB based on the provided prompt. - - Returns: - set[str]: A set of text chunks retrieved from the database. - """ - try: - llm = self.require_llm() - logger.info("Initialising vector query engine...") - vector_query_engine = self.vector_db.get_vector_store_index().as_query_engine( - llm=llm, similarity_top_k=self.top_k - ) - logger.info( - f"Retrieving chunks for {self.doc_id} using SubQuestionQueryEngine." - ) - query_engine_tools = [ - QueryEngineTool( - query_engine=vector_query_engine, - metadata=ToolMetadata( - name=self.doc_id, description=f"Nodes for {self.doc_id}" - ), - ), - ] - query_bundle = QueryBundle(query_str=self.prompt) - - question_gen = LLMQuestionGenerator.from_defaults( - llm=llm, - ) - query_engine = SubQuestionQueryEngine.from_defaults( - query_engine_tools=query_engine_tools, - question_gen=question_gen, - use_async=True, - llm=llm, - ) - - response = query_engine.query(str_or_query_bundle=query_bundle) - - chunks: set[str] = {node.text for node in response.source_nodes} - logger.info(f"Successfully retrieved {len(chunks)} chunks.") - return chunks - - except (ValueError, AttributeError, KeyError, ImportError) as e: - logger.error(f"Error during retrieving chunks {self.doc_id}: {e}") - raise RetrievalError(str(e)) from e - except Exception as e: - logger.error(f"Unexpected error during retrieving chunks {self.doc_id}: {e}") - raise RetrievalError(f"Unexpected error: {str(e)}") from e diff --git a/prompt-service/src/unstract/prompt_service/dto.py b/prompt-service/src/unstract/prompt_service/dto.py deleted file mode 100644 index 8c9e4f3d3c..0000000000 --- a/prompt-service/src/unstract/prompt_service/dto.py +++ /dev/null @@ -1,39 +0,0 @@ -from dataclasses import dataclass, field -from typing import Any - - -@dataclass -class InstanceIdentifiers: - embedding_instance_id: str - vector_db_instance_id: str - x2text_instance_id: str - llm_instance_id: str - tool_id: str - tags: list[str] | None = None - - -@dataclass -class FileInfo: - file_path: str - file_hash: str - - -@dataclass -class ChunkingConfig: - chunk_size: int - chunk_overlap: int - - def __post_init__(self) -> None: - if self.chunk_size == 0: - raise ValueError( - "Indexing cannot be done for zero chunks." - "Please provide a valid chunk_size." - ) - - -@dataclass -class ProcessingOptions: - reindex: bool = False - enable_highlight: bool = False - enable_word_confidence: bool = False - usage_kwargs: dict[Any, Any] = field(default_factory=dict) diff --git a/prompt-service/src/unstract/prompt_service/exceptions.py b/prompt-service/src/unstract/prompt_service/exceptions.py deleted file mode 100644 index 98634ed5fd..0000000000 --- a/prompt-service/src/unstract/prompt_service/exceptions.py +++ /dev/null @@ -1,56 +0,0 @@ -from unstract.core.flask.exceptions import APIError - - -class BadRequest(APIError): - code = 400 - message = "Bad Request / No payload" - - -class RateLimitError(APIError): - code = 429 - message = "Running into rate limit errors, please try again later" - - -class MissingFieldError(APIError): - """Custom error for missing fields.""" - - def __init__(self, missing_fields: list[str]): - message = f"Missing required fields: {', '.join(missing_fields)}" - super().__init__(message=message) - - -class RetrievalError(APIError): - """Custom exception raised for errors during retrieval from VectorDB.""" - - DEFAULT_MESSAGE = ( - "Error while retrieving data from the VectorDB. " - "Please contact the admin for further assistance." - ) - - -class ExtractionError(APIError): - DEFAULT_MESSAGE = "Error while extracting from a document" - - -class UnprocessableEntity(APIError): - code = 422 - message = "Unprocessable Entity" - - -class CustomDataError(APIError): - """Custom exception raised for errors with custom_data variables.""" - - code = 400 - - def __init__(self, variable: str, reason: str, is_ide: bool = True): - if is_ide: - help_text = "Please define this key in Prompt Studio Settings > Custom Data." - else: - help_text = ( - "Please include this key in the 'custom_data' field of your API request." - ) - variable_display = "{{custom_data." + variable + "}}" - message = ( - f"Custom data error for variable '{variable_display}': {reason} {help_text}" - ) - super().__init__(message=message) diff --git a/prompt-service/src/unstract/prompt_service/extensions.py b/prompt-service/src/unstract/prompt_service/extensions.py deleted file mode 100644 index 1626591dc0..0000000000 --- a/prompt-service/src/unstract/prompt_service/extensions.py +++ /dev/null @@ -1,36 +0,0 @@ -from collections.abc import Generator -from contextlib import contextmanager -from os import environ as env -from typing import Any - -from playhouse.postgres_ext import PostgresqlExtDatabase - -from unstract.prompt_service.utils.env_loader import get_env_or_die - -# Load required environment variables -db_host = get_env_or_die("PG_BE_HOST") -db_port = get_env_or_die("PG_BE_PORT") -db_user = get_env_or_die("PG_BE_USERNAME") -db_pass = get_env_or_die("PG_BE_PASSWORD") -db_name = get_env_or_die("PG_BE_DATABASE") -application_name = env.get("APPLICATION_NAME", "unstract-prompt-service") - -# Initialize and connect to the database -db = PostgresqlExtDatabase( - database=db_name, - user=db_user, - host=db_host, - password=db_pass, - port=db_port, - options=f"-c application_name={application_name}", -) - - -@contextmanager -def db_context() -> Generator[PostgresqlExtDatabase, Any, None]: - try: - db.connect() - yield db - finally: - if not db.is_closed(): - db.close() diff --git a/prompt-service/src/unstract/prompt_service/helpers/auth.py b/prompt-service/src/unstract/prompt_service/helpers/auth.py deleted file mode 100644 index cd9fd58128..0000000000 --- a/prompt-service/src/unstract/prompt_service/helpers/auth.py +++ /dev/null @@ -1,85 +0,0 @@ -from typing import Any - -from flask import Request, request -from flask import current_app as app - -from unstract.prompt_service.constants import DBTableV2 -from unstract.prompt_service.extensions import db, db_context -from unstract.prompt_service.utils.db_utils import DBUtils -from unstract.prompt_service.utils.env_loader import get_env_or_die - -DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract") - - -class AuthHelper: - @staticmethod - def validate_bearer_token(token: str | None) -> bool: - try: - if token is None: - app.logger.error("Authentication failed. Empty bearer token") - return False - - platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}' - with db_context(): - query = f"SELECT * FROM {platform_key_table} WHERE key = %s" - cursor = db.execute_sql(query, (token,)) - result_row = cursor.fetchone() - cursor.close() - if not result_row or len(result_row) == 0: - app.logger.error(f"Authentication failed. bearer token not found {token}") - return False - platform_key = str(result_row[1]) - is_active = bool(result_row[2]) - if not is_active: - app.logger.error( - f"Token is not active. Activate \ - before using it. token {token}" - ) - return False - if platform_key != token: - app.logger.error(f"Authentication failed. Invalid bearer token: {token}") - return False - - except Exception as e: - app.logger.error( - f"Error while validating bearer token: {e}", - stack_info=True, - exc_info=True, - ) - return False - return True - - @staticmethod - def get_token_from_auth_header(request: Request) -> str | None: - try: - bearer_token = request.headers.get("Authorization") - if not bearer_token: - return None - token: str = bearer_token.strip().replace("Bearer ", "") - return token - except Exception as e: - app.logger.info(f"Exception while getting token {e}") - return None - - @staticmethod - def get_account_from_bearer_token(token: str | None) -> str: - platform_key_table = DBTableV2.PLATFORM_KEY - organization_table = DBTableV2.ORGANIZATION - - query = f"SELECT organization_id FROM {platform_key_table} WHERE key=%s" - organization = DBUtils.execute_query(query, (token,)) - query_org = f"SELECT schema_name FROM {organization_table} WHERE id=%s" - schema_name: str = DBUtils.execute_query(query_org, (organization,)) - return schema_name - - @staticmethod - def auth_required(func: Any) -> Any: - def wrapper(*args: Any, **kwargs: Any) -> Any: - token = AuthHelper.get_token_from_auth_header(request) - # Check if bearer token exists and validate it - if not token or not AuthHelper.validate_bearer_token(token): - return "Unauthorized", 401 - request.token = token - return func(*args, **kwargs) - - return wrapper diff --git a/prompt-service/src/unstract/prompt_service/helpers/postprocessor.py b/prompt-service/src/unstract/prompt_service/helpers/postprocessor.py deleted file mode 100644 index 1e7aad72d6..0000000000 --- a/prompt-service/src/unstract/prompt_service/helpers/postprocessor.py +++ /dev/null @@ -1,114 +0,0 @@ -import json -import logging -from typing import Any - -import requests - -logger = logging.getLogger(__name__) - - -def _validate_structured_output(data: Any) -> bool: - """Validate that structured output is a dict or list.""" - return isinstance(data, (dict, list)) - - -def _validate_highlight_data(updated_data: Any, original_data: Any) -> Any: - """Validate highlight data and return appropriate value.""" - if ( - updated_data is not None - and updated_data != original_data - and not isinstance(updated_data, list) - ): - logger.warning( - "Ignoring webhook highlight_data due to invalid type (expected list)" - ) - return original_data - return updated_data - - -def _process_successful_response( - response_data: dict, parsed_data: dict, highlight_data: list | None -) -> tuple[dict[str, Any], list | None]: - """Process successful webhook response.""" - if "structured_output" not in response_data: - logger.warning("Response missing 'structured_output' key") - return parsed_data, highlight_data - - updated_parsed_data = response_data["structured_output"] - - if not _validate_structured_output(updated_parsed_data): - logger.warning("Ignoring postprocessing due to invalid structured_output type") - return parsed_data, highlight_data - - updated_highlight_data = response_data.get("highlight_data", highlight_data) - updated_highlight_data = _validate_highlight_data( - updated_highlight_data, highlight_data - ) - - return updated_parsed_data, updated_highlight_data - - -def _make_webhook_request( - webhook_url: str, payload: dict, timeout: float -) -> tuple[dict[str, Any], list | None] | None: - """Make webhook request and return processed response or None on failure.""" - try: - response = requests.post( - webhook_url, - json=payload, - timeout=timeout, - headers={"Content-Type": "application/json"}, - allow_redirects=False, # Prevent redirect-based SSRF - ) - - if response.status_code != 200: - logger.warning( - f"Postprocessing server returned status code: {response.status_code}" - ) - return None - - return response.json() - - except json.JSONDecodeError as e: - logger.warning(f"Invalid JSON response from postprocessing server: {e}") - except requests.exceptions.Timeout: - logger.warning(f"Postprocessing server request timed out after {timeout}s") - except requests.exceptions.RequestException as e: - logger.warning(f"Postprocessing server request failed: {e}") - except Exception as e: - logger.warning(f"Unexpected error during postprocessing: {e}") - - return None - - -def postprocess_data( - parsed_data: dict[str, Any], - webhook_enabled: bool = False, - webhook_url: str | None = None, - timeout: float = 2.0, - highlight_data: list | None = None, -) -> tuple[dict[str, Any], list | None]: - """Post-process parsed data by sending it to an external server. - - Args: - parsed_data: The parsed data to be post-processed - webhook_enabled: Whether webhook postprocessing is enabled - webhook_url: URL endpoint for the webhook - timeout: Request timeout in seconds (default: 2.0) - highlight_data: Highlight data from metadata to send to webhook - - Returns: - tuple: (postprocessed_data, updated_highlight_data) if successful, otherwise (original_parsed_data, original_highlight_data) - """ - if not webhook_enabled or not webhook_url: - return parsed_data, highlight_data - - payload = {"structured_output": parsed_data} - if highlight_data is not None: - payload["highlight_data"] = highlight_data - - response_data = _make_webhook_request(webhook_url, payload, timeout) - if response_data is None: - return parsed_data, highlight_data - - return _process_successful_response(response_data, parsed_data, highlight_data) diff --git a/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py deleted file mode 100644 index e2a30a0e2e..0000000000 --- a/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import warnings - -from flask import current_app - -from unstract.prompt_service.constants import PromptServiceConstants -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.tool.stream import StreamMixin -from unstract.sdk1.utils.common import PY_TO_UNSTRACT_LOG_LEVEL - - -class PromptServiceBaseTool(StreamMixin): - # Remove unused log_level arg - def __init__(self, log_level: LogLevel | None = None, platform_key: str = "") -> None: - """Args: - tool (UnstractAbstractTool): Instance of UnstractAbstractTool. - Deprecated, will be removed in future versions - Notes: - - PLATFORM_SERVICE_API_KEY environment variable is required. - """ - if log_level: - warnings.warn( - "The 'log_level' argument is deprecated and will be " - "removed in a future version.", - DeprecationWarning, - stacklevel=2, # ensures the warning points to the caller of the method - ) - self.log_level = PY_TO_UNSTRACT_LOG_LEVEL.get( - current_app.logger.getEffectiveLevel(), LogLevel.INFO - ) - self.platform_key = platform_key - super().__init__(log_level=self.log_level) - - def get_env_or_die(self, env_key: str) -> str: - """Returns the value of an env variable. - - If it is None, raises an error and exits - - Args: - env_key (str): Key to retrieve - - Returns: - str: Value of the env - """ - # HACK: Adding platform key for multitenancy - if env_key == PromptServiceConstants.PLATFORM_SERVICE_API_KEY: - if not self.platform_key: - current_app.logger.error(f"{env_key} is required") - else: - key: str = self.platform_key - return key - else: - env_value = os.environ.get(env_key) - if env_value is None: - current_app.logger.error(f"Env variable {env_key} is required") - return env_value # type:ignore - return "" diff --git a/prompt-service/src/unstract/prompt_service/helpers/usage.py b/prompt-service/src/unstract/prompt_service/helpers/usage.py deleted file mode 100644 index 63a14c4336..0000000000 --- a/prompt-service/src/unstract/prompt_service/helpers/usage.py +++ /dev/null @@ -1,140 +0,0 @@ -import logging -import traceback -from logging import Logger -from typing import Any - -from flask import current_app as app - -from unstract.prompt_service.constants import DBTableV2 -from unstract.prompt_service.extensions import db, db_context -from unstract.prompt_service.utils.db_utils import DBUtils -from unstract.prompt_service.utils.env_loader import get_env_or_die -from unstract.sdk1.audit import Audit - -logger = logging.getLogger(__name__) - - -class UsageHelper: - @staticmethod - def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]: - DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract") - organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token) - run_id: str = metadata["run_id"] - query: str = f""" - SELECT - usage_type, - llm_usage_reason, - model_name, - SUM(prompt_tokens) AS input_tokens, - SUM(completion_tokens) AS output_tokens, - SUM(total_tokens) AS total_tokens, - SUM(embedding_tokens) AS embedding_tokens, - SUM(cost_in_dollars) AS cost_in_dollars - FROM "{DB_SCHEMA}"."{DBTableV2.TOKEN_USAGE}" - WHERE run_id = %s and organization_id = %s - GROUP BY usage_type, llm_usage_reason, model_name; - """ - logger: Logger = app.logger - try: - logger.info( - "Querying usage metadata for org_id: %s, run_id: %s", org_id, run_id - ) - with db_context(): - with db.execute_sql(query, (run_id, organization_uid)) as cursor: - results: list[tuple] = cursor.fetchall() - # Process results as needed - for row in results: - key, item = UsageHelper._get_key_and_item(row) - # Initialize the key as an empty list if it doesn't exist - if key not in metadata: - metadata[key] = [] - # Append the item to the list associated with the key - metadata[key].append(item) - except Exception as e: - logger.error(f"Error while querying usage metadata: {e}") - return metadata - - @staticmethod - def _get_key_and_item(row: tuple) -> tuple[str, dict[str, Any]]: - ( - usage_type, - llm_usage_reason, - model_name, - input_tokens, - output_tokens, - total_tokens, - embedding_tokens, - cost_in_dollars, - ) = row - cost_in_dollars: str = UsageHelper._format_float_positional(cost_in_dollars) - key: str = usage_type - item: dict[str, Any] = { - "model_name": model_name, - "cost_in_dollars": cost_in_dollars, - } - if llm_usage_reason: - key = f"{llm_usage_reason}_{key}" - item["input_tokens"] = input_tokens - item["output_tokens"] = output_tokens - item["total_tokens"] = total_tokens - else: - item["embedding_tokens"] = embedding_tokens - return key, item - - @staticmethod - def _format_float_positional(value: float, precision: int = 10) -> str: - formatted: str = f"{value:.{precision}f}" - return formatted.rstrip("0").rstrip(".") if "." in formatted else formatted - - @staticmethod - def push_usage_data( - event_type: str, - kwargs: dict[str, Any], - platform_api_key: str, - token_counter=None, - model_name: str = "", - ) -> bool: - """Push usage data to the audit service. - - Args: - event_type: Type of usage event being recorded - kwargs: Additional data to include with the event - platform_api_key: API key for authentication with the audit service - token_counter: Token counter object with token usage metrics - model_name: Name of the model used (if applicable) - - Returns: - bool: True if successful, False otherwise - Note: - This method handles all exceptions internally and returns False on failure - rather than propagating exceptions to the caller. - """ - if not kwargs or not isinstance(kwargs, dict): - logger.error("Invalid kwargs provided to push_usage_data") - return False - - if not platform_api_key or not isinstance(platform_api_key, str): - logger.error("Invalid platform_api_key provided to push_usage_data") - return False - - try: - logger.debug( - f"Pushing usage data for event_type: {event_type}, model: {model_name}" - ) - - # Call the Audit SDK with the appropriate parameters - Audit().push_usage_data( - platform_api_key=platform_api_key, - token_counter=token_counter, - model_name=model_name, - event_type=event_type, - kwargs=kwargs, - ) - - logger.info(f"Successfully pushed usage data for {model_name}") - return True - except Exception as e: - logger.exception( - f"Error pushing usage data: {str(e)} - {traceback.format_exc()}" - ) - return False diff --git a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py deleted file mode 100644 index cd794ce590..0000000000 --- a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py +++ /dev/null @@ -1,204 +0,0 @@ -import json -import re -from functools import lru_cache -from typing import Any - -from flask import current_app as app - -from unstract.prompt_service.constants import VariableConstants, VariableType -from unstract.prompt_service.exceptions import CustomDataError -from unstract.prompt_service.utils.request import HTTPMethod, make_http_request - - -class VariableReplacementHelper: - @staticmethod - def replace_static_variable( - prompt: str, structured_output: dict[str, Any], variable: str - ) -> str: - output_value = VariableReplacementHelper.check_static_variable_run_status( - structure_output=structured_output, variable=variable - ) - if not output_value: - return prompt - static_variable_marker_string = "".join(["{{", variable, "}}"]) - - replaced_prompt: str = VariableReplacementHelper.replace_generic_string_value( - prompt=prompt, variable=static_variable_marker_string, value=output_value - ) - - return replaced_prompt - - @staticmethod - def check_static_variable_run_status( - structure_output: dict[str, Any], variable: str - ) -> Any: - output = None - try: - output = structure_output[variable] - except KeyError: - app.logger.warning( - f"Prompt with {variable} is not executed yet." - " Unable to replace the variable" - ) - return output - - @staticmethod - def replace_generic_string_value(prompt: str, variable: str, value: Any) -> str: - formatted_value: str = value - if not isinstance(value, str): - formatted_value = VariableReplacementHelper.handle_json_and_str_types(value) - replaced_prompt = prompt.replace(variable, formatted_value) - return replaced_prompt - - @staticmethod - def handle_json_and_str_types(value: Any) -> str: - try: - formatted_value = json.dumps(value) - except ValueError: - formatted_value = str(value) - return formatted_value - - @staticmethod - def identify_variable_type(variable: str) -> VariableType: - variable_type: VariableType - - # Check for custom_data variable type first - custom_data_pattern = re.compile(VariableConstants.CUSTOM_DATA_VARIABLE_REGEX) - if re.findall(custom_data_pattern, variable): - variable_type = VariableType.CUSTOM_DATA - else: - # Check for dynamic variable type - dynamic_pattern = re.compile(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX) - if re.findall(dynamic_pattern, variable): - variable_type = VariableType.DYNAMIC - else: - variable_type = VariableType.STATIC - return variable_type - - @staticmethod - def replace_dynamic_variable( - prompt: str, variable: str, structured_output: dict[str, Any] - ) -> str: - url = re.search(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX, variable).group(0) - data = re.findall(VariableConstants.DYNAMIC_VARIABLE_DATA_REGEX, variable)[0] - output_value = VariableReplacementHelper.check_static_variable_run_status( - structure_output=structured_output, variable=data - ) - if not output_value: - return prompt - api_response: Any = VariableReplacementHelper.fetch_dynamic_variable_value( - url=url, data=output_value - ) - formatted_api_response: str = VariableReplacementHelper.handle_json_and_str_types( - api_response - ) - static_variable_marker_string = "".join(["{{", variable, "}}"]) - replaced_prompt: str = VariableReplacementHelper.replace_generic_string_value( - prompt=prompt, - variable=static_variable_marker_string, - value=formatted_api_response, - ) - return replaced_prompt - - @staticmethod - def replace_custom_data_variable( - prompt: str, - variable: str, - custom_data: dict[str, Any], - is_ide: bool = True, - ) -> str: - """Replace custom_data variable in prompt. - - Args: - prompt: The prompt containing variables - variable: The variable to replace (e.g., "custom_data.name") - custom_data: The custom_data data dictionary - is_ide: Whether this is running from Prompt Studio IDE (affects error messages) - - Returns: - prompt with variable replaced - """ - # Extract the path from custom_data.path.to.value - custom_data_match = re.search( - VariableConstants.CUSTOM_DATA_VARIABLE_REGEX, variable - ) - if not custom_data_match: - error_msg = "Invalid variable format." - app.logger.error(f"{error_msg}: {variable}") - raise CustomDataError(variable=variable, reason=error_msg, is_ide=is_ide) - - path_str = custom_data_match.group(1) - path_parts = path_str.split(".") - - if not custom_data: - error_msg = "Custom data is not configured." - app.logger.error(error_msg) - raise CustomDataError(variable=path_str, reason=error_msg, is_ide=is_ide) - - # Navigate through the nested dictionary - try: - value = custom_data - for part in path_parts: - value = value[part] - except (KeyError, TypeError) as e: - error_msg = f"Key '{path_str}' not found in custom data." - app.logger.error(error_msg) - raise CustomDataError( - variable=path_str, reason=error_msg, is_ide=is_ide - ) from e - - # Replace in prompt - let replace_generic_string_value handle formatting - # (it only applies json.dumps for non-string values) - variable_marker_string = "".join(["{{", variable, "}}"]) - - replaced_prompt = VariableReplacementHelper.replace_generic_string_value( - prompt=prompt, - variable=variable_marker_string, - value=value, - ) - - return replaced_prompt - - @staticmethod - @lru_cache(maxsize=128) - def _extract_variables_cached(prompt_text: str) -> tuple[str, ...]: - """Internal cached extraction - returns tuple for lru_cache compatibility.""" - return tuple(re.findall(VariableConstants.VARIABLE_REGEX, prompt_text)) - - @staticmethod - def extract_variables_from_prompt(prompt_text: str) -> list[str]: - """Extract variables from prompt with caching and stats logging. - - Uses lru_cache internally and logs cache statistics periodically - to help determine if caching is beneficial. - """ - result = VariableReplacementHelper._extract_variables_cached(prompt_text) - - # Log stats periodically (every 50 calls) - info_after = VariableReplacementHelper._extract_variables_cached.cache_info() - total_calls = info_after.hits + info_after.misses - - if total_calls % 50 == 0 and total_calls > 0: - hit_rate = info_after.hits / total_calls * 100 - app.logger.info( - f"[VariableCache] total={total_calls} hits={info_after.hits} " - f"misses={info_after.misses} hit_rate={hit_rate:.1f}% " - f"size={info_after.currsize}/{info_after.maxsize} " - f"prompt_chars={len(prompt_text)}" - ) - - return list(result) - - @staticmethod - def fetch_dynamic_variable_value(url: str, data: str) -> Any: - # This prototype method currently supports - # only endpoints that do not require authentication. - # Additionally, it only accepts plain text - # inputs for POST requests in this version. - # Future versions may include support for - # authentication and other input formats. - - verb: HTTPMethod = HTTPMethod.POST - headers = {"Content-Type": "text/plain"} - response: Any = make_http_request(verb=verb, url=url, data=data, headers=headers) - return response diff --git a/prompt-service/src/unstract/prompt_service/run.py b/prompt-service/src/unstract/prompt_service/run.py deleted file mode 100644 index b50e972506..0000000000 --- a/prompt-service/src/unstract/prompt_service/run.py +++ /dev/null @@ -1,3 +0,0 @@ -from unstract.prompt_service.config import create_app - -app = create_app() diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py deleted file mode 100644 index 9f8cbf9c28..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py +++ /dev/null @@ -1,521 +0,0 @@ -import ipaddress -import socket -from logging import Logger -from typing import Any -from urllib.parse import urlparse - -from flask import current_app as app - -from unstract.core.flask import PluginManager -from unstract.core.flask.exceptions import APIError -from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys, RunLevel -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.exceptions import RateLimitError -from unstract.prompt_service.helpers.postprocessor import postprocess_data -from unstract.prompt_service.utils.env_loader import get_env_or_die -from unstract.prompt_service.utils.json_repair_helper import ( - repair_json_with_best_structure, -) -from unstract.prompt_service.utils.log import publish_log -from unstract.sdk1.constants import LogLevel -from unstract.sdk1.exceptions import RateLimitError as SdkRateLimitError -from unstract.sdk1.exceptions import SdkError -from unstract.sdk1.file_storage import FileStorage, FileStorageProvider -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper -from unstract.sdk1.llm import LLM - - -def _is_safe_public_url(url: str) -> bool: - """Validate webhook URL for SSRF protection. - - Only allows HTTPS and blocks private/loopback/internal addresses. - Resolves all DNS records (A/AAAA) to prevent DNS rebinding attacks. - """ - try: - p = urlparse(url) - if p.scheme not in ("https",): # Only allow HTTPS for security - return False - host = p.hostname or "" - # Block obvious local hosts - if host in ("localhost",): - return False - - addrs: set[str] = set() - # If literal IP, validate directly; else resolve all records (A/AAAA) - try: - ipaddress.ip_address(host) - addrs.add(host) - except ValueError: - try: - for family, _type, _proto, _canonname, sockaddr in socket.getaddrinfo( - host, None, type=socket.SOCK_STREAM - ): - addr = sockaddr[0] - addrs.add(addr) - except Exception: - return False - - if not addrs: - return False - - # Validate all resolved addresses - for addr in addrs: - try: - ip = ipaddress.ip_address(addr) - except ValueError: - return False - if ( - ip.is_private - or ip.is_loopback - or ip.is_link_local - or ip.is_reserved - or ip.is_multicast - ): - return False - return True - except Exception: - return False - - -class AnswerPromptService: - @staticmethod - def extract_variable( - structured_output: dict[str, Any], - variable_names: list[Any], - output: dict[str, Any], - promptx: str, - ) -> str: - logger: Logger = app.logger - for variable_name in variable_names: - if promptx.find(f"%{variable_name}%") >= 0: - if variable_name in structured_output: - promptx = promptx.replace( - f"%{variable_name}%", - str(structured_output[variable_name]), - ) - else: - raise ValueError( - f"Variable {variable_name} not found in structured output" - ) - - if promptx != output[PSKeys.PROMPT]: - logger.info(f"Prompt after variable replacement: {promptx}") - return promptx - - @staticmethod - def construct_and_run_prompt( - tool_settings: dict[str, Any], - output: dict[str, Any], - llm: LLM, - context: str, - prompt: str, - metadata: dict[str, Any], - file_path: str = "", - execution_source: str | None = ExecutionSource.IDE.value, - ) -> str: - platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "") - word_confidence_postamble = tool_settings.get( - PSKeys.WORD_CONFIDENCE_POSTAMBLE, "" - ) - summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE) - enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False) - enable_word_confidence = tool_settings.get(PSKeys.ENABLE_WORD_CONFIDENCE, False) - # Dependency: enable_word_confidence only works if enable_highlight is enabled - if not enable_highlight: - enable_word_confidence = False - prompt_type = output.get(PSKeys.TYPE, PSKeys.TEXT) - if not enable_highlight or summarize_as_source: - platform_postamble = "" - if not enable_word_confidence or summarize_as_source: - word_confidence_postamble = "" - plugin = PluginManager().get_plugin("json-extraction") - if plugin and hasattr(plugin["entrypoint_cls"], "update_settings"): - plugin["entrypoint_cls"].update_settings(tool_settings, output) - prompt = AnswerPromptService.construct_prompt( - preamble=tool_settings.get(PSKeys.PREAMBLE, ""), - prompt=output[prompt], - postamble=tool_settings.get(PSKeys.POSTAMBLE, ""), - grammar_list=tool_settings.get(PSKeys.GRAMMAR, []), - context=context, - platform_postamble=platform_postamble, - word_confidence_postamble=word_confidence_postamble, - prompt_type=prompt_type, - ) - output[PSKeys.COMBINED_PROMPT] = prompt - return AnswerPromptService.run_completion( - llm=llm, - prompt=prompt, - metadata=metadata, - prompt_key=output[PSKeys.NAME], - prompt_type=prompt_type, - enable_highlight=enable_highlight, - enable_word_confidence=enable_word_confidence, - file_path=file_path, - execution_source=execution_source, - ) - - @staticmethod - def construct_prompt( - preamble: str, - prompt: str, - postamble: str, - grammar_list: list[dict[str, Any]], - context: str, - platform_postamble: str, - word_confidence_postamble: str, - prompt_type: str = PSKeys.TEXT, - ) -> str: - prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}" - if grammar_list is not None and len(grammar_list) > 0: - prompt += "\n" - for grammar in grammar_list: - word = "" - synonyms = [] - if PSKeys.WORD in grammar: - word = grammar[PSKeys.WORD] - if PSKeys.SYNONYMS in grammar: - synonyms = grammar[PSKeys.SYNONYMS] - if len(synonyms) > 0 and word != "": - prompt += ( - f"\nNote: You can consider that the word '{word}' " - f"is the same as {', '.join(synonyms)} in both the question and the context." - ) # noqa - if prompt_type == PSKeys.JSON: - json_postamble = get_env_or_die( - PSKeys.JSON_POSTAMBLE, PSKeys.DEFAULT_JSON_POSTAMBLE - ) - postamble += f"\n{json_postamble}" - if platform_postamble: - platform_postamble += "\n\n" - if word_confidence_postamble: - platform_postamble += f"{word_confidence_postamble}\n\n" - prompt += ( - f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n" - f"-----------------\n\n{platform_postamble}Answer:" - ) - return prompt - - @staticmethod - def run_completion( - llm: LLM, - prompt: str, - metadata: dict[str, str] | None = None, - prompt_key: str | None = None, - prompt_type: str | None = PSKeys.TEXT, - enable_highlight: bool = False, - enable_word_confidence: bool = False, - file_path: str = "", - execution_source: str | None = None, - ) -> str: - logger: Logger = app.logger - try: - highlight_data_plugin: dict[str, Any] = PluginManager().get_plugin( - PSKeys.HIGHLIGHT_DATA_PLUGIN - ) - highlight_data = None - if highlight_data_plugin and enable_highlight: - fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL) - if execution_source == ExecutionSource.IDE.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.PERMANENT, - env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE, - ) - if execution_source == ExecutionSource.TOOL.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.SHARED_TEMPORARY, - env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE, - ) - logger.info( - f"Initializing highlight plugin with: " - f"file_path={file_path}, " - f"execution_source={execution_source}, " - f"fs_instance={fs_instance}" - ) - highlight_data = highlight_data_plugin["entrypoint_cls"]( - file_path=file_path, - fs_instance=fs_instance, - enable_word_confidence=enable_word_confidence, - ).run - completion = llm.complete( - prompt=prompt, - process_text=highlight_data, - extract_json=prompt_type.lower() != PSKeys.TEXT, - ) - answer: str = completion[PSKeys.RESPONSE].text - highlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, []) - confidence_data = completion.get(PSKeys.CONFIDENCE_DATA) - word_confidence_data = completion.get(PSKeys.WORD_CONFIDENCE_DATA) - line_numbers = completion.get(PSKeys.LINE_NUMBERS, []) - whisper_hash = completion.get(PSKeys.WHISPER_HASH, "") - if metadata is not None and prompt_key: - metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = ( - highlight_data - ) - metadata.setdefault(PSKeys.LINE_NUMBERS, {})[prompt_key] = line_numbers - metadata[PSKeys.WHISPER_HASH] = whisper_hash - if confidence_data: - metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[prompt_key] = ( - confidence_data - ) - if enable_word_confidence and word_confidence_data: - metadata.setdefault(PSKeys.WORD_CONFIDENCE_DATA, {})[prompt_key] = ( - word_confidence_data - ) - return answer - except SdkRateLimitError as e: - raise RateLimitError(f"Rate limit error. {str(e)}") from e - except SdkError as e: - logger.error(f"Error fetching response for prompt: {e}.") - # Preserve the status code from the SDK error for proper HTTP response - status_code = getattr(e, "status_code", None) or 500 - raise APIError(message=str(e), code=status_code) from e - - @staticmethod - def extract_table( - output: dict[str, Any], - structured_output: dict[str, Any], - llm: LLM, - execution_source: str, - prompt: str, - ) -> dict[str, Any]: - table_settings = output[PSKeys.TABLE_SETTINGS] - - # Check if prompt has valid schema data using json_repair - has_valid_schema = False - schema_data = None - - if prompt and isinstance(prompt, str): - try: - # Try to repair and parse the prompt as JSON - schema_data = repair_json_with_best_structure(prompt) - # Check if the result is a valid dict (schema object) - if isinstance(schema_data, dict) and schema_data: - has_valid_schema = True - app.logger.info( - "Valid schema detected in prompt, using Smart Table Extractor" - ) - except Exception as e: - app.logger.debug(f"Prompt does not contain valid schema: {e}") - - # If we have a valid schema, use the smart table extractor - if has_valid_schema: - smart_table_plugin: dict[str, Any] = PluginManager().get_plugin( - "smart-table-extractor" - ) - - if smart_table_plugin: - fs_instance = AnswerPromptService._get_file_storage_instance( - execution_source - ) - - try: - # Get the input file from table settings - input_file = table_settings.get("input_file") - - # Run the smart table extractor - result = smart_table_plugin["entrypoint_cls"].run( - llm=llm, - table_settings=table_settings, - fs_instance=fs_instance, - prompt=prompt, - input_file=input_file, - ) - - # Extract the data from the result - answer = result.get("data", []) - structured_output[output[PSKeys.NAME]] = answer - - # We do not support summary and eval for table. - # Hence returning the result - return structured_output - except Exception as e: - app.logger.error(f"Smart Table Extractor failed: {e}") - # Fall back to regular table extractor - app.logger.info("Falling back to regular table extractor") - - # Use regular table extractor (original code) - table_extractor: dict[str, Any] = PluginManager().get_plugin("table-extractor") - if not table_extractor: - raise APIError( - "Unable to extract table details. " - "Please contact admin to resolve this issue." - ) - fs_instance = AnswerPromptService._get_file_storage_instance(execution_source) - - try: - answer = table_extractor["entrypoint_cls"].run_table_extraction( - llm=llm, - table_settings=table_settings, - fs_instance=fs_instance, - prompt=prompt, - ) - structured_output[output[PSKeys.NAME]] = answer - # We do not support summary and eval for table. - # Hence returning the result - return structured_output - except table_extractor["exception_cls"] as e: - msg = f"Couldn't extract table. {e}" - raise APIError(message=msg) - - @staticmethod - def _get_file_storage_instance(execution_source) -> FileStorage: - fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL) - if execution_source == ExecutionSource.IDE.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.PERMANENT, - env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE, - ) - if execution_source == ExecutionSource.TOOL.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.SHARED_TEMPORARY, - env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE, - ) - - return fs_instance - - @staticmethod - def handle_json( - answer: str, - structured_output: dict[str, Any], - output: dict[str, Any], - log_events_id: str, - tool_id: str, - doc_name: str, - llm: LLM, - enable_highlight: bool = False, - enable_word_confidence: bool = False, - execution_source: str = ExecutionSource.IDE.value, - metadata: dict[str, Any] | None = None, - file_path: str = "", - ) -> None: - """Handle JSON responses from the LLM.""" - prompt_key = output[PSKeys.NAME] - if answer.lower() == "na": - structured_output[prompt_key] = None - else: - # Use the utility function to repair JSON with the best structure - parsed_data = repair_json_with_best_structure(answer) - - if isinstance(parsed_data, str): - err_msg = "Error parsing response to JSON" - app.logger.error(err_msg) - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_key, - "doc_name": doc_name, - }, - LogLevel.WARN, - RunLevel.RUN, - "Unable to parse JSON response from LLM, try using our" - " cloud / enterprise feature 'record' or 'table' type", - ) - structured_output[prompt_key] = {} - else: - # Get webhook configuration from output settings - webhook_enabled = output.get(PSKeys.ENABLE_POSTPROCESSING_WEBHOOK, False) - webhook_url = output.get(PSKeys.POSTPROCESSING_WEBHOOK_URL) - - # Get highlight data from metadata if available and highlighting is enabled - highlight_data = None - if enable_highlight and metadata and PSKeys.HIGHLIGHT_DATA in metadata: - highlight_data = metadata[PSKeys.HIGHLIGHT_DATA].get(prompt_key) - - # Process data (optionally via webhook) with safety guards and fallback - processed_data = parsed_data - updated_highlight_data = None - - if webhook_enabled: - if not webhook_url: - app.logger.warning( - "Postprocessing webhook enabled but URL missing; skipping." - ) - elif not _is_safe_public_url(webhook_url): - app.logger.warning( - "Postprocessing webhook URL is not allowed; skipping." - ) - else: - try: - processed_data, updated_highlight_data = postprocess_data( - parsed_data, - webhook_enabled=True, - webhook_url=webhook_url, - highlight_data=highlight_data, - timeout=60, - ) - except Exception as e: - app.logger.warning( - f"Postprocessing webhook failed: {e}. Using unprocessed data." - ) - - structured_output[prompt_key] = processed_data - - # Update metadata with processed highlight data if available and highlighting is enabled - if enable_highlight and metadata and updated_highlight_data is not None: - metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = ( - updated_highlight_data - ) - - @staticmethod - def extract_line_item( - tool_settings: dict[str, Any], - output: dict[str, Any], - structured_output: dict[str, Any], - llm: LLM, - file_path: str, - metadata: dict[str, str] | None, - execution_source: str, - ) -> dict[str, Any]: - line_item_extraction_plugin: dict[str, Any] = PluginManager().get_plugin( - "line-item-extraction" - ) - if not line_item_extraction_plugin: - raise APIError(PSKeys.PAID_FEATURE_MSG) - - extract_file_path = file_path - - # Read file content into context - fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL) - if execution_source == ExecutionSource.IDE.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.PERMANENT, - env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE, - ) - if execution_source == ExecutionSource.TOOL.value: - fs_instance = EnvHelper.get_storage( - storage_type=StorageType.SHARED_TEMPORARY, - env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE, - ) - - if not fs_instance.exists(extract_file_path): - raise FileNotFoundError( - f"The file at path '{extract_file_path}' does not exist." - ) - context = fs_instance.read(path=extract_file_path, encoding="utf-8", mode="r") - - prompt = AnswerPromptService.construct_prompt( - preamble=tool_settings.get(PSKeys.PREAMBLE, ""), - prompt=output["promptx"], - postamble=tool_settings.get(PSKeys.POSTAMBLE, ""), - grammar_list=tool_settings.get(PSKeys.GRAMMAR, []), - context=context, - platform_postamble="", - word_confidence_postamble="", - ) - try: - line_item_extraction = line_item_extraction_plugin["entrypoint_cls"]( - llm=llm, - tool_settings=tool_settings, - output=output, - prompt=prompt, - structured_output=structured_output, - ) - answer = line_item_extraction.run() - structured_output[output[PSKeys.NAME]] = answer - metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = [context] - return structured_output - except line_item_extraction_plugin["exception_cls"] as e: - msg = f"Couldn't extract table. {e}" - raise APIError(message=msg) diff --git a/prompt-service/src/unstract/prompt_service/services/extraction.py b/prompt-service/src/unstract/prompt_service/services/extraction.py deleted file mode 100644 index 76430657f9..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/extraction.py +++ /dev/null @@ -1,92 +0,0 @@ -from pathlib import Path -from typing import Any - -from unstract.prompt_service.constants import ExecutionSource -from unstract.prompt_service.constants import IndexingConstants as IKeys -from unstract.prompt_service.exceptions import ExtractionError -from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool -from unstract.prompt_service.utils.file_utils import FileUtils -from unstract.sdk1.adapters.exceptions import AdapterError -from unstract.sdk1.adapters.x2text.constants import X2TextConstants -from unstract.sdk1.adapters.x2text.llm_whisperer.src import LLMWhisperer -from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src import LLMWhispererV2 -from unstract.sdk1.utils.common import log_elapsed -from unstract.sdk1.utils.tool import ToolUtils -from unstract.sdk1.x2txt import TextExtractionResult, X2Text - - -class ExtractionService: - @staticmethod - @log_elapsed(operation="EXTRACTION") - def perform_extraction( - x2text_instance_id: str, - file_path: str, - run_id: str, - platform_key: str, - output_file_path: str | None = None, - enable_highlight: bool = False, - usage_kwargs: dict[Any, Any] = {}, - tags: list[str] | None = None, - execution_source: str | None = None, - tool_exec_metadata: dict[str, Any] | None = None, - execution_run_data_folder: str | None = None, - ) -> str: - extracted_text = "" - util = PromptServiceBaseTool(platform_key=platform_key) - x2text = X2Text( - tool=util, adapter_instance_id=x2text_instance_id, usage_kwargs=usage_kwargs - ) - fs = FileUtils.get_fs_instance(execution_source=execution_source) - try: - if enable_highlight and ( - isinstance(x2text.x2text_instance, LLMWhisperer) - or isinstance(x2text.x2text_instance, LLMWhispererV2) - ): - process_response: TextExtractionResult = x2text.process( - input_file_path=file_path, - output_file_path=output_file_path, - enable_highlight=enable_highlight, - tags=tags, - fs=fs, - ) - ExtractionService.update_exec_metadata( - fs, - execution_source, - tool_exec_metadata, - execution_run_data_folder, - process_response, - ) - else: - process_response: TextExtractionResult = x2text.process( - input_file_path=file_path, - output_file_path=output_file_path, - tags=tags, - fs=fs, - ) - extracted_text = process_response.extracted_text - return extracted_text - except AdapterError as e: - msg = f"Error from text extractor '{x2text.x2text_instance.get_name()}'. " - msg += str(e) - code = e.status_code if e.status_code != -1 else 500 - raise ExtractionError(msg, code=code) from e - - @staticmethod - def update_exec_metadata( - fs, - execution_source, - tool_exec_metadata, - execution_run_data_folder, - process_response, - ): - if execution_source == ExecutionSource.TOOL.value: - whisper_hash_value = process_response.extraction_metadata.whisper_hash - metadata = {X2TextConstants.WHISPER_HASH: whisper_hash_value} - for key, value in metadata.items(): - tool_exec_metadata[key] = value - metadata_path = str(Path(execution_run_data_folder) / IKeys.METADATA_FILE) - ToolUtils.dump_json( - file_to_dump=metadata_path, - json_to_dump=metadata, - fs=fs, - ) diff --git a/prompt-service/src/unstract/prompt_service/services/indexing.py b/prompt-service/src/unstract/prompt_service/services/indexing.py deleted file mode 100644 index b71a4f9897..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/indexing.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging - -from unstract.prompt_service.core.index_v2 import Index -from unstract.prompt_service.dto import ( - ChunkingConfig, - FileInfo, - InstanceIdentifiers, - ProcessingOptions, -) -from unstract.prompt_service.exceptions import APIError -from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool -from unstract.prompt_service.utils.file_utils import FileUtils -from unstract.sdk1.embedding import EmbeddingCompat -from unstract.sdk1.utils.indexing import IndexingUtils -from unstract.sdk1.vector_db import VectorDB - -logger = logging.getLogger(__name__) - - -class IndexingService: - @staticmethod - def index( - execution_source: str, - chunking_config: ChunkingConfig, - file_info: FileInfo, - instance_identifiers: InstanceIdentifiers, - processing_options: ProcessingOptions, - platform_key: str, - run_id: str, - extracted_text: str, - ) -> str: - try: - fs_instance = FileUtils.get_fs_instance(execution_source=execution_source) - util = PromptServiceBaseTool(platform_key=platform_key) - index: Index = Index( - tool=util, - run_id=run_id, - capture_metrics=True, - instance_identifiers=instance_identifiers, - chunking_config=chunking_config, - processing_options=processing_options, - ) - doc_id: str = IndexingUtils.generate_index_key( - vector_db=instance_identifiers.vector_db_instance_id, - embedding=instance_identifiers.embedding_instance_id, - file_path=file_info.file_path, - file_hash=file_info.file_hash, - x2text=instance_identifiers.x2text_instance_id, - chunk_size=str(chunking_config.chunk_size), - chunk_overlap=str(chunking_config.chunk_overlap), - tool=util, - fs=fs_instance, - ) - - # Skip embedding creation and indexing when chunk_size is 0 - # When chunk_size is 0, we don't need vector operations - if chunking_config.chunk_size == 0: - logger.info(f"Skipping indexing for chunk_size=0. Doc ID: {doc_id}") - return doc_id - - embedding = EmbeddingCompat( - adapter_instance_id=instance_identifiers.embedding_instance_id, - tool=util, - kwargs={ - **processing_options.usage_kwargs, - }, - ) - - vector_db = VectorDB( - tool=util, - adapter_instance_id=instance_identifiers.vector_db_instance_id, - embedding=embedding, - ) - - doc_id_found = index.is_document_indexed( - doc_id=doc_id, - embedding=embedding, - vector_db=vector_db, - ) - - # Index and return doc_id - index.perform_indexing( - vector_db=vector_db, - doc_id=doc_id, - extracted_text=extracted_text, - doc_id_found=doc_id_found, - ) - return doc_id - except Exception as e: - status_code = getattr(e, "status_code", 500) - raise APIError(f"Error while indexing: {str(e)}", code=status_code) from e - finally: - if "vector_db" in locals(): - vector_db.close() diff --git a/prompt-service/src/unstract/prompt_service/services/rentrolls_extractor/interface.py b/prompt-service/src/unstract/prompt_service/services/rentrolls_extractor/interface.py deleted file mode 100644 index 4bf6914211..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/rentrolls_extractor/interface.py +++ /dev/null @@ -1,70 +0,0 @@ -import logging -import os -from typing import Any - -import requests - -logger = logging.getLogger(__name__) - - -class RentRollExtractor: - """Main class for rent roll extraction.""" - - def __init__(self): - """Initialize the rent roll extractor.""" - logger.info("Initialized RentRollExtractor") - pass - - def process( - self, - extractor_settings: dict[str, Any], - extracted_data: str, - llm_config: dict[str, Any], - schema: str, - ) -> dict[str, Any]: - """Process the extraction with the given settings. - - Args: - extractor_settings: Dictionary containing extraction settings. - extracted_data: Extracted data from the document. - llm_config: Dictionary containing LLM configuration. - schema: Schema for the extraction. - - Returns: - Dict containing the extraction results. - """ - logger.info("Starting rent roll extraction process") - try: - # Load environment variables for rent roll service - rentroll_host = os.environ.get("RENTROLL_SERVICE_HOST", "http://localhost") - rentroll_port = os.environ.get("RENTROLL_SERVICE_PORT", "5003") - rentroll_url = f"{rentroll_host}:{rentroll_port}/api/extract-rentrolls" - - # Prepare the request payload - payload = { - "text": extracted_data, - "settings": extractor_settings, - "llm_config": llm_config, - "schema": schema, - } - - # Make HTTP call to rent roll service - logger.info(f"Calling rent roll service at: {rentroll_url}") - response = requests.post(rentroll_url, json=payload) - - if response.status_code == 200: - result = response.json() - - logger.info("Successfully received response from rent roll service") - return result - else: - error_text = response.text - logger.error( - f"Rent roll service returned status {response.status_code}: {error_text}" - ) - raise Exception( - f"Rent roll service error: {response.status_code} - {error_text}" - ) - except Exception as e: - logger.error(f"Error in rent roll extraction: {str(e)}", exc_info=True) - raise diff --git a/prompt-service/src/unstract/prompt_service/services/retrieval.py b/prompt-service/src/unstract/prompt_service/services/retrieval.py deleted file mode 100644 index 92ba46c930..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/retrieval.py +++ /dev/null @@ -1,156 +0,0 @@ -import datetime -from typing import Any - -from flask import current_app as app - -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.constants import RetrievalStrategy -from unstract.prompt_service.core.retrievers.automerging import AutomergingRetriever -from unstract.prompt_service.core.retrievers.fusion import FusionRetriever -from unstract.prompt_service.core.retrievers.keyword_table import KeywordTableRetriever -from unstract.prompt_service.core.retrievers.recursive import RecursiveRetrieval -from unstract.prompt_service.core.retrievers.router import RouterRetriever -from unstract.prompt_service.core.retrievers.simple import SimpleRetriever -from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever -from unstract.prompt_service.services.answer_prompt import AnswerPromptService -from unstract.prompt_service.utils.file_utils import FileUtils -from unstract.prompt_service.utils.metrics import Metrics -from unstract.sdk1.llm import LLM -from unstract.sdk1.vector_db import VectorDB - - -class RetrievalService: - @staticmethod - def perform_retrieval( # type:ignore - tool_settings: dict[str, Any], - output: dict[str, Any], - doc_id: str, - llm: LLM, - vector_db: VectorDB, - retrieval_type: str, - metadata: dict[str, Any], - chunk_size: int, - execution_source: str, - file_path: str, - context_retrieval_metrics: dict[str, Any], - ) -> tuple[str, list[str]]: - prompt_name = output.get(PSKeys.NAME, "") - vector_db_id = ( - getattr(vector_db, "_adapter_instance_id", None) if vector_db else None - ) - app.logger.info( - f"[Retrieval] prompt='{prompt_name}' doc_id={doc_id} " - f"chunk_size={chunk_size} method={'complete_context' if chunk_size == 0 else 'chunked'}" - + (f" vector_db={vector_db_id}" if vector_db_id else "") - ) - - context: list[str] - if chunk_size == 0: - context = RetrievalService.retrieve_complete_context( - execution_source=execution_source, - file_path=file_path, - context_retrieval_metrics=context_retrieval_metrics, - prompt_key=prompt_name, - ) - else: - context = RetrievalService.run_retrieval( - output=output, - doc_id=doc_id, - llm=llm, - vector_db=vector_db, - retrieval_type=retrieval_type, - context_retrieval_metrics=context_retrieval_metrics, - ) - answer = AnswerPromptService.construct_and_run_prompt( # type:ignore - tool_settings=tool_settings, - output=output, - llm=llm, - context="\n".join(context), - prompt="promptx", - metadata=metadata, - execution_source=execution_source, - file_path=file_path, - ) - return answer, context - - @staticmethod - def run_retrieval( # type:ignore - output: dict[str, Any], - doc_id: str, - llm: LLM, - vector_db: VectorDB, - retrieval_type: str, - context_retrieval_metrics: dict[str, Any], - ) -> list[str]: - context: set[str] - prompt = output[PSKeys.PROMPTX] - top_k = output[PSKeys.SIMILARITY_TOP_K] - prompt_key = output[PSKeys.NAME] - retrieval_start_time = datetime.datetime.now() - - # Map retrieval type to retriever class - retriever_map = { - RetrievalStrategy.SIMPLE.value: SimpleRetriever, - RetrievalStrategy.SUBQUESTION.value: SubquestionRetriever, - RetrievalStrategy.FUSION.value: FusionRetriever, - RetrievalStrategy.RECURSIVE.value: RecursiveRetrieval, - RetrievalStrategy.ROUTER.value: RouterRetriever, - RetrievalStrategy.KEYWORD_TABLE.value: KeywordTableRetriever, - RetrievalStrategy.AUTOMERGING.value: AutomergingRetriever, - } - - # Get the appropriate retriever class - retriever_class = retriever_map.get(retrieval_type) - if not retriever_class: - raise ValueError(f"Unknown retrieval type: {retrieval_type}") - - # Create and execute retriever - retriever = retriever_class( - vector_db=vector_db, - doc_id=doc_id, - prompt=prompt, - top_k=top_k, - llm=llm, - ) - context = retriever.retrieve() - elapsed = Metrics.elapsed_time(start_time=retrieval_start_time) - context_retrieval_metrics[prompt_key] = {"time_taken(s)": elapsed} - - app.logger.info( - f"[Retrieval] prompt='{prompt_key}' doc_id={doc_id} " - f"strategy='{retrieval_type}' top_k={top_k} chunks={len(context)} time={elapsed:.3f}s" - ) - - return list(context) - - @staticmethod - def retrieve_complete_context( - execution_source: str, - file_path: str, - context_retrieval_metrics: dict[str, Any], - prompt_key: str, - ) -> list[str]: - """Loads full context from raw file for zero chunk size retrieval. - - Args: - execution_source: Source of execution (e.g., "api", "workflow"). - file_path: Path to the extracted text file. - context_retrieval_metrics: Dict to store retrieval timing metrics - (modified in-place). - prompt_key: Name/identifier of the prompt for metrics tracking. - - Returns: - List containing the complete file content as a single string. - """ - fs_instance = FileUtils.get_fs_instance(execution_source=execution_source) - retrieval_start_time = datetime.datetime.now() - context = fs_instance.read(path=file_path, mode="r") - elapsed = Metrics.elapsed_time(start_time=retrieval_start_time) - context_retrieval_metrics[prompt_key] = {"time_taken(s)": elapsed} - - app.logger.info( - f"[Retrieval] prompt='{prompt_key}' complete_context " - f"chars={len(context)} time={elapsed:.3f}s" - ) - - return [context] diff --git a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py deleted file mode 100644 index 41ba342ac2..0000000000 --- a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py +++ /dev/null @@ -1,134 +0,0 @@ -from typing import Any - -from flask import current_app as app - -from unstract.prompt_service.constants import PromptServiceConstants as PSKeys -from unstract.prompt_service.constants import RunLevel, VariableType -from unstract.prompt_service.helpers.variable_replacement import ( - VariableReplacementHelper, -) -from unstract.prompt_service.utils.log import publish_log -from unstract.sdk1.constants import LogLevel - - -class VariableReplacementService: - @staticmethod - def is_variables_present(prompt_text: str) -> bool: - """Determines if variables are present in the prompt. - - Args: - prompt (str): Prompt to check - - Returns: - bool: True if variables are present else False - """ - return bool( - len(VariableReplacementHelper.extract_variables_from_prompt(prompt_text)) - ) - - @staticmethod - def replace_variables_in_prompt( - prompt: dict[str, Any], - structured_output: dict[str, Any], - log_events_id: str, - tool_id: str, - prompt_name: str, - doc_name: str, - custom_data: dict[str, Any] = None, - is_ide: bool = True, - ) -> str: - """Replaces variables in prompt. - - Args: - prompt (dict[str, Any]): Dict representing the prompt card - structured_output (dict[str, Any]): Structured data used for variable - replacement when variable map is not present in `prompt`. - log_events_id (str): UUID for the WS communication - tool_id (str): UUID for the prompt studio project - prompt_name (str): Name of the prompt being run - doc_name (str): Name of the document being run with - Returns: - prompt_text (str): Prompt with variables replaced - """ - app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}") - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.RUN, - "Replacing variables in prompt", - ) - # Initialized with the prompt because - # it is used in finally block - prompt_text = prompt[PSKeys.PROMPT] - try: - variable_map = prompt[PSKeys.VARIABLE_MAP] - prompt_text = VariableReplacementService._execute_variable_replacement( - prompt_text=prompt[PSKeys.PROMPT], - variable_map=variable_map, - custom_data=custom_data, - is_ide=is_ide, - ) - except KeyError: - # Executed incase of structured tool and - # APIs where we do not set the variable map - prompt_text = VariableReplacementService._execute_variable_replacement( - prompt_text=prompt_text, - variable_map=structured_output, - custom_data=custom_data, - is_ide=is_ide, - ) - finally: - publish_log( - log_events_id, - { - "tool_id": tool_id, - "prompt_key": prompt_name, - "doc_name": doc_name, - }, - LogLevel.DEBUG, - RunLevel.RUN, - f"Variables replaced in prompt with key :{prompt_name}", - ) - return prompt_text - - @staticmethod - def _execute_variable_replacement( - prompt_text: str, - variable_map: dict[str, Any], - custom_data: dict[str, Any] = None, - is_ide: bool = True, - ) -> str: - variables: list[str] = VariableReplacementHelper.extract_variables_from_prompt( - prompt_text=prompt_text - ) - for variable in variables: - variable_type = VariableReplacementHelper.identify_variable_type( - variable=variable - ) - if variable_type == VariableType.STATIC: - prompt_text = VariableReplacementHelper.replace_static_variable( - prompt=prompt_text, - structured_output=variable_map, - variable=variable, - ) - - elif variable_type == VariableType.DYNAMIC: - prompt_text = VariableReplacementHelper.replace_dynamic_variable( - prompt=prompt_text, - variable=variable, - structured_output=variable_map, - ) - - elif variable_type == VariableType.CUSTOM_DATA: - prompt_text = VariableReplacementHelper.replace_custom_data_variable( - prompt=prompt_text, - variable=variable, - custom_data=custom_data or {}, - is_ide=is_ide, - ) - return prompt_text diff --git a/prompt-service/src/unstract/prompt_service/tests/conftest.py b/prompt-service/src/unstract/prompt_service/tests/conftest.py deleted file mode 100644 index c04fa27d3a..0000000000 --- a/prompt-service/src/unstract/prompt_service/tests/conftest.py +++ /dev/null @@ -1,29 +0,0 @@ -import pytest -from dotenv import load_dotenv -from flask import Flask -from flask_wtf.csrf import CSRFProtect - -from unstract.prompt_service.controllers.extraction import extraction_bp -from unstract.prompt_service.controllers.indexing import indexing_bp - -# Load test environment variables -load_dotenv(".env.test") - - -@pytest.fixture -def client(): - app = Flask(__name__) - app.register_blueprint(indexing_bp) - app.register_blueprint(extraction_bp) - app.config.update( - { - "TESTING": True, - "DEBUG": True, - "PROPAGATE_EXCEPTIONS": True, - "WTF_CSRF_ENABLED": False, - } - ) - csrf = CSRFProtect() - csrf.init_app(app) # Compliant - with app.test_client() as client: - yield client diff --git a/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf b/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf deleted file mode 100644 index 960486fddb..0000000000 Binary files a/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf and /dev/null differ diff --git a/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py b/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py deleted file mode 100644 index 3adaa820d5..0000000000 --- a/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py +++ /dev/null @@ -1,85 +0,0 @@ -import json -import os - -from dotenv import load_dotenv -from flask.testing import FlaskClient - -from unstract.prompt_service.helpers.auth import AuthHelper - -load_dotenv(".env.test") - - -def test_index(client: FlaskClient, mocker): - # Mock the AuthHelper to bypass authentication - mocker.patch.object(AuthHelper, "auth_required", lambda x: x) - mocker.patch.object( - AuthHelper, - "get_token_from_auth_header", - return_value=os.getenv("TEST_PLATFORM_KEY"), - ) - - # Define the payload - payload = { - "tool_id": "mock_tool_id", - "embedding_instance_id": os.getenv("TEST_EMBEDDING_ID"), - "vector_db_instance_id": os.getenv("TEST_VECTOR_DB_ID"), - "x2text_instance_id": os.getenv("TEST_X2TEXT_ID"), - "chunk_size": os.getenv("TEST_CHUNK_SIZE"), - "chunk_overlap": os.getenv("TEST_CHUNK_OVERLAP"), - "execution_source": "ide", - "run_id": os.getenv("TEST_RUN_ID"), - "extracted_text": os.getenv("TEST_EXTRACTED_TEXT"), - "usage_kwargs": {}, - "tags": os.getenv("TEST_TAGS"), - "reindex": False, - "enable_highlight": False, - } - - # Send a POST request to the index endpoint - try: - response = client.post( - "/index", data=json.dumps(payload), content_type="application/json" - ) - - except Exception as e: - import traceback - - traceback.print_exc() - assert False, f"Test failed due to: {e}" - # Assert the response - print(response.data.decode()) # Full response content - assert response.status_code == 200 - response_data = json.loads(response.data) - assert "doc_id" in response_data - - -def test_extract(client: FlaskClient, mocker): - # Mock the AuthHelper to bypass authentication - mocker.patch.object(AuthHelper, "auth_required", lambda x: x) - mocker.patch.object( - AuthHelper, - "get_token_from_auth_header", - return_value=os.getenv("TEST_PLATFORM_KEY"), - ) - - # Define the payload - payload = { - "x2text_instance_id": os.getenv("X2TEXT_INSTANCE_ID"), - "file_path": os.getenv("TEST_SOURCE_FILE_PATH"), - "execution_source": "ide", - "run_id": "mock_run_id", - "tags": "mock_tag", - "output_file_path": os.getenv("TEST_OUTPUT_FILE_PATH"), - "enable_highlight": False, - "usage_kwargs": {}, - } - - # Send a POST request to the extract endpoint - response = client.post( - "/extract", data=json.dumps(payload), content_type="application/json" - ) - - # Assert the response - assert response.status_code == 200 - response_data = json.loads(response.data) - assert "extracted_text" in response_data diff --git a/prompt-service/src/unstract/prompt_service/tests/sample.env.test b/prompt-service/src/unstract/prompt_service/tests/sample.env.test deleted file mode 100644 index 7554cc2157..0000000000 --- a/prompt-service/src/unstract/prompt_service/tests/sample.env.test +++ /dev/null @@ -1,24 +0,0 @@ -TEST_PLATFORM_KEY= -TEST_EMBEDDING_ID=d -TEST_VECTOR_DB_ID= -TEST_X2TEXT_ID= -TEST_TOOL_ID=mock_tool_id -TEST_FILE_PATH=unstract/prompt-studio-data/sample1.pdf -TEST_CHUNK_SIZE=512 -TEST_CHUNK_OVERLAP=128 -TEST_EXECUTION_SOURCE=ide -TEST_RUN_ID=mock_run_id -TEST_EXTRACTED_TEXT=mock_extracted_text -TEST_TAGS=mock_tag1,mock_tag2 -TEST_SOURCE_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf -TEST_OUTPUT_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/output/output.txt - -# Platform Service -# Platform service is expected to be up and running for this integration test to run -PLATFORM_SERVICE_HOST=http://127.0.0.1/ -PLATFORM_SERVICE_PORT=3001 - -#Remote storage related envs -PERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}' -TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}' -REMOTE_PROMPT_STUDIO_FILE_PATH="unstract/prompt-studio-data/" diff --git a/prompt-service/src/unstract/prompt_service/tests/unit/conftest.py b/prompt-service/src/unstract/prompt_service/tests/unit/conftest.py deleted file mode 100644 index b78a6985a1..0000000000 --- a/prompt-service/src/unstract/prompt_service/tests/unit/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Pytest configuration for unit tests. - -Unit tests should not require external dependencies or the full app. -This conftest intentionally does NOT import Flask app components. - -WARNING: This file is NOT auto-loaded when running via tox because ---noconftest is used to skip the parent tests/conftest.py (which -imports Flask blueprints and triggers the full adapter import chain). -If you add shared fixtures here, either remove --noconftest from -tox.ini and fix the parent conftest's eager imports, or define -fixtures directly in test files. -""" diff --git a/prompt-service/src/unstract/prompt_service/tests/unit/test_retriever_llm.py b/prompt-service/src/unstract/prompt_service/tests/unit/test_retriever_llm.py deleted file mode 100644 index 377988b20c..0000000000 --- a/prompt-service/src/unstract/prompt_service/tests/unit/test_retriever_llm.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Unit tests for RetrieverLLM bridge class and BaseRetriever.llm property.""" - -import sys -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from llama_index.core.base.llms.types import ( - ChatMessage, - ChatResponse, - CompletionResponse, - LLMMetadata, - MessageRole, -) -from llama_index.core.llms.llm import LLM as LlamaIndexBaseLLM # noqa: N811 -from unstract.prompt_service.core.retrievers.retriever_llm import RetrieverLLM -from unstract.sdk1.llm import LLM, LLMCompat -from unstract.sdk1.llm import ChatMessage as EmulatedChatMessage -from unstract.sdk1.llm import ChatResponse as EmulatedChatResponse -from unstract.sdk1.llm import CompletionResponse as EmulatedCompletionResponse -from unstract.sdk1.llm import LLMMetadata as EmulatedLLMMetadata -from unstract.sdk1.llm import MessageRole as EmulatedMessageRole - -# ── Fixtures ───────────────────────────────────────────────────────────────── - - -@pytest.fixture -def mock_sdk1_llm(): - """Create a mock SDK1 LLM instance with adapter attributes.""" - llm = MagicMock(spec=LLM) - llm._adapter_id = "openai" - llm._adapter_metadata = {"api_key": "test-key"} - llm._adapter_instance_id = "inst-123" - llm._tool = MagicMock() - llm._usage_kwargs = {"run_id": "test-run"} - llm._capture_metrics = False - llm.get_model_name.return_value = "gpt-4" - llm.platform_kwargs = {"run_id": "test-run"} - return llm - - -@pytest.fixture -def mock_compat(): - """Create a mock LLMCompat instance.""" - compat = MagicMock(spec=LLMCompat) - compat.get_model_name.return_value = "gpt-4" - compat.metadata = EmulatedLLMMetadata( - is_chat_model=True, model_name="gpt-4" - ) - return compat - - -# ── BaseRetriever.llm property tests ──────────────────────────────────────── -# BaseRetriever imports VectorDB which triggers the full adapter registration -# chain (Pinecone, Milvus, etc.). We stub unstract.sdk1.vector_db in -# sys.modules so that BaseRetriever can be imported without those heavy deps. - - -@pytest.fixture -def _stub_vector_db(): - """Temporarily stub VectorDB module for BaseRetriever import.""" - stub = MagicMock() - key = "unstract.sdk1.vector_db" - original = sys.modules.get(key) - sys.modules[key] = stub - # Also ensure the base_retriever module is re-imported with the stub. - mod_key = ( - "unstract.prompt_service.core.retrievers.base_retriever" - ) - sys.modules.pop(mod_key, None) - yield stub - # Restore original module (or remove stub). - if original is not None: - sys.modules[key] = original - else: - sys.modules.pop(key, None) - sys.modules.pop(mod_key, None) - - -@pytest.fixture -def base_retriever_cls(_stub_vector_db): - """Import and return BaseRetriever with VectorDB stubbed.""" - from unstract.prompt_service.core.retrievers.base_retriever import ( - BaseRetriever, - ) - - return BaseRetriever - - -class TestBaseRetrieverLlmProperty: - """Tests for BaseRetriever.llm lazy property.""" - - def test_returns_none_when_no_llm_provided( - self, base_retriever_cls - ): - """llm property should return None when constructed without LLM.""" - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - ) - assert retriever.llm is None - - def test_returns_retriever_llm_instance( - self, base_retriever_cls, mock_sdk1_llm - ): - """llm property should return a RetrieverLLM wrapping SDK1 LLM.""" - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - llm=mock_sdk1_llm, - ) - result = retriever.llm - - assert isinstance(result, RetrieverLLM) - assert isinstance(result, LlamaIndexBaseLLM) - - def test_lazily_creates_retriever_llm( - self, base_retriever_cls, mock_sdk1_llm - ): - """RetrieverLLM should not be created until .llm is accessed.""" - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - llm=mock_sdk1_llm, - ) - # Before accessing .llm, the internal cache should be None. - assert retriever._retriever_llm is None - - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - _ = retriever.llm - - # After access, it should be populated. - assert retriever._retriever_llm is not None - - def test_caches_retriever_llm_across_accesses( - self, base_retriever_cls, mock_sdk1_llm - ): - """Repeated .llm accesses should return the same instance.""" - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - llm=mock_sdk1_llm, - ) - first = retriever.llm - second = retriever.llm - - assert first is second - - def test_from_llm_called_once_on_repeated_access( - self, base_retriever_cls, mock_sdk1_llm - ): - """LLMCompat.from_llm should only be called once across accesses.""" - with patch.object( - LLMCompat, "from_llm", return_value=MagicMock() - ) as mock_factory: - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - llm=mock_sdk1_llm, - ) - _ = retriever.llm - _ = retriever.llm - _ = retriever.llm - - mock_factory.assert_called_once() - - def test_raw_llm_still_accessible( - self, base_retriever_cls, mock_sdk1_llm - ): - """The raw SDK1 LLM should remain accessible via _llm.""" - retriever = base_retriever_cls( - vector_db=MagicMock(), - prompt="test", - doc_id="doc-1", - top_k=5, - llm=mock_sdk1_llm, - ) - assert retriever._llm is mock_sdk1_llm - - -# ── RetrieverLLM tests ─────────────────────────────────────────────────────── - - -class TestRetrieverLLM: - """Tests for the RetrieverLLM bridge class.""" - - def test_isinstance_llama_index_llm(self, mock_sdk1_llm): - """RetrieverLLM must pass llama-index's isinstance check.""" - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - assert isinstance(retriever_llm, LlamaIndexBaseLLM) - - def test_uses_from_llm_factory(self, mock_sdk1_llm): - """RetrieverLLM should use LLMCompat.from_llm, not private attrs.""" - with patch.object( - LLMCompat, "from_llm", return_value=MagicMock() - ) as mock_factory: - RetrieverLLM(llm=mock_sdk1_llm) - mock_factory.assert_called_once_with(mock_sdk1_llm) - - def test_metadata_returns_llama_index_type( - self, mock_sdk1_llm, mock_compat - ): - """Metadata property should return llama-index LLMMetadata.""" - with patch.object(LLMCompat, "from_llm", return_value=mock_compat): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - meta = retriever_llm.metadata - assert isinstance(meta, LLMMetadata) - assert meta.is_chat_model is True - assert meta.model_name == "gpt-4" - - def test_chat_delegates_and_converts_types( - self, mock_sdk1_llm, mock_compat - ): - """chat() should delegate to compat and return llama-index types.""" - mock_compat.chat.return_value = EmulatedChatResponse( - message=EmulatedChatMessage( - role=EmulatedMessageRole.ASSISTANT, - content="Hello from LLM", - ), - raw={"id": "resp-1"}, - ) - with patch.object(LLMCompat, "from_llm", return_value=mock_compat): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - messages = [ - ChatMessage(role=MessageRole.USER, content="Hi") - ] - result = retriever_llm.chat(messages) - - assert isinstance(result, ChatResponse) - assert result.message.content == "Hello from LLM" - assert result.raw == {"id": "resp-1"} - mock_compat.chat.assert_called_once() - - def test_complete_delegates_and_converts_types( - self, mock_sdk1_llm, mock_compat - ): - """complete() should delegate to compat and return llama-index types.""" - mock_compat.complete.return_value = EmulatedCompletionResponse( - text="Completed text", raw={"id": "resp-2"} - ) - with patch.object(LLMCompat, "from_llm", return_value=mock_compat): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - result = retriever_llm.complete("Test prompt") - - assert isinstance(result, CompletionResponse) - assert result.text == "Completed text" - assert result.raw == {"id": "resp-2"} - mock_compat.complete.assert_called_once() - - @pytest.mark.asyncio - async def test_achat_delegates_and_converts_types( - self, mock_sdk1_llm, mock_compat - ): - """achat() should delegate to compat and return llama-index types.""" - mock_compat.achat = AsyncMock( - return_value=EmulatedChatResponse( - message=EmulatedChatMessage( - role=EmulatedMessageRole.ASSISTANT, - content="Async hello", - ), - raw={"id": "resp-3"}, - ) - ) - with patch.object(LLMCompat, "from_llm", return_value=mock_compat): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - messages = [ - ChatMessage(role=MessageRole.USER, content="Hi async") - ] - result = await retriever_llm.achat(messages) - - assert isinstance(result, ChatResponse) - assert result.message.content == "Async hello" - - @pytest.mark.asyncio - async def test_acomplete_delegates_and_converts_types( - self, mock_sdk1_llm, mock_compat - ): - """acomplete() should delegate to compat and return llama-index types.""" - mock_compat.acomplete = AsyncMock( - return_value=EmulatedCompletionResponse( - text="Async completed", raw={"id": "resp-4"} - ) - ) - with patch.object(LLMCompat, "from_llm", return_value=mock_compat): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - result = await retriever_llm.acomplete("Async prompt") - - assert isinstance(result, CompletionResponse) - assert result.text == "Async completed" - - def test_stream_chat_not_implemented(self, mock_sdk1_llm): - """stream_chat() should raise NotImplementedError.""" - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - with pytest.raises(NotImplementedError): - retriever_llm.stream_chat([]) - - def test_stream_complete_not_implemented(self, mock_sdk1_llm): - """stream_complete() should raise NotImplementedError.""" - with patch.object(LLMCompat, "from_llm", return_value=MagicMock()): - retriever_llm = RetrieverLLM(llm=mock_sdk1_llm) - with pytest.raises(NotImplementedError): - retriever_llm.stream_complete("prompt") diff --git a/prompt-service/src/unstract/prompt_service/utils/db_utils.py b/prompt-service/src/unstract/prompt_service/utils/db_utils.py deleted file mode 100644 index a350f28061..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/db_utils.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Any - -from unstract.prompt_service.constants import DBTableV2 -from unstract.prompt_service.extensions import db, db_context -from unstract.prompt_service.utils.env_loader import get_env_or_die - -DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract") - - -class DBUtils: - @classmethod - def get_organization_from_bearer_token(cls, token: str) -> tuple[int | None, str]: - """Retrieve organization ID and identifier using a bearer token. - - Args: - token (str): The bearer token (platform key). - - Returns: - tuple[int, str]: organization uid and organization identifier - """ - platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}' - organization_table = f'"{DB_SCHEMA}".{DBTableV2.ORGANIZATION}' - - organization_uid: int | None = cls.execute_query( - f"SELECT organization_id FROM {platform_key_table} WHERE key=%s", (token,) - ) - if organization_uid is None: - return None, None - - organization_identifier: str | None = cls.execute_query( - f"SELECT organization_id FROM {organization_table} WHERE id=%s", - (organization_uid,), - ) - return organization_uid, organization_identifier - - @classmethod - def execute_query(cls, query: str, params: tuple = ()) -> Any: - with db_context(): - cursor = db.execute_sql(query, params) - result_row = cursor.fetchone() - cursor.close() - if not result_row or len(result_row) == 0: - return None - return result_row[0] diff --git a/prompt-service/src/unstract/prompt_service/utils/env_loader.py b/prompt-service/src/unstract/prompt_service/utils/env_loader.py deleted file mode 100644 index 481a3eb34a..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/env_loader.py +++ /dev/null @@ -1,20 +0,0 @@ -import os - - -def get_env_or_die(env_key: str, default: str | None = None) -> str: - """Get the value of an environment variable or raise an error if it is not set. - - Args: - env_key (str): Name of the environment variable. - default (Optional[str]): Default value if the variable is not set. - - Returns: - str: The value of the environment variable. - - Raises: - ValueError: If the variable is not set. - """ - env_value = os.environ.get(env_key, default=default) - if env_value is None: - raise ValueError(f"Env variable {env_key} is required") - return env_value diff --git a/prompt-service/src/unstract/prompt_service/utils/file_utils.py b/prompt-service/src/unstract/prompt_service/utils/file_utils.py deleted file mode 100644 index ceb32acab8..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/file_utils.py +++ /dev/null @@ -1,33 +0,0 @@ -from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys -from unstract.sdk1.file_storage import FileStorage -from unstract.sdk1.file_storage.constants import StorageType -from unstract.sdk1.file_storage.env_helper import EnvHelper - - -class FileUtils: - @staticmethod - def get_fs_instance(execution_source: str) -> FileStorage: - """Returns a FileStorage instance based on the execution source. - - Args: - execution_source (str): The source from which the execution is triggered. - - Returns: - FileStorage: The file storage instance - Permanent/Shared temporary - - Raises: - ValueError: If the execution source is invalid. - """ - if execution_source == ExecutionSource.IDE.value: - return EnvHelper.get_storage( - storage_type=StorageType.PERMANENT, - env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE, - ) - - if execution_source == ExecutionSource.TOOL.value: - return EnvHelper.get_storage( - storage_type=StorageType.SHARED_TEMPORARY, - env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE, - ) - - raise ValueError(f"Invalid execution source: {execution_source}") diff --git a/prompt-service/src/unstract/prompt_service/utils/json_repair_helper.py b/prompt-service/src/unstract/prompt_service/utils/json_repair_helper.py deleted file mode 100644 index c15918dc26..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/json_repair_helper.py +++ /dev/null @@ -1,64 +0,0 @@ -"""JSON repair utility functions.""" - -from typing import Any - -from json_repair import repair_json - - -def repair_json_with_best_structure(json_str: str) -> Any: - """Intelligently repair JSON string using the best parsing strategy. - - This function attempts to parse JSON in two ways: - 1. As-is (could be valid object, array, or partial JSON) - 2. With array wrapping (useful for comma-separated objects) - - It chooses the result based on structural integrity rather than string length. - - Args: - json_str: The JSON string to repair - - Returns: - The parsed JSON object with the best structure - """ - # Attempt parsing as-is - parsed_as_is = repair_json(json_str=json_str, return_objects=True, ensure_ascii=False) - - # Attempt parsing with array wrap - parsed_with_wrap = repair_json( - json_str="[" + json_str, return_objects=True, ensure_ascii=False - ) - - # If both results are strings, return the as-is result - if isinstance(parsed_as_is, str) and isinstance(parsed_with_wrap, str): - return parsed_as_is - - # If only one is a string, return the non-string result - if isinstance(parsed_as_is, str): - return parsed_with_wrap - if isinstance(parsed_with_wrap, str): - return parsed_as_is - - # Both are valid structures - choose based on structure analysis - # If parsed_with_wrap is a list with exactly one element that equals parsed_as_is, - # then the original was already valid and wrapping just added unnecessary array - if ( - isinstance(parsed_with_wrap, list) - and len(parsed_with_wrap) == 1 - and parsed_with_wrap[0] == parsed_as_is - ): - return parsed_as_is - - # If parsed_as_is is a valid structure (dict or list), prefer it - # unless parsed_with_wrap provides a more complete structure - if isinstance(parsed_as_is, (dict, list)): - # Check if the wrapped version provides multiple objects that were - # incorrectly concatenated in the original (e.g., {},{},{}) - if isinstance(parsed_with_wrap, list) and len(parsed_with_wrap) > 1: - # The original likely had multiple comma-separated objects - return parsed_with_wrap - else: - # The original was already a valid structure - return parsed_as_is - - # Default to wrapped version if we can't determine otherwise - return parsed_with_wrap diff --git a/prompt-service/src/unstract/prompt_service/utils/log.py b/prompt-service/src/unstract/prompt_service/utils/log.py deleted file mode 100644 index a6ff8236a6..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/log.py +++ /dev/null @@ -1,26 +0,0 @@ -from unstract.core.pubsub_helper import LogPublisher -from unstract.prompt_service.constants import RunLevel -from unstract.sdk1.constants import LogLevel - - -def publish_log( - log_events_id: str, - component: dict[str, str], - level: LogLevel, - state: RunLevel, - message: str, -) -> None: - """Publishes a log to the web socket. - - Args: - log_events_id (str): UUID for the connection - component (dict[str, str]): Dict of tool_id, doc_name and prompt_name to - log context - level (LogLevel): Log level, one of INFO, WARNING, DEBUG, ERROR - state (RunLevel): Run level, one of EVAL, RUN, CHALLENGE, TABLE_EXTRACTION - message (str): Message to log - """ - LogPublisher.publish( - log_events_id, - LogPublisher.log_prompt(component, level.value, state.value, message), - ) diff --git a/prompt-service/src/unstract/prompt_service/utils/metrics.py b/prompt-service/src/unstract/prompt_service/utils/metrics.py deleted file mode 100644 index b7b419d337..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/metrics.py +++ /dev/null @@ -1,8 +0,0 @@ -import datetime - - -class Metrics: - @staticmethod - def elapsed_time(start_time) -> float: - """Returns the elapsed time since the process was started.""" - return (datetime.datetime.now() - start_time).total_seconds() diff --git a/prompt-service/src/unstract/prompt_service/utils/request.py b/prompt-service/src/unstract/prompt_service/utils/request.py deleted file mode 100644 index 8a5a82d462..0000000000 --- a/prompt-service/src/unstract/prompt_service/utils/request.py +++ /dev/null @@ -1,66 +0,0 @@ -from enum import Enum -from typing import Any - -import requests as pyrequests -from flask import current_app as app -from requests.exceptions import RequestException - -from unstract.prompt_service.exceptions import APIError, BadRequest, MissingFieldError - - -class HTTPMethod(str, Enum): - GET = "GET" - POST = "POST" - DELETE = "DELETE" - PUT = "PUT" - PATCH = "PATCH" - - -def make_http_request( - verb: HTTPMethod, - url: str, - data: dict[str, Any] | None = None, - headers: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, -) -> str: - """Generic helper function to help make a HTTP request.""" - try: - if verb == HTTPMethod.GET: - response = pyrequests.get(url, params=params, headers=headers) - elif verb == HTTPMethod.POST: - response = pyrequests.post(url, json=data, params=params, headers=headers) - elif verb == HTTPMethod.DELETE: - response = pyrequests.delete(url, params=params, headers=headers) - else: - raise ValueError("Invalid HTTP verb. Supported verbs: GET, POST, DELETE") - - response.raise_for_status() - return_val: str = ( - response.json() - if response.headers.get("content-type") == "application/json" - else response.text - ) - return return_val - except RequestException as e: - app.logger.error(f"HTTP request error: {e}") - # Extract status code from requests exception if available - status_code = None - if getattr(e, "response", None) is not None: - status_code = getattr(e.response, "status_code", None) - raise APIError( - f"HTTP {verb.value} request to {url} failed: {e!s}", - code=status_code or 500, - ) from e - except Exception as e: - app.logger.error(f"An unexpected error occurred: {e}") - raise e - - -def validate_request_payload(payload, required_fields): - if not payload: - raise BadRequest() - - # Validate required fields - missing_fields = [field for field in required_fields if field not in payload] - if missing_fields: - raise MissingFieldError(missing_fields) diff --git a/prompt-service/uv.lock b/prompt-service/uv.lock deleted file mode 100644 index ee7052d091..0000000000 --- a/prompt-service/uv.lock +++ /dev/null @@ -1,3043 +0,0 @@ -version = 1 -revision = 3 -requires-python = "==3.12.*" - -[[package]] -name = "adlfs" -version = "2024.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "azure-core" }, - { name = "azure-datalake-store" }, - { name = "azure-identity" }, - { name = "azure-storage-blob" }, - { name = "fsspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/1e/6d5146676044247af566fa5843b335b1a647e6446070cec9c8b61c31b369/adlfs-2024.7.0.tar.gz", hash = "sha256:106995b91f0eb5e775bcd5957d180d9a14faef3271a063b1f65c66fd5ab05ddf", size = 48588, upload-time = "2024-07-22T12:10:33.849Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/51/a71c457bd0bc8af3e522b6999ff300852c7c446e384fd9904b0794f875df/adlfs-2024.7.0-py3-none-any.whl", hash = "sha256:2005c8e124fda3948f2a6abb2dbebb2c936d2d821acaca6afd61932edfa9bc07", size = 41349, upload-time = "2024-07-22T12:10:32.226Z" }, -] - -[[package]] -name = "aiobotocore" -version = "2.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "aioitertools" }, - { name = "botocore" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/d2/d7e46bcc4c0b5b8e751092824d6ca9af5928adae0f864336e43c7f7a436a/aiobotocore-2.13.1.tar.gz", hash = "sha256:134f9606c2f91abde38cbc61c3241113e26ff244633e0c31abb7e09da3581c9b", size = 104475, upload-time = "2024-06-24T18:30:36.509Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/07/42f884c1600169e4267575cdd261c75dea31782d8fd877bbea358d559416/aiobotocore-2.13.1-py3-none-any.whl", hash = "sha256:1bef121b99841ee3cc788e4ed97c332ba32353b1f00e886d1beb3aae95520858", size = 76864, upload-time = "2024-06-24T18:30:33.379Z" }, -] - -[package.optional-dependencies] -boto3 = [ - { name = "boto3" }, -] - -[[package]] -name = "aiohappyeyeballs" -version = "2.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, -] - -[[package]] -name = "aiohttp" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "typing-extensions" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, - { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, - { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, - { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, - { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, - { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, - { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, - { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, - { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, - { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, - { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, - { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, - { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, - { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, - { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, - { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, -] - -[[package]] -name = "aioitertools" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/de/38491a84ab323b47c7f86e94d2830e748780525f7a10c8600b67ead7e9ea/aioitertools-0.12.0.tar.gz", hash = "sha256:c2a9055b4fbb7705f561b9d86053e8af5d10cc845d22c32008c43490b2d8dd6b", size = 19369, upload-time = "2024-09-02T03:33:40.349Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/13/58b70a580de00893223d61de8fea167877a3aed97d4a5e1405c9159ef925/aioitertools-0.12.0-py3-none-any.whl", hash = "sha256:fc1f5fac3d737354de8831cbba3eb04f79dd649d8f3afb4c5b114925e662a796", size = 24345, upload-time = "2024-09-02T03:34:59.454Z" }, -] - -[[package]] -name = "aiosignal" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, -] - -[[package]] -name = "aiosqlite" -version = "0.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7d/8bca2bf9a247c2c5dfeec1d7a5f40db6518f88d314b8bca9da29670d2671/aiosqlite-0.21.0.tar.gz", hash = "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3", size = 13454, upload-time = "2025-02-03T07:30:16.235Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, -] - -[[package]] -name = "amqp" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "anyio" -version = "4.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, -] - -[[package]] -name = "asyncpg" -version = "0.30.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2f/4c/7c991e080e106d854809030d8584e15b2e996e26f16aee6d757e387bc17d/asyncpg-0.30.0.tar.gz", hash = "sha256:c551e9928ab6707602f44811817f82ba3c446e018bfe1d3abecc8ba5f3eac851", size = 957746, upload-time = "2024-10-20T00:30:41.127Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/64/9d3e887bb7b01535fdbc45fbd5f0a8447539833b97ee69ecdbb7a79d0cb4/asyncpg-0.30.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c902a60b52e506d38d7e80e0dd5399f657220f24635fee368117b8b5fce1142e", size = 673162, upload-time = "2024-10-20T00:29:41.88Z" }, - { url = "https://files.pythonhosted.org/packages/6e/eb/8b236663f06984f212a087b3e849731f917ab80f84450e943900e8ca4052/asyncpg-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aca1548e43bbb9f0f627a04666fedaca23db0a31a84136ad1f868cb15deb6e3a", size = 637025, upload-time = "2024-10-20T00:29:43.352Z" }, - { url = "https://files.pythonhosted.org/packages/cc/57/2dc240bb263d58786cfaa60920779af6e8d32da63ab9ffc09f8312bd7a14/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c2a2ef565400234a633da0eafdce27e843836256d40705d83ab7ec42074efb3", size = 3496243, upload-time = "2024-10-20T00:29:44.922Z" }, - { url = "https://files.pythonhosted.org/packages/f4/40/0ae9d061d278b10713ea9021ef6b703ec44698fe32178715a501ac696c6b/asyncpg-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1292b84ee06ac8a2ad8e51c7475aa309245874b61333d97411aab835c4a2f737", size = 3575059, upload-time = "2024-10-20T00:29:46.891Z" }, - { url = "https://files.pythonhosted.org/packages/c3/75/d6b895a35a2c6506952247640178e5f768eeb28b2e20299b6a6f1d743ba0/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5712350388d0cd0615caec629ad53c81e506b1abaaf8d14c93f54b35e3595a", size = 3473596, upload-time = "2024-10-20T00:29:49.201Z" }, - { url = "https://files.pythonhosted.org/packages/c8/e7/3693392d3e168ab0aebb2d361431375bd22ffc7b4a586a0fc060d519fae7/asyncpg-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db9891e2d76e6f425746c5d2da01921e9a16b5a71a1c905b13f30e12a257c4af", size = 3641632, upload-time = "2024-10-20T00:29:50.768Z" }, - { url = "https://files.pythonhosted.org/packages/32/ea/15670cea95745bba3f0352341db55f506a820b21c619ee66b7d12ea7867d/asyncpg-0.30.0-cp312-cp312-win32.whl", hash = "sha256:68d71a1be3d83d0570049cd1654a9bdfe506e794ecc98ad0873304a9f35e411e", size = 560186, upload-time = "2024-10-20T00:29:52.394Z" }, - { url = "https://files.pythonhosted.org/packages/7e/6b/fe1fad5cee79ca5f5c27aed7bd95baee529c1bf8a387435c8ba4fe53d5c1/asyncpg-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a0292c6af5c500523949155ec17b7fe01a00ace33b68a476d6b5059f9630305", size = 621064, upload-time = "2024-10-20T00:29:53.757Z" }, -] - -[[package]] -name = "attrs" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, -] - -[[package]] -name = "authlib" -version = "1.6.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/30/6691fdc63b35f54a5a65e04fa1e59d827f4d4e8f4a39678ba7d3088ce0c8/authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd", size = 165368, upload-time = "2026-05-04T08:11:31.826Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/51/9b0b5cd4cf683a02db937a6f9bbebcdc9c56558a7bb3763ce7d3512103c3/authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab", size = 244473, upload-time = "2026-05-04T08:11:30.354Z" }, -] - -[[package]] -name = "azure-core" -version = "1.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, -] - -[[package]] -name = "azure-datalake-store" -version = "0.0.53" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "msal" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/ff/61369d06422b5ac48067215ff404841342651b14a89b46c8d8e1507c8f17/azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393", size = 71430, upload-time = "2023-05-10T21:17:05.665Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/2a/75f56b14f115189155cf12e46b366ad1fe3357af5a1a7c09f7446662d617/azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b", size = 55308, upload-time = "2023-05-10T21:17:02.629Z" }, -] - -[[package]] -name = "azure-identity" -version = "1.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "msal" }, - { name = "msal-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4e/9e/4c9682a286c3c89e437579bd9f64f311020e5125c1321fd3a653166b5716/azure_identity-1.25.0.tar.gz", hash = "sha256:4177df34d684cddc026e6cf684e1abb57767aa9d84e7f2129b080ec45eee7733", size = 278507, upload-time = "2025-09-12T01:30:04.418Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/54/81683b6756676a22e037b209695b08008258e603f7e47c56834029c5922a/azure_identity-1.25.0-py3-none-any.whl", hash = "sha256:becaec086bbdf8d1a6aa4fb080c2772a0f824a97d50c29637ec8cc4933f1e82d", size = 190861, upload-time = "2025-09-12T01:30:06.474Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/96/95/3e3414491ce45025a1cde107b6ae72bf72049e6021597c201cd6a3029b9a/azure_storage_blob-12.26.0.tar.gz", hash = "sha256:5dd7d7824224f7de00bfeb032753601c982655173061e242f13be6e26d78d71f", size = 583332, upload-time = "2025-07-16T21:34:07.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/64/63dbfdd83b31200ac58820a7951ddfdeed1fbee9285b0f3eae12d1357155/azure_storage_blob-12.26.0-py3-none-any.whl", hash = "sha256:8c5631b8b22b4f53ec5fff2f3bededf34cfef111e2af613ad42c9e6de00a77fe", size = 412907, upload-time = "2025-07-16T21:34:09.367Z" }, -] - -[[package]] -name = "banks" -version = "2.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "filetype" }, - { name = "griffe" }, - { name = "jinja2" }, - { name = "platformdirs" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/51/08fb68d23f4b0f6256fe85dc86e9576941550f890b079352fba719e07b39/banks-2.4.2.tar.gz", hash = "sha256:cda6013bd377ea7b701933578bfb9370fc21ad70bc13cedfc3f5cb2c034ca3dc", size = 188633, upload-time = "2026-04-27T12:15:22.021Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/b6/8dc5477681b782e2f99de703e7a99828883364b9e03a60d3e2c47053d56a/banks-2.4.2-py3-none-any.whl", hash = "sha256:5fe407cc48c101f3e13d1cf732b83b8246003337612f13c0705d2e81f6faffb7", size = 35050, upload-time = "2026-04-27T12:15:20.785Z" }, -] - -[[package]] -name = "beautifulsoup4" -version = "4.14.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/77/e9/df2358efd7659577435e2177bfa69cba6c33216681af51a707193dec162a/beautifulsoup4-4.14.2.tar.gz", hash = "sha256:2a98ab9f944a11acee9cc848508ec28d9228abfd522ef0fad6a02a72e0ded69e", size = 625822, upload-time = "2025-09-29T10:05:42.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/fe/3aed5d0be4d404d12d36ab97e2f1791424d9ca39c2f754a6285d59a3b01d/beautifulsoup4-4.14.2-py3-none-any.whl", hash = "sha256:5ef6fa3a8cbece8488d66985560f97ed091e22bbc4e9c2338508a9d5de6d4515", size = 106392, upload-time = "2025-09-29T10:05:43.771Z" }, -] - -[[package]] -name = "blinker" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, -] - -[[package]] -name = "boto3" -version = "1.34.131" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/d9/35978a20f6f9a585ff83afb384faf71526a1b25c4131755b1cdb6687b1d9/boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216", size = 108719, upload-time = "2024-06-20T19:34:56.629Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/ce/f5e3fdab6012f5fa4a8f5e97e86cc42549729382a98faffbc1785f85e89f/boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b", size = 139172, upload-time = "2024-06-20T19:34:44.219Z" }, -] - -[[package]] -name = "botocore" -version = "1.34.131" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/40/74bda5977985383b8ed403dced9d76ad5e1146db7b6c32089726b3130c8b/botocore-1.34.131.tar.gz", hash = "sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc", size = 12544482, upload-time = "2024-06-20T19:34:04.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/1a/01785fad12a9b1dbeffebd97cd226ea5923114057c64a610dd4eb8a28c7b/botocore-1.34.131-py3-none-any.whl", hash = "sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef", size = 12332729, upload-time = "2024-06-20T19:33:51.589Z" }, -] - -[[package]] -name = "cachetools" -version = "6.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/61/e4fad8155db4a04bfb4734c7c8ff0882f078f24294d42798b3568eb63bff/cachetools-6.2.0.tar.gz", hash = "sha256:38b328c0889450f05f5e120f56ab68c8abaf424e1275522b138ffc93253f7e32", size = 30988, upload-time = "2025-08-25T18:57:30.924Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/56/3124f61d37a7a4e7cc96afc5492c78ba0cb551151e530b54669ddd1436ef/cachetools-6.2.0-py3-none-any.whl", hash = "sha256:1c76a8960c0041fcc21097e357f882197c79da0dbff766e7317890a65d7d8ba6", size = 11276, upload-time = "2025-08-25T18:57:29.684Z" }, -] - -[[package]] -name = "certifi" -version = "2025.8.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, -] - -[[package]] -name = "cffi" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, -] - -[[package]] -name = "chardet" -version = "5.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618, upload-time = "2023-08-01T19:23:02.662Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385, upload-time = "2023-08-01T19:23:00.661Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, - { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, - { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, - { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, - { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, - { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, - { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, - { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, - { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, - { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, -] - -[[package]] -name = "click" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "cryptography" -version = "48.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, - { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, - { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, - { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, - { url = "https://files.pythonhosted.org/packages/89/c6/24266ac10c47f6cd2a865f4446062b466da1d1f10b27189eac00e61bf0c9/cryptography-48.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b024e784ad6c077ee0147b35ea9cbfc1e34e1fd4c1dcca214c2794d73a12df08", size = 5300085, upload-time = "2026-06-09T22:31:58.703Z" }, - { url = "https://files.pythonhosted.org/packages/d2/bb/cc4b78784f97efc8c5874c2a9743708d172be6663024b34a0467885ae0c8/cryptography-48.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6", size = 4746137, upload-time = "2026-06-09T22:31:31.01Z" }, - { url = "https://files.pythonhosted.org/packages/1f/52/0c44de3f5267f8fbe8e835138017522a333436166e406f0db9b9e6e3033f/cryptography-48.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:bd81490cd5801d755cf97bb68ac191f14b708470b1c7cf4580f669b9c9264cd8", size = 4333867, upload-time = "2026-06-09T22:32:28.096Z" }, - { url = "https://files.pythonhosted.org/packages/9a/2e/772d7adbfa931537bc401640b7cac9976bff689bda187833e5d63b428e49/cryptography-48.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429", size = 4701805, upload-time = "2026-06-09T22:31:38.284Z" }, - { url = "https://files.pythonhosted.org/packages/f8/a3/b06844f303873493c963caf581c04df31c7035e0c1b0f02c4814d319ec80/cryptography-48.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:3fd2ca57062b241c856670b073487d2e86c4637937ca5601e48f97bf8e11fc8f", size = 5258461, upload-time = "2026-06-09T22:31:04.187Z" }, - { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, - { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, - { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, - { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, - { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, - { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, - { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, - { url = "https://files.pythonhosted.org/packages/e5/0b/aa68b221dde92d09cb29a024ede17550ee21e77a404e59fc093c82bb51e1/cryptography-48.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9621de99d2da096006b629979efd8ae7eb2d8b822488d0c89ee4000c306c59b1", size = 5289970, upload-time = "2026-06-09T22:31:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/78/13/fba657f958d2af66ea959a4ba01212632089249d34af1ae48054136344d7/cryptography-48.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d", size = 4731873, upload-time = "2026-06-09T22:31:22.253Z" }, - { url = "https://files.pythonhosted.org/packages/4c/4c/9a964756d24a26b3e34dfcb16f961b89838786e6700b635b0d1e3adff4b6/cryptography-48.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:43c5835e2cb98c8733d86f57d6fc879b613f5c3478607281c3e36daffc6dd8a6", size = 4330804, upload-time = "2026-06-09T22:31:36.56Z" }, - { url = "https://files.pythonhosted.org/packages/4b/0f/a10f3a6eb12950a10e3a874070283aa2dd5875b2bfd15fad8a3e17b3f13e/cryptography-48.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46", size = 4696217, upload-time = "2026-06-09T22:31:13.351Z" }, - { url = "https://files.pythonhosted.org/packages/f3/6f/5cd12f951165ea73ef85266775d97e4c763b2474ccfd816dd69d3a18d6f8/cryptography-48.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:b7a2d1a937a738a881737cec135a38bb61470589b17515b9f73f571d0ae10401", size = 5245252, upload-time = "2026-06-09T22:32:02.193Z" }, - { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, - { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, - { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, - { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, -] - -[[package]] -name = "dataclasses-json" -version = "0.6.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "marshmallow" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, -] - -[[package]] -name = "dataproperty" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mbstrdecoder" }, - { name = "typepy", extra = ["datetime"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0b/81/8c8b64ae873cb9014815214c07b63b12e3b18835780fb342223cfe3fe7d8/dataproperty-1.1.0.tar.gz", hash = "sha256:b038437a4097d1a1c497695c3586ea34bea67fdd35372b9a50f30bf044d77d04", size = 42574, upload-time = "2024-12-31T14:37:26.033Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/21/c2/e12e95e289e6081a40454199ab213139ef16a528c7c86432de545b05a23a/DataProperty-1.1.0-py3-none-any.whl", hash = "sha256:c61fcb2e2deca35e6d1eb1f251a7f22f0dcde63e80e61f0cc18c19f42abfd25b", size = 27581, upload-time = "2024-12-31T14:37:22.657Z" }, -] - -[[package]] -name = "debugpy" -version = "1.8.17" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/ad/71e708ff4ca377c4230530d6a7aa7992592648c122a2cd2b321cf8b35a76/debugpy-1.8.17.tar.gz", hash = "sha256:fd723b47a8c08892b1a16b2c6239a8b96637c62a59b94bb5dab4bac592a58a8e", size = 1644129, upload-time = "2025-09-17T16:33:20.633Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/2b/9d8e65beb2751876c82e1aceb32f328c43ec872711fa80257c7674f45650/debugpy-1.8.17-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:f14467edef672195c6f6b8e27ce5005313cb5d03c9239059bc7182b60c176e2d", size = 2549522, upload-time = "2025-09-17T16:33:38.466Z" }, - { url = "https://files.pythonhosted.org/packages/b4/78/eb0d77f02971c05fca0eb7465b18058ba84bd957062f5eec82f941ac792a/debugpy-1.8.17-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:24693179ef9dfa20dca8605905a42b392be56d410c333af82f1c5dff807a64cc", size = 4309417, upload-time = "2025-09-17T16:33:41.299Z" }, - { url = "https://files.pythonhosted.org/packages/37/42/c40f1d8cc1fed1e75ea54298a382395b8b937d923fcf41ab0797a554f555/debugpy-1.8.17-cp312-cp312-win32.whl", hash = "sha256:6a4e9dacf2cbb60d2514ff7b04b4534b0139facbf2abdffe0639ddb6088e59cf", size = 5277130, upload-time = "2025-09-17T16:33:43.554Z" }, - { url = "https://files.pythonhosted.org/packages/72/22/84263b205baad32b81b36eac076de0cdbe09fe2d0637f5b32243dc7c925b/debugpy-1.8.17-cp312-cp312-win_amd64.whl", hash = "sha256:e8f8f61c518952fb15f74a302e068b48d9c4691768ade433e4adeea961993464", size = 5319053, upload-time = "2025-09-17T16:33:53.033Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d0/89247ec250369fc76db477720a26b2fce7ba079ff1380e4ab4529d2fe233/debugpy-1.8.17-py2.py3-none-any.whl", hash = "sha256:60c7dca6571efe660ccb7a9508d73ca14b8796c4ed484c2002abba714226cfef", size = 5283210, upload-time = "2025-09-17T16:34:25.835Z" }, -] - -[[package]] -name = "decorator" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, -] - -[[package]] -name = "defusedxml" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, -] - -[[package]] -name = "deprecated" -version = "1.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, -] - -[[package]] -name = "deprecation" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, -] - -[[package]] -name = "dirtyjson" -version = "1.0.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, -] - -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - -[[package]] -name = "fastuuid" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/7d/d9daedf0f2ebcacd20d599928f8913e9d2aea1d56d2d355a93bfa2b611d7/fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26", size = 18232, upload-time = "2025-10-19T22:19:22.402Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/a2/e78fcc5df65467f0d207661b7ef86c5b7ac62eea337c0c0fcedbeee6fb13/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a", size = 510164, upload-time = "2025-10-19T22:31:45.635Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b3/c846f933f22f581f558ee63f81f29fa924acd971ce903dab1a9b6701816e/fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d", size = 261837, upload-time = "2025-10-19T22:38:38.53Z" }, - { url = "https://files.pythonhosted.org/packages/54/ea/682551030f8c4fa9a769d9825570ad28c0c71e30cf34020b85c1f7ee7382/fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070", size = 251370, upload-time = "2025-10-19T22:40:26.07Z" }, - { url = "https://files.pythonhosted.org/packages/14/dd/5927f0a523d8e6a76b70968e6004966ee7df30322f5fc9b6cdfb0276646a/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796", size = 277766, upload-time = "2025-10-19T22:37:23.779Z" }, - { url = "https://files.pythonhosted.org/packages/16/6e/c0fb547eef61293153348f12e0f75a06abb322664b34a1573a7760501336/fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09", size = 278105, upload-time = "2025-10-19T22:26:56.821Z" }, - { url = "https://files.pythonhosted.org/packages/2d/b1/b9c75e03b768f61cf2e84ee193dc18601aeaf89a4684b20f2f0e9f52b62c/fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8", size = 301564, upload-time = "2025-10-19T22:30:31.604Z" }, - { url = "https://files.pythonhosted.org/packages/fc/fa/f7395fdac07c7a54f18f801744573707321ca0cee082e638e36452355a9d/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741", size = 459659, upload-time = "2025-10-19T22:31:32.341Z" }, - { url = "https://files.pythonhosted.org/packages/66/49/c9fd06a4a0b1f0f048aacb6599e7d96e5d6bc6fa680ed0d46bf111929d1b/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057", size = 478430, upload-time = "2025-10-19T22:26:22.962Z" }, - { url = "https://files.pythonhosted.org/packages/be/9c/909e8c95b494e8e140e8be6165d5fc3f61fdc46198c1554df7b3e1764471/fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8", size = 450894, upload-time = "2025-10-19T22:27:01.647Z" }, - { url = "https://files.pythonhosted.org/packages/90/eb/d29d17521976e673c55ef7f210d4cdd72091a9ec6755d0fd4710d9b3c871/fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176", size = 154374, upload-time = "2025-10-19T22:29:19.879Z" }, - { url = "https://files.pythonhosted.org/packages/cc/fc/f5c799a6ea6d877faec0472d0b27c079b47c86b1cdc577720a5386483b36/fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397", size = 156550, upload-time = "2025-10-19T22:27:49.658Z" }, -] - -[[package]] -name = "filelock" -version = "3.20.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, -] - -[[package]] -name = "filetype" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/29/745f7d30d47fe0f251d3ad3dc2978a23141917661998763bebb6da007eb1/filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb", size = 998020, upload-time = "2022-11-02T17:34:04.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, -] - -[[package]] -name = "flask" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blinker" }, - { name = "click" }, - { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "markupsafe" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, -] - -[[package]] -name = "flask-wtf" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flask" }, - { name = "itsdangerous" }, - { name = "wtforms" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/9b/f1cd6e41bbf874f3436368f2c7ee3216c1e82d666ff90d1d800e20eb1317/flask_wtf-1.2.2.tar.gz", hash = "sha256:79d2ee1e436cf570bccb7d916533fa18757a2f18c290accffab1b9a0b684666b", size = 42641, upload-time = "2024-10-24T07:18:58.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/19/354449145fbebb65e7c621235b6ad69bebcfaec2142481f044d0ddc5b5c5/flask_wtf-1.2.2-py3-none-any.whl", hash = "sha256:e93160c5c5b6b571cf99300b6e01b72f9a101027cab1579901f8b10c5daf0b70", size = 12779, upload-time = "2024-10-24T07:18:56.976Z" }, -] - -[[package]] -name = "frozenlist" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" }, - { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" }, - { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" }, - { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" }, - { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" }, - { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" }, - { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" }, - { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" }, - { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" }, - { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" }, - { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" }, - { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" }, - { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" }, - { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" }, - { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" }, - { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, -] - -[[package]] -name = "fsspec" -version = "2024.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a0/52/f16a068ebadae42526484c31f4398e62962504e5724a8ba5dc3409483df2/fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493", size = 286853, upload-time = "2024-10-21T01:21:16.969Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/b2/454d6e7f0158951d8a78c2e1eb4f69ae81beb8dca5fee9809c6c99e9d0d0/fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871", size = 179641, upload-time = "2024-10-21T01:21:14.793Z" }, -] - -[[package]] -name = "gcsfs" -version = "2024.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "decorator" }, - { name = "fsspec" }, - { name = "google-auth" }, - { name = "google-auth-oauthlib" }, - { name = "google-cloud-storage" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e5/1e/1d8c4593d9e2eb04918fec43253ab152823d67ad51ad9e3ab6b3a78c431a/gcsfs-2024.10.0.tar.gz", hash = "sha256:5df54cfe568e8fdeea5aafa7fed695cdc69a9a674e991ca8c1ce634f5df1d314", size = 79588, upload-time = "2024-10-21T13:43:26.163Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/96/d60e835fb7d10166c77aef0c1fa30e634153c03a0f486786977b95f88fde/gcsfs-2024.10.0-py2.py3-none-any.whl", hash = "sha256:bb2d23547e61203ea2dda5fa6c4b91a0c34b74ebe8bb6ab1926f6c33381bceb2", size = 34953, upload-time = "2024-10-21T13:43:24.951Z" }, -] - -[[package]] -name = "google-api-core" -version = "2.25.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/21/e9d043e88222317afdbdb567165fdbc3b0aad90064c7e0c9eb0ad9955ad8/google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8", size = 165443, upload-time = "2025-06-12T20:52:20.439Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/4b/ead00905132820b623732b175d66354e9d3e69fcf2a5dcdab780664e7896/google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7", size = 160807, upload-time = "2025-06-12T20:52:19.334Z" }, -] - -[[package]] -name = "google-auth" -version = "2.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "pyasn1-modules" }, - { name = "rsa" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/c5/87742f5b5f055514c67f970f7174a876fccff2289a69d460b0614cc7ccfb/google_auth-2.41.0.tar.gz", hash = "sha256:c9d7b534ea4a5d9813c552846797fafb080312263cd4994d6622dd50992ae101", size = 292282, upload-time = "2025-09-29T21:36:35.791Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/ff/a1c426fc9bea7268230bf92340da7d112fae18cf946cafe13ab17d14e6ee/google_auth-2.41.0-py2.py3-none-any.whl", hash = "sha256:d8bed9b53ab63b7b0374656b8e1bef051f95bb14ecc0cf21ba49de7911d62e09", size = 221168, upload-time = "2025-09-29T21:36:33.925Z" }, -] - -[[package]] -name = "google-auth-oauthlib" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "requests-oauthlib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/87/e10bf24f7bcffc1421b84d6f9c3377c30ec305d082cd737ddaa6d8f77f7c/google_auth_oauthlib-1.2.2.tar.gz", hash = "sha256:11046fb8d3348b296302dd939ace8af0a724042e8029c1b872d87fabc9f41684", size = 20955, upload-time = "2025-04-22T16:40:29.172Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/84/40ee070be95771acd2f4418981edb834979424565c3eec3cd88b6aa09d24/google_auth_oauthlib-1.2.2-py3-none-any.whl", hash = "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2", size = 19072, upload-time = "2025-04-22T16:40:28.174Z" }, -] - -[[package]] -name = "google-cloud-core" -version = "2.4.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d6/b8/2b53838d2acd6ec6168fd284a990c76695e84c65deee79c9f3a4276f6b4f/google_cloud_core-2.4.3.tar.gz", hash = "sha256:1fab62d7102844b278fe6dead3af32408b1df3eb06f5c7e8634cbd40edc4da53", size = 35861, upload-time = "2025-03-10T21:05:38.948Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/86/bda7241a8da2d28a754aad2ba0f6776e35b67e37c36ae0c45d49370f1014/google_cloud_core-2.4.3-py2.py3-none-any.whl", hash = "sha256:5130f9f4c14b4fafdff75c79448f9495cfade0d8775facf1b09c3bf67e027f6e", size = 29348, upload-time = "2025-03-10T21:05:37.785Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "2.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/76/4d965702e96bb67976e755bed9828fa50306dca003dbee08b67f41dd265e/google_cloud_storage-2.19.0.tar.gz", hash = "sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2", size = 5535488, upload-time = "2024-12-05T01:35:06.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/94/6db383d8ee1adf45dc6c73477152b82731fa4c4a46d9c1932cc8757e0fd4/google_cloud_storage-2.19.0-py2.py3-none-any.whl", hash = "sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba", size = 131787, upload-time = "2024-12-05T01:35:04.736Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/ae/87802e6d9f9d69adfaedfcfd599266bf386a54d0be058b532d04c794f76d/google_crc32c-1.7.1.tar.gz", hash = "sha256:2bff2305f98846f3e825dbeec9ee406f89da7962accdb29356e4eadc251bd472", size = 14495, upload-time = "2025-03-26T14:29:13.32Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b7/787e2453cf8639c94b3d06c9d61f512234a82e1d12d13d18584bd3049904/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:2d73a68a653c57281401871dd4aeebbb6af3191dcac751a76ce430df4d403194", size = 30470, upload-time = "2025-03-26T14:34:31.655Z" }, - { url = "https://files.pythonhosted.org/packages/ed/b4/6042c2b0cbac3ec3a69bb4c49b28d2f517b7a0f4a0232603c42c58e22b44/google_crc32c-1.7.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:22beacf83baaf59f9d3ab2bbb4db0fb018da8e5aebdce07ef9f09fce8220285e", size = 30315, upload-time = "2025-03-26T15:01:54.634Z" }, - { url = "https://files.pythonhosted.org/packages/29/ad/01e7a61a5d059bc57b702d9ff6a18b2585ad97f720bd0a0dbe215df1ab0e/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19eafa0e4af11b0a4eb3974483d55d2d77ad1911e6cf6f832e1574f6781fd337", size = 33180, upload-time = "2025-03-26T14:41:32.168Z" }, - { url = "https://files.pythonhosted.org/packages/3b/a5/7279055cf004561894ed3a7bfdf5bf90a53f28fadd01af7cd166e88ddf16/google_crc32c-1.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d86616faaea68101195c6bdc40c494e4d76f41e07a37ffdef270879c15fb65", size = 32794, upload-time = "2025-03-26T14:41:33.264Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d6/77060dbd140c624e42ae3ece3df53b9d811000729a5c821b9fd671ceaac6/google_crc32c-1.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:b7491bdc0c7564fcf48c0179d2048ab2f7c7ba36b84ccd3a3e1c3f7a72d3bba6", size = 33477, upload-time = "2025-03-26T14:29:10.94Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/58/5a/0efdc02665dca14e0837b62c8a1a93132c264bd02054a15abb2218afe0ae/google_resumable_media-2.7.2.tar.gz", hash = "sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0", size = 2163099, upload-time = "2024-08-07T22:20:38.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/35/b8d3baf8c46695858cb9d8835a53baa1eeb9906ddaf2f728a5f5b640fd1e/google_resumable_media-2.7.2-py2.py3-none-any.whl", hash = "sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa", size = 81251, upload-time = "2024-08-07T22:20:36.409Z" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.70.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, -] - -[[package]] -name = "greenlet" -version = "3.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, - { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, - { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, - { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, - { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, - { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, - { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, - { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, - { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, -] - -[[package]] -name = "griffe" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/d7/6c09dd7ce4c7837e4cdb11dce980cb45ae3cd87677298dc3b781b6bce7d3/griffe-1.14.0.tar.gz", hash = "sha256:9d2a15c1eca966d68e00517de5d69dd1bc5c9f2335ef6c1775362ba5b8651a13", size = 424684, upload-time = "2025-09-05T15:02:29.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/b1/9ff6578d789a89812ff21e4e0f80ffae20a65d5dd84e7a17873fe3b365be/griffe-1.14.0-py3-none-any.whl", hash = "sha256:0e9d52832cccf0f7188cfe585ba962d2674b241c01916d780925df34873bceb0", size = 144439, upload-time = "2025-09-05T15:02:27.511Z" }, -] - -[[package]] -name = "grpcio" -version = "1.76.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/e0/318c1ce3ae5a17894d5791e87aea147587c9e702f24122cc7a5c8bbaeeb1/grpcio-1.76.0.tar.gz", hash = "sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73", size = 12785182, upload-time = "2025-10-21T16:23:12.106Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/05/8e29121994b8d959ffa0afd28996d452f291b48cfc0875619de0bde2c50c/grpcio-1.76.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:81fd9652b37b36f16138611c7e884eb82e0cec137c40d3ef7c3f9b3ed00f6ed8", size = 5799718, upload-time = "2025-10-21T16:21:17.939Z" }, - { url = "https://files.pythonhosted.org/packages/d9/75/11d0e66b3cdf998c996489581bdad8900db79ebd83513e45c19548f1cba4/grpcio-1.76.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:04bbe1bfe3a68bbfd4e52402ab7d4eb59d72d02647ae2042204326cf4bbad280", size = 11825627, upload-time = "2025-10-21T16:21:20.466Z" }, - { url = "https://files.pythonhosted.org/packages/28/50/2f0aa0498bc188048f5d9504dcc5c2c24f2eb1a9337cd0fa09a61a2e75f0/grpcio-1.76.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d388087771c837cdb6515539f43b9d4bf0b0f23593a24054ac16f7a960be16f4", size = 6359167, upload-time = "2025-10-21T16:21:23.122Z" }, - { url = "https://files.pythonhosted.org/packages/66/e5/bbf0bb97d29ede1d59d6588af40018cfc345b17ce979b7b45424628dc8bb/grpcio-1.76.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f8f757bebaaea112c00dba718fc0d3260052ce714e25804a03f93f5d1c6cc11", size = 7044267, upload-time = "2025-10-21T16:21:25.995Z" }, - { url = "https://files.pythonhosted.org/packages/f5/86/f6ec2164f743d9609691115ae8ece098c76b894ebe4f7c94a655c6b03e98/grpcio-1.76.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:980a846182ce88c4f2f7e2c22c56aefd515daeb36149d1c897f83cf57999e0b6", size = 6573963, upload-time = "2025-10-21T16:21:28.631Z" }, - { url = "https://files.pythonhosted.org/packages/60/bc/8d9d0d8505feccfdf38a766d262c71e73639c165b311c9457208b56d92ae/grpcio-1.76.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f92f88e6c033db65a5ae3d97905c8fea9c725b63e28d5a75cb73b49bda5024d8", size = 7164484, upload-time = "2025-10-21T16:21:30.837Z" }, - { url = "https://files.pythonhosted.org/packages/67/e6/5d6c2fc10b95edf6df9b8f19cf10a34263b7fd48493936fffd5085521292/grpcio-1.76.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4baf3cbe2f0be3289eb68ac8ae771156971848bb8aaff60bad42005539431980", size = 8127777, upload-time = "2025-10-21T16:21:33.577Z" }, - { url = "https://files.pythonhosted.org/packages/3f/c8/dce8ff21c86abe025efe304d9e31fdb0deaaa3b502b6a78141080f206da0/grpcio-1.76.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:615ba64c208aaceb5ec83bfdce7728b80bfeb8be97562944836a7a0a9647d882", size = 7594014, upload-time = "2025-10-21T16:21:41.882Z" }, - { url = "https://files.pythonhosted.org/packages/e0/42/ad28191ebf983a5d0ecef90bab66baa5a6b18f2bfdef9d0a63b1973d9f75/grpcio-1.76.0-cp312-cp312-win32.whl", hash = "sha256:45d59a649a82df5718fd9527ce775fd66d1af35e6d31abdcdc906a49c6822958", size = 3984750, upload-time = "2025-10-21T16:21:44.006Z" }, - { url = "https://files.pythonhosted.org/packages/9e/00/7bd478cbb851c04a48baccaa49b75abaa8e4122f7d86da797500cccdd771/grpcio-1.76.0-cp312-cp312-win_amd64.whl", hash = "sha256:c088e7a90b6017307f423efbb9d1ba97a22aa2170876223f9709e9d1de0b5347", size = 4704003, upload-time = "2025-10-21T16:21:46.244Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.71.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ad/9a/edfefb47f11ef6b0f39eea4d8f022c5bb05ac1d14fcc7058e84a51305b73/grpcio_tools-1.71.2.tar.gz", hash = "sha256:b5304d65c7569b21270b568e404a5a843cf027c66552a6a0978b23f137679c09", size = 5330655, upload-time = "2025-06-28T04:22:00.308Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/d3/3ed30a9c5b2424627b4b8411e2cd6a1a3f997d3812dbc6a8630a78bcfe26/grpcio_tools-1.71.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:bfc0b5d289e383bc7d317f0e64c9dfb59dc4bef078ecd23afa1a816358fb1473", size = 2385479, upload-time = "2025-06-28T04:21:10.413Z" }, - { url = "https://files.pythonhosted.org/packages/54/61/e0b7295456c7e21ef777eae60403c06835160c8d0e1e58ebfc7d024c51d3/grpcio_tools-1.71.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b4669827716355fa913b1376b1b985855d5cfdb63443f8d18faf210180199006", size = 5431521, upload-time = "2025-06-28T04:21:12.261Z" }, - { url = "https://files.pythonhosted.org/packages/75/d7/7bcad6bcc5f5b7fab53e6bce5db87041f38ef3e740b1ec2d8c49534fa286/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:d4071f9b44564e3f75cdf0f05b10b3e8c7ea0ca5220acbf4dc50b148552eef2f", size = 2350289, upload-time = "2025-06-28T04:21:13.625Z" }, - { url = "https://files.pythonhosted.org/packages/b2/8a/e4c1c4cb8c9ff7f50b7b2bba94abe8d1e98ea05f52a5db476e7f1c1a3c70/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a28eda8137d587eb30081384c256f5e5de7feda34776f89848b846da64e4be35", size = 2743321, upload-time = "2025-06-28T04:21:15.007Z" }, - { url = "https://files.pythonhosted.org/packages/fd/aa/95bc77fda5c2d56fb4a318c1b22bdba8914d5d84602525c99047114de531/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b19c083198f5eb15cc69c0a2f2c415540cbc636bfe76cea268e5894f34023b40", size = 2474005, upload-time = "2025-06-28T04:21:16.443Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ff/ca11f930fe1daa799ee0ce1ac9630d58a3a3deed3dd2f465edb9a32f299d/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:784c284acda0d925052be19053d35afbf78300f4d025836d424cf632404f676a", size = 2851559, upload-time = "2025-06-28T04:21:18.139Z" }, - { url = "https://files.pythonhosted.org/packages/64/10/c6fc97914c7e19c9bb061722e55052fa3f575165da9f6510e2038d6e8643/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:381e684d29a5d052194e095546eef067201f5af30fd99b07b5d94766f44bf1ae", size = 3300622, upload-time = "2025-06-28T04:21:20.291Z" }, - { url = "https://files.pythonhosted.org/packages/e5/d6/965f36cfc367c276799b730d5dd1311b90a54a33726e561393b808339b04/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3e4b4801fabd0427fc61d50d09588a01b1cfab0ec5e8a5f5d515fbdd0891fd11", size = 2913863, upload-time = "2025-06-28T04:21:22.196Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f0/c05d5c3d0c1d79ac87df964e9d36f1e3a77b60d948af65bec35d3e5c75a3/grpcio_tools-1.71.2-cp312-cp312-win32.whl", hash = "sha256:84ad86332c44572305138eafa4cc30040c9a5e81826993eae8227863b700b490", size = 945744, upload-time = "2025-06-28T04:21:23.463Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e9/c84c1078f0b7af7d8a40f5214a9bdd8d2a567ad6c09975e6e2613a08d29d/grpcio_tools-1.71.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e1108d37eecc73b1c4a27350a6ed921b5dda25091700c1da17cfe30761cd462", size = 1117695, upload-time = "2025-06-28T04:21:25.22Z" }, -] - -[[package]] -name = "gunicorn" -version = "23.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "h2" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpack" }, - { name = "hyperframe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, -] - -[[package]] -name = "hf-xet" -version = "1.1.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" }, - { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" }, - { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" }, - { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" }, - { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" }, -] - -[[package]] -name = "hpack" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[package.optional-dependencies] -http2 = [ - { name = "h2" }, -] - -[[package]] -name = "huggingface-hub" -version = "0.35.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" }, -] - -[[package]] -name = "hyperframe" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, -] - -[[package]] -name = "idna" -version = "3.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, -] - -[[package]] -name = "importlib-metadata" -version = "8.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - -[[package]] -name = "itsdangerous" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "jiter" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/c0/a3bb4cc13aced219dd18191ea66e874266bd8aa7b96744e495e1c733aa2d/jiter-0.11.0.tar.gz", hash = "sha256:1d9637eaf8c1d6a63d6562f2a6e5ab3af946c66037eb1b894e8fad75422266e4", size = 167094, upload-time = "2025-09-15T09:20:38.212Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/b5/3009b112b8f673e568ef79af9863d8309a15f0a8cdcc06ed6092051f377e/jiter-0.11.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:2fb7b377688cc3850bbe5c192a6bd493562a0bc50cbc8b047316428fbae00ada", size = 305510, upload-time = "2025-09-15T09:19:25.893Z" }, - { url = "https://files.pythonhosted.org/packages/fe/82/15514244e03b9e71e086bbe2a6de3e4616b48f07d5f834200c873956fb8c/jiter-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a1b7cbe3f25bd0d8abb468ba4302a5d45617ee61b2a7a638f63fee1dc086be99", size = 316521, upload-time = "2025-09-15T09:19:27.525Z" }, - { url = "https://files.pythonhosted.org/packages/92/94/7a2e905f40ad2d6d660e00b68d818f9e29fb87ffe82774f06191e93cbe4a/jiter-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0a7f0ec81d5b7588c5cade1eb1925b91436ae6726dc2df2348524aeabad5de6", size = 338214, upload-time = "2025-09-15T09:19:28.727Z" }, - { url = "https://files.pythonhosted.org/packages/a8/9c/5791ed5bdc76f12110158d3316a7a3ec0b1413d018b41c5ed399549d3ad5/jiter-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07630bb46ea2a6b9c6ed986c6e17e35b26148cce2c535454b26ee3f0e8dcaba1", size = 361280, upload-time = "2025-09-15T09:19:30.013Z" }, - { url = "https://files.pythonhosted.org/packages/d4/7f/b7d82d77ff0d2cb06424141000176b53a9e6b16a1125525bb51ea4990c2e/jiter-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7764f27d28cd4a9cbc61704dfcd80c903ce3aad106a37902d3270cd6673d17f4", size = 487895, upload-time = "2025-09-15T09:19:31.424Z" }, - { url = "https://files.pythonhosted.org/packages/42/44/10a1475d46f1fc1fd5cc2e82c58e7bca0ce5852208e0fa5df2f949353321/jiter-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d4a6c4a737d486f77f842aeb22807edecb4a9417e6700c7b981e16d34ba7c72", size = 378421, upload-time = "2025-09-15T09:19:32.746Z" }, - { url = "https://files.pythonhosted.org/packages/9a/5f/0dc34563d8164d31d07bc09d141d3da08157a68dcd1f9b886fa4e917805b/jiter-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf408d2a0abd919b60de8c2e7bc5eeab72d4dafd18784152acc7c9adc3291591", size = 347932, upload-time = "2025-09-15T09:19:34.612Z" }, - { url = "https://files.pythonhosted.org/packages/f7/de/b68f32a4fcb7b4a682b37c73a0e5dae32180140cd1caf11aef6ad40ddbf2/jiter-0.11.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cdef53eda7d18e799625023e1e250dbc18fbc275153039b873ec74d7e8883e09", size = 386959, upload-time = "2025-09-15T09:19:35.994Z" }, - { url = "https://files.pythonhosted.org/packages/76/0a/c08c92e713b6e28972a846a81ce374883dac2f78ec6f39a0dad9f2339c3a/jiter-0.11.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:53933a38ef7b551dd9c7f1064f9d7bb235bb3168d0fa5f14f0798d1b7ea0d9c5", size = 517187, upload-time = "2025-09-15T09:19:37.426Z" }, - { url = "https://files.pythonhosted.org/packages/89/b5/4a283bec43b15aad54fcae18d951f06a2ec3f78db5708d3b59a48e9c3fbd/jiter-0.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11840d2324c9ab5162fc1abba23bc922124fedcff0d7b7f85fffa291e2f69206", size = 509461, upload-time = "2025-09-15T09:19:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/34/a5/f8bad793010534ea73c985caaeef8cc22dfb1fedb15220ecdf15c623c07a/jiter-0.11.0-cp312-cp312-win32.whl", hash = "sha256:4f01a744d24a5f2bb4a11657a1b27b61dc038ae2e674621a74020406e08f749b", size = 206664, upload-time = "2025-09-15T09:19:40.096Z" }, - { url = "https://files.pythonhosted.org/packages/ed/42/5823ec2b1469395a160b4bf5f14326b4a098f3b6898fbd327366789fa5d3/jiter-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:29fff31190ab3a26de026da2f187814f4b9c6695361e20a9ac2123e4d4378a4c", size = 203520, upload-time = "2025-09-15T09:19:41.798Z" }, -] - -[[package]] -name = "jmespath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, -] - -[[package]] -name = "joblib" -version = "1.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/5d/447af5ea094b9e4c4054f82e223ada074c552335b9b4b2d14bd9b35a67c4/joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55", size = 331077, upload-time = "2025-08-27T12:15:46.575Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/e8/685f47e0d754320684db4425a0967f7d3fa70126bffd76110b7009a0090f/joblib-1.5.2-py3-none-any.whl", hash = "sha256:4e1f0bdbb987e6d843c70cf43714cb276623def372df3c22fe5266b2670bc241", size = 308396, upload-time = "2025-08-27T12:15:45.188Z" }, -] - -[[package]] -name = "json-repair" -version = "0.42.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f4/1e/aea70e484e271e3d841773d5d185bedf488cf8f881088d47faae31f19116/json_repair-0.42.0.tar.gz", hash = "sha256:1a901f706c5b6b4325f0f79b53b0d998c5b327070e98b530da71cc5a3eda8616", size = 31349, upload-time = "2025-04-22T11:12:17.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/c7/dd23f764de95771300a8a7ae17293c47ba0e48f826154229d18ecfe147cd/json_repair-0.42.0-py3-none-any.whl", hash = "sha256:7b6805162053dfe65722e961bc51b5eecec0582ec8a8e0fd218d33e8de757daf", size = 21612, upload-time = "2025-04-22T11:12:16.302Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778, upload-time = "2024-07-08T18:40:05.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/4a/4f9dbeb84e8850557c02365a0eee0649abe5eb1d84af92a25731c6c0f922/jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566", size = 88462, upload-time = "2024-07-08T18:40:00.165Z" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, -] - -[[package]] -name = "kombu" -version = "5.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "packaging" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992, upload-time = "2025-06-01T10:19:22.281Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034, upload-time = "2025-06-01T10:19:20.436Z" }, -] - -[[package]] -name = "litellm" -version = "1.85.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "click" }, - { name = "fastuuid" }, - { name = "httpx" }, - { name = "importlib-metadata" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "openai" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "tiktoken" }, - { name = "tokenizers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, -] - -[[package]] -name = "llama-cloud" -version = "0.1.35" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9b/72/816e6e900448e1b4a8137d90e65876b296c5264a23db6ae888bd3e6660ba/llama_cloud-0.1.35.tar.gz", hash = "sha256:200349d5d57424d7461f304cdb1355a58eea3e6ca1e6b0d75c66b2e937216983", size = 106403, upload-time = "2025-07-28T17:22:06.41Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/d2/8d18a021ab757cea231428404f21fe3186bf1ebaac3f57a73c379483fd3f/llama_cloud-0.1.35-py3-none-any.whl", hash = "sha256:b7abab4423118e6f638d2f326749e7a07c6426543bea6da99b623c715b22af71", size = 303280, upload-time = "2025-07-28T17:22:04.946Z" }, -] - -[[package]] -name = "llama-cloud-services" -version = "0.6.54" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "llama-cloud" }, - { name = "llama-index-core" }, - { name = "platformdirs" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "tenacity" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8a/0c/8ca87d33bea0340a8ed791f36390112aeb29fd3eebfd64b6aef6204a03f0/llama_cloud_services-0.6.54.tar.gz", hash = "sha256:baf65d9bffb68f9dca98ac6e22908b6675b2038b021e657ead1ffc0e43cbd45d", size = 53468, upload-time = "2025-08-01T20:09:20.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/48/4e295e3f791b279885a2e584f71e75cbe4ac84e93bba3c36e2668f60a8ac/llama_cloud_services-0.6.54-py3-none-any.whl", hash = "sha256:07f595f7a0ba40c6a1a20543d63024ca7600fe65c4811d1951039977908997be", size = 63874, upload-time = "2025-08-01T20:09:20.076Z" }, -] - -[[package]] -name = "llama-index" -version = "0.14.13" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-cli" }, - { name = "llama-index-core" }, - { name = "llama-index-embeddings-openai" }, - { name = "llama-index-indices-managed-llama-cloud" }, - { name = "llama-index-llms-openai" }, - { name = "llama-index-readers-file" }, - { name = "llama-index-readers-llama-parse" }, - { name = "nltk" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/26/e46c43ca97f849169d54b1c11e599ea51e650f4c27355c28e384cb0c7e94/llama_index-0.14.13.tar.gz", hash = "sha256:6392822f8ad0e747da2f0adbfcd170bc91fafdff4341cd50da809feae5ca828a", size = 8461, upload-time = "2026-01-21T20:44:49.651Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/32/5c13c0b0e7fdedaf4f98a137a64558ee87bbdbc14e16b394c103f6a5418e/llama_index-0.14.13-py3-none-any.whl", hash = "sha256:54b3c15d9327a05c981f332643e15a7adc52168d173302987aeb0f9dc7f0267c", size = 7459, upload-time = "2026-01-21T20:44:47.96Z" }, -] - -[[package]] -name = "llama-index-cli" -version = "0.5.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "llama-index-embeddings-openai" }, - { name = "llama-index-llms-openai" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/67/84/41e820efffbe327c38228d3b37fe42512a37e0c3ee4ff6bf97a394e9577a/llama_index_cli-0.5.3.tar.gz", hash = "sha256:ebaf39e785efbfa8d50d837f60cb0f95125c04bf73ed1f92092a2a5f506172f8", size = 24821, upload-time = "2025-09-29T18:03:10.798Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/81/b7b3778aa8662913760fbbee77578daf4407aeaa677ccbf0125c4cfa2e67/llama_index_cli-0.5.3-py3-none-any.whl", hash = "sha256:7deb1e953e582bd885443881ce8bd6ab2817b594fef00079dce9993c47d990f7", size = 28173, upload-time = "2025-09-29T18:03:10.024Z" }, -] - -[[package]] -name = "llama-index-core" -version = "0.14.13" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "aiosqlite" }, - { name = "banks" }, - { name = "dataclasses-json" }, - { name = "deprecated" }, - { name = "dirtyjson" }, - { name = "filetype" }, - { name = "fsspec" }, - { name = "httpx" }, - { name = "llama-index-workflows" }, - { name = "nest-asyncio" }, - { name = "networkx" }, - { name = "nltk" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "platformdirs" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "setuptools" }, - { name = "sqlalchemy", extra = ["asyncio"] }, - { name = "tenacity" }, - { name = "tiktoken" }, - { name = "tqdm" }, - { name = "typing-extensions" }, - { name = "typing-inspect" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/54/d6043a088e5e9c1d62300db7ad0ef417c6b9a92f7b4a5cade066aeafdaca/llama_index_core-0.14.13.tar.gz", hash = "sha256:c3b30d20ae0407e5d0a1d35bb3376a98e242661ebfc22da754b5a3da1f8108c0", size = 11589074, upload-time = "2026-01-21T20:44:16.287Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/59/9769f03f1cccadcc014b3b65c166de18999b51459a0f0a579d80f6c91d80/llama_index_core-0.14.13-py3-none-any.whl", hash = "sha256:392f0a5a09433e9dea786964ef5fe5ca2a2b10aee9f979a9507c19a14da2a20a", size = 11934761, upload-time = "2026-01-21T20:44:18.892Z" }, -] - -[[package]] -name = "llama-index-embeddings-openai" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "openai" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/36/90336d054a5061a3f5bc17ac2c18ef63d9d84c55c14d557de484e811ea4d/llama_index_embeddings_openai-0.5.1.tar.gz", hash = "sha256:1c89867a48b0d0daa3d2d44f5e76b394b2b2ef9935932daf921b9e77939ccda8", size = 7020, upload-time = "2025-09-08T20:17:44.681Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/23/4a/8ab11026cf8deff8f555aa73919be0bac48332683111e5fc4290f352dc50/llama_index_embeddings_openai-0.5.1-py3-none-any.whl", hash = "sha256:a2fcda3398bbd987b5ce3f02367caee8e84a56b930fdf43cc1d059aa9fd20ca5", size = 7011, upload-time = "2025-09-08T20:17:44.015Z" }, -] - -[[package]] -name = "llama-index-indices-managed-llama-cloud" -version = "0.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "llama-cloud" }, - { name = "llama-index-core" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/4a/79044fcb3209583d1ffe0c2a7c19dddfb657a03faeb9fe0cf5a74027e646/llama_index_indices_managed_llama_cloud-0.9.4.tar.gz", hash = "sha256:b5e00752ab30564abf19c57595a2107f5697c3b03b085817b4fca84a38ebbd59", size = 15146, upload-time = "2025-09-08T20:29:58.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/6a/0e33245df06afc9766c46a1fe92687be8a09da5d0d0128bc08d84a9f5efa/llama_index_indices_managed_llama_cloud-0.9.4-py3-none-any.whl", hash = "sha256:535a08811046803ca6ab7f8e9d510e926aa5306608b02201ad3d9d21701383bc", size = 17005, upload-time = "2025-09-08T20:29:57.876Z" }, -] - -[[package]] -name = "llama-index-instrumentation" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/e5/a3628da5d716d6bbc2c0a8d39b629dff81b33d5625c5b934e1456370064f/llama_index_instrumentation-0.4.1.tar.gz", hash = "sha256:a79d0dd2baba34f05ff4354d63a99b212322635b8afa6cc96ed00a7e11ebfdc3", size = 45788, upload-time = "2025-09-15T03:53:00.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/7a/c414f4dc9a7dd90d050c387489436bab2d678a566b704ede2f5b62f82ad7/llama_index_instrumentation-0.4.1-py3-none-any.whl", hash = "sha256:0d3ac926d0db3d39c0ec34ee72da5322d61e06b87fe956407e4a1e7a2708b936", size = 15063, upload-time = "2025-09-15T03:52:59.098Z" }, -] - -[[package]] -name = "llama-index-llms-openai" -version = "0.6.17" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "openai" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c8/94/55e9855f565b30604fe9e154a7c3df1f4437ce62ed8926a8d6b5112cf73b/llama_index_llms_openai-0.6.17.tar.gz", hash = "sha256:63294c1d8d221d2009023b2c294c919f6141f4cdce9cd48bffe700d8be2a37e0", size = 25941, upload-time = "2026-02-03T11:03:26.355Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/85/1dee39a45b31528611cc6139777fc5631b9946fe085b37cd981062aa6d50/llama_index_llms_openai-0.6.17-py3-none-any.whl", hash = "sha256:d275f7e218611afc6bb157fe722f6e8590e8e20ac3ea1392cf560ac674c175af", size = 26940, upload-time = "2026-02-03T11:03:25.059Z" }, -] - -[[package]] -name = "llama-index-readers-file" -version = "0.5.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "defusedxml" }, - { name = "llama-index-core" }, - { name = "pandas" }, - { name = "pypdf" }, - { name = "striprtf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/d9/c67ad2b9cba8dacf1d4a55fe5432357b6eceaecfb096a0de5c1cbd959b98/llama_index_readers_file-0.5.4.tar.gz", hash = "sha256:5e766f32597622e66529464101914548ad683770a0a5d2bdc9ee84eb3a110332", size = 32565, upload-time = "2025-09-08T20:39:40.287Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/e3/76d72a7281b9c88d488908731c9034e1ee1a2cad5aa1dead76b051eca989/llama_index_readers_file-0.5.4-py3-none-any.whl", hash = "sha256:135be5ddda66c5b35883911918b2d99f67a2ab010d180af5630c872ea9509d45", size = 51827, upload-time = "2025-09-08T20:39:39.408Z" }, -] - -[[package]] -name = "llama-index-readers-llama-parse" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "llama-parse" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/77/5bfaab20e6ec8428dbf2352e18be550c957602723d69383908176b5686cd/llama_index_readers_llama_parse-0.5.1.tar.gz", hash = "sha256:2b78b73faa933e30e6c69df351e4e9f36dfe2ae142e2ab3969ddd2ac48930e37", size = 3858, upload-time = "2025-09-08T20:41:29.201Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/81/52410c7245dcbf1a54756a9ce3892cdd167ec0b884d696de1304ca3f452e/llama_index_readers_llama_parse-0.5.1-py3-none-any.whl", hash = "sha256:0d41450ed29b0c49c024e206ef6c8e662b1854e77a1c5faefed3b958be54f880", size = 3203, upload-time = "2025-09-08T20:41:28.438Z" }, -] - -[[package]] -name = "llama-index-vector-stores-milvus" -version = "0.9.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "pymilvus" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/79/72bf70fac3b77770a5c2b1b7d441aa1998bb522d50fe88b0f9c4071854e5/llama_index_vector_stores_milvus-0.9.6.tar.gz", hash = "sha256:6d38ac5939a570e0240687f54fbee4e1ff6c5faa2d28d25377a3f38d2ca07e2b", size = 15584, upload-time = "2026-01-13T11:46:41.394Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/04/98359849c095b5a3eb06f0649865a30b5a733b9fdbfeac3c6951253f804c/llama_index_vector_stores_milvus-0.9.6-py3-none-any.whl", hash = "sha256:916cbd9b07035ec137905970ef6a49dd77d3ece6e0a79271db35705cca5f5f84", size = 15792, upload-time = "2026-01-13T11:46:39.708Z" }, -] - -[[package]] -name = "llama-index-vector-stores-pinecone" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "pinecone" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/a0/2e2e969a133894f10b3a55b5148feef0c546ca8047b461f51f79d115c5b9/llama_index_vector_stores_pinecone-0.7.1.tar.gz", hash = "sha256:0ab3cc44f309bca1d74e58f221dade672169da01561114b067f4734293bd0280", size = 7852, upload-time = "2025-09-08T20:28:54.11Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/22/ae8c3073e4866a41eb53030db7cfecca1d84192c16a67d40a76f8d593e6d/llama_index_vector_stores_pinecone-0.7.1-py3-none-any.whl", hash = "sha256:861c4d01b3766cdca318f1285c03cd5e52dabf3d2f136cb38db421b16103129a", size = 8041, upload-time = "2025-09-08T20:28:53.406Z" }, -] - -[[package]] -name = "llama-index-vector-stores-postgres" -version = "0.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asyncpg" }, - { name = "llama-index-core" }, - { name = "pgvector" }, - { name = "psycopg2-binary" }, - { name = "sqlalchemy", extra = ["asyncio"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/78/04ff0cb9e14b8c1c3cb8716fab35c95ec2a4b551d769c65031c5c8624337/llama_index_vector_stores_postgres-0.7.3.tar.gz", hash = "sha256:7b5c62e462d681d7b8d8668b93e5b0023bfd3aaafcf76e2b4bfcf885dc3b49c6", size = 11950, upload-time = "2026-01-22T15:14:13.007Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/de/140f678d930ea869fc989adaa0140b9267c5ee0c7b971d061112d0d5b75a/llama_index_vector_stores_postgres-0.7.3-py3-none-any.whl", hash = "sha256:65b70266cc6041ab5011d64d1183d8783112ba5b38eb32ca21e00ea5b96aa058", size = 11635, upload-time = "2026-01-22T15:14:13.722Z" }, -] - -[[package]] -name = "llama-index-vector-stores-qdrant" -version = "0.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "llama-index-core" }, - { name = "qdrant-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/99/63/646ec9d7035429d4fb5488146ab7a9d55e955fb855f0a7e7fc29f6eb136f/llama_index_vector_stores_qdrant-0.9.1.tar.gz", hash = "sha256:215e24278bde44c6746d60c7df3f8811f943c20524a496e0c954eeb6449e8319", size = 14688, upload-time = "2026-01-13T11:13:07.123Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/d2/e258a2b5526fe42d6ca60a054b3830948d6411e3f075de542ba492c5f5d1/llama_index_vector_stores_qdrant-0.9.1-py3-none-any.whl", hash = "sha256:de71d0e867c04c87aa81ab35b092c32d684961c26a1cda601856faf29b21a598", size = 14944, upload-time = "2026-01-13T11:13:08.08Z" }, -] - -[[package]] -name = "llama-index-vector-stores-weaviate" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, - { name = "weaviate-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/25/75/bc95030d2c4a7c1c225a0035117442f4cd9fc1d975bde7abbfc4e94601a9/llama_index_vector_stores_weaviate-1.4.1.tar.gz", hash = "sha256:9fa3aa71732066f15349ddc796fbe230e74d345726db21d340334b0c4ebfdd3c", size = 8537, upload-time = "2025-09-08T20:45:46.877Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/cd/c70c0006a1d82f472700e6fcad9bad953f48405bb524c87888b8de215f7e/llama_index_vector_stores_weaviate-1.4.1-py3-none-any.whl", hash = "sha256:a7f32be4e1963b9718f9cbc12ac7f38db1a0a1041ba6fe45d809260a8d43f123", size = 9325, upload-time = "2025-09-08T20:45:45.845Z" }, -] - -[[package]] -name = "llama-index-workflows" -version = "2.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-instrumentation" }, - { name = "pydantic" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/7b/00c35d14dc4a7dc64c63dad8b1532f55cd5b5f8856c34f5bf587693ac270/llama_index_workflows-2.13.1.tar.gz", hash = "sha256:55cd3cff9c92a37272ab8651ad750288abc339165b009066f130cb0c5c65994b", size = 84279, upload-time = "2026-01-25T14:51:50.493Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/8d/ba97cda62829b60658a86d4c012d59e3b13c99d9b336fcf0e507d03812df/llama_index_workflows-2.13.1-py3-none-any.whl", hash = "sha256:e779078817d413b29a5297521fb71694a80e502f18dfd41e8b342f83e45f2c19", size = 107344, upload-time = "2026-01-25T14:51:49.297Z" }, -] - -[[package]] -name = "llama-parse" -version = "0.6.54" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-cloud-services" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/f6/93b5d123c480bc8c93e6dc3ea930f4f8df8da27f829bb011100ba3ce23dc/llama_parse-0.6.54.tar.gz", hash = "sha256:c707b31152155c9bae84e316fab790bbc8c85f4d8825ce5ee386ebeb7db258f1", size = 3577, upload-time = "2025-08-01T20:09:23.762Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/50/c5ccd2a50daa0a10c7f3f7d4e6992392454198cd8a7d99fcb96cb60d0686/llama_parse-0.6.54-py3-none-any.whl", hash = "sha256:c66c8d51cf6f29a44eaa8595a595de5d2598afc86e5a33a4cebe5fe228036920", size = 4879, upload-time = "2025-08-01T20:09:22.651Z" }, -] - -[[package]] -name = "llmwhisperer-client" -version = "2.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "tenacity" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/44/18d4158618ebbd76ceb8e43b8deb77f4983e6f1ccff2dffd73d6f3fb1628/llmwhisperer_client-2.6.2.tar.gz", hash = "sha256:ce846af62e7e7337dfcfe2960ec72de2989457b717ab7b9dd4110ee82c002ed0", size = 3268197, upload-time = "2026-02-23T10:52:17.634Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/eb/0f9edd21302eddc6020a1b43a78e27f361e8b6b8af7611134a58487f7d8a/llmwhisperer_client-2.6.2-py3-none-any.whl", hash = "sha256:7226344506bc85a663e4d4f8feb763f853ec9bcb6cea9bd9cf170ba135c50cdd", size = 10857, upload-time = "2026-02-23T10:52:16.325Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, -] - -[[package]] -name = "marshmallow" -version = "3.26.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095, upload-time = "2025-12-22T06:53:53.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964, upload-time = "2025-12-22T06:53:51.801Z" }, -] - -[[package]] -name = "mbstrdecoder" -version = "1.1.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "chardet" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/31/ab/05ae008357c8bdb6245ebf8a101d99f26c096e0ea20800b318153da23796/mbstrdecoder-1.1.4.tar.gz", hash = "sha256:8105ef9cf6b7d7d69fe7fd6b68a2d8f281ca9b365d7a9b670be376b2e6c81b21", size = 14527, upload-time = "2025-01-18T10:07:31.089Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/ac/5ce64a1d4cce00390beab88622a290420401f1cabf05caf2fc0995157c21/mbstrdecoder-1.1.4-py3-none-any.whl", hash = "sha256:03dae4ec50ec0d2ff4743e63fdbd5e0022815857494d35224b60775d3d934a8c", size = 7933, upload-time = "2025-01-18T10:07:29.562Z" }, -] - -[[package]] -name = "milvus-lite" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tqdm" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713, upload-time = "2025-06-30T04:23:37.028Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451, upload-time = "2025-06-30T04:23:51.747Z" }, - { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093, upload-time = "2025-06-30T04:24:06.706Z" }, - { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911, upload-time = "2025-06-30T04:24:19.434Z" }, -] - -[[package]] -name = "msal" -version = "1.34.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/c857c46d653e104019a84f22d4494f2119b4fe9f896c92b4b864b3b045cc/msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f", size = 153961, upload-time = "2025-09-22T23:05:48.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/dc/18d48843499e278538890dc709e9ee3dea8375f8be8e82682851df1b48b5/msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1", size = 116987, upload-time = "2025-09-22T23:05:47.294Z" }, -] - -[[package]] -name = "msal-extensions" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/99/5d239b6156eddf761a636bded1118414d161bd6b7b37a9335549ed159396/msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4", size = 23315, upload-time = "2025-03-14T23:51:03.902Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, -] - -[[package]] -name = "multidict" -version = "6.6.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, - { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, - { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, - { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, - { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, - { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, - { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, - { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, - { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, - { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, - { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, - { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, - { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, - { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, - { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - -[[package]] -name = "networkx" -version = "3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, -] - -[[package]] -name = "nltk" -version = "3.9.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, -] - -[[package]] -name = "numpy" -version = "2.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014, upload-time = "2025-09-09T15:56:29.966Z" }, - { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220, upload-time = "2025-09-09T15:56:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918, upload-time = "2025-09-09T15:56:34.175Z" }, - { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922, upload-time = "2025-09-09T15:56:36.149Z" }, - { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991, upload-time = "2025-09-09T15:56:40.548Z" }, - { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643, upload-time = "2025-09-09T15:56:43.343Z" }, - { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787, upload-time = "2025-09-09T15:56:46.141Z" }, - { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598, upload-time = "2025-09-09T15:56:49.844Z" }, - { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800, upload-time = "2025-09-09T15:56:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615, upload-time = "2025-09-09T15:56:54.422Z" }, - { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936, upload-time = "2025-09-09T15:56:56.541Z" }, -] - -[[package]] -name = "oauthlib" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, -] - -[[package]] -name = "openai" -version = "2.24.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/13/17e87641b89b74552ed408a92b231283786523edddc95f3545809fab673c/openai-2.24.0.tar.gz", hash = "sha256:1e5769f540dbd01cb33bc4716a23e67b9d695161a734aff9c5f925e2bf99a673", size = 658717, upload-time = "2026-02-24T20:02:07.958Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/30/844dc675ee6902579b8eef01ed23917cc9319a1c9c0c14ec6e39340c96d0/openai-2.24.0-py3-none-any.whl", hash = "sha256:fed30480d7d6c884303287bde864980a4b137b60553ffbcf9ab4a233b7a73d94", size = 1120122, upload-time = "2026-02-24T20:02:05.669Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/04/05040d7ce33a907a2a02257e601992f0cdf11c73b33f13c4492bf6c3d6d5/opentelemetry_api-1.37.0.tar.gz", hash = "sha256:540735b120355bd5112738ea53621f8d5edb35ebcd6fe21ada3ab1c61d1cd9a7", size = 64923, upload-time = "2025-09-11T10:29:01.662Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, -] - -[[package]] -name = "opentelemetry-distro" -version = "0.58b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-sdk" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/20/597f387b42c649bac39af9ff8ad5bfdc163ce1a30cdecb16474ab8e57905/opentelemetry_distro-0.58b0.tar.gz", hash = "sha256:ef993c845c11fd156046a96e5ffe1ecfe33f7282fa6149cf9decb26ff8716666", size = 2583, upload-time = "2025-09-11T11:42:12.034Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/34/53016553489592262408b72e94466403da3c84ebe044b073bbcc1a6b228b/opentelemetry_distro-0.58b0-py3-none-any.whl", hash = "sha256:d90dddc3ae93d60d917a267a0099bd72f87fa3454b49ca8799f97cb58c777ef4", size = 3346, upload-time = "2025-09-11T11:40:56.853Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/df/47fde1de15a3d5ad410e98710fac60cd3d509df5dc7ec1359b71d6bf7e70/opentelemetry_exporter_otlp-1.37.0.tar.gz", hash = "sha256:f85b1929dd0d750751cc9159376fb05aa88bb7a08b6cdbf84edb0054d93e9f26", size = 6145, upload-time = "2025-09-11T10:29:03.075Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/23/7e35e41111e3834d918e414eca41555d585e8860c9149507298bb3b9b061/opentelemetry_exporter_otlp-1.37.0-py3-none-any.whl", hash = "sha256:bd44592c6bc7fc3e5c0a9b60f2ee813c84c2800c449e59504ab93f356cc450fc", size = 7019, upload-time = "2025-09-11T10:28:44.094Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-proto" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/6c/10018cbcc1e6fff23aac67d7fd977c3d692dbe5f9ef9bb4db5c1268726cc/opentelemetry_exporter_otlp_proto_common-1.37.0.tar.gz", hash = "sha256:c87a1bdd9f41fdc408d9cc9367bb53f8d2602829659f2b90be9f9d79d0bfe62c", size = 20430, upload-time = "2025-09-11T10:29:03.605Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/13/b4ef09837409a777f3c0af2a5b4ba9b7af34872bc43609dda0c209e4060d/opentelemetry_exporter_otlp_proto_common-1.37.0-py3-none-any.whl", hash = "sha256:53038428449c559b0c564b8d718df3314da387109c4d36bd1b94c9a641b0292e", size = 18359, upload-time = "2025-09-11T10:28:44.939Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "grpcio" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d1/11/4ad0979d0bb13ae5a845214e97c8d42da43980034c30d6f72d8e0ebe580e/opentelemetry_exporter_otlp_proto_grpc-1.37.0.tar.gz", hash = "sha256:f55bcb9fc848ce05ad3dd954058bc7b126624d22c4d9e958da24d8537763bec5", size = 24465, upload-time = "2025-09-11T10:29:04.172Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/17/46630b74751031a658706bef23ac99cdc2953cd3b2d28ec90590a0766b3e/opentelemetry_exporter_otlp_proto_grpc-1.37.0-py3-none-any.whl", hash = "sha256:aee5104835bf7993b7ddaaf380b6467472abaedb1f1dbfcc54a52a7d781a3890", size = 19305, upload-time = "2025-09-11T10:28:45.776Z" }, -] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/e3/6e320aeb24f951449e73867e53c55542bebbaf24faeee7623ef677d66736/opentelemetry_exporter_otlp_proto_http-1.37.0.tar.gz", hash = "sha256:e52e8600f1720d6de298419a802108a8f5afa63c96809ff83becb03f874e44ac", size = 17281, upload-time = "2025-09-11T10:29:04.844Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/e9/70d74a664d83976556cec395d6bfedd9b85ec1498b778367d5f93e373397/opentelemetry_exporter_otlp_proto_http-1.37.0-py3-none-any.whl", hash = "sha256:54c42b39945a6cc9d9a2a33decb876eabb9547e0dcb49df090122773447f1aef", size = 19576, upload-time = "2025-09-11T10:28:46.726Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation" -version = "0.58b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "packaging" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/36/7c307d9be8ce4ee7beb86d7f1d31027f2a6a89228240405a858d6e4d64f9/opentelemetry_instrumentation-0.58b0.tar.gz", hash = "sha256:df640f3ac715a3e05af145c18f527f4422c6ab6c467e40bd24d2ad75a00cb705", size = 31549, upload-time = "2025-09-11T11:42:14.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/db/5ff1cd6c5ca1d12ecf1b73be16fbb2a8af2114ee46d4b0e6d4b23f4f4db7/opentelemetry_instrumentation-0.58b0-py3-none-any.whl", hash = "sha256:50f97ac03100676c9f7fc28197f8240c7290ca1baa12da8bfbb9a1de4f34cc45", size = 33019, upload-time = "2025-09-11T11:41:00.624Z" }, -] - -[[package]] -name = "opentelemetry-proto" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/ea/a75f36b463a36f3c5a10c0b5292c58b31dbdde74f6f905d3d0ab2313987b/opentelemetry_proto-1.37.0.tar.gz", hash = "sha256:30f5c494faf66f77faeaefa35ed4443c5edb3b0aa46dad073ed7210e1a789538", size = 46151, upload-time = "2025-09-11T10:29:11.04Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c4/25/f89ea66c59bd7687e218361826c969443c4fa15dfe89733f3bf1e2a9e971/opentelemetry_proto-1.37.0-py3-none-any.whl", hash = "sha256:8ed8c066ae8828bbf0c39229979bdf583a126981142378a9cbe9d6fd5701c6e2", size = 72534, upload-time = "2025-09-11T10:28:56.831Z" }, -] - -[[package]] -name = "opentelemetry-sdk" -version = "1.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/62/2e0ca80d7fe94f0b193135375da92c640d15fe81f636658d2acf373086bc/opentelemetry_sdk-1.37.0.tar.gz", hash = "sha256:cc8e089c10953ded765b5ab5669b198bbe0af1b3f89f1007d19acd32dc46dda5", size = 170404, upload-time = "2025-09-11T10:29:11.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/62/9f4ad6a54126fb00f7ed4bb5034964c6e4f00fcd5a905e115bd22707e20d/opentelemetry_sdk-1.37.0-py3-none-any.whl", hash = "sha256:8f3c3c22063e52475c5dbced7209495c2c16723d016d39287dfc215d1771257c", size = 131941, upload-time = "2025-09-11T10:28:57.83Z" }, -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.58b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/aa/1b/90701d91e6300d9f2fb352153fb1721ed99ed1f6ea14fa992c756016e63a/opentelemetry_semantic_conventions-0.58b0.tar.gz", hash = "sha256:6bd46f51264279c433755767bb44ad00f1c9e2367e1b42af563372c5a6fa0c25", size = 129867, upload-time = "2025-09-11T10:29:12.597Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "pandas" -version = "2.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a3/fb2734118db0af37ea7433f57f722c0a56687e14b14690edff0cdb4b7e58/pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9", size = 12529893, upload-time = "2024-09-20T13:09:09.655Z" }, - { url = "https://files.pythonhosted.org/packages/e1/0c/ad295fd74bfac85358fd579e271cded3ac969de81f62dd0142c426b9da91/pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4", size = 11363475, upload-time = "2024-09-20T13:09:14.718Z" }, - { url = "https://files.pythonhosted.org/packages/c6/2a/4bba3f03f7d07207481fed47f5b35f556c7441acddc368ec43d6643c5777/pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3", size = 15188645, upload-time = "2024-09-20T19:02:03.88Z" }, - { url = "https://files.pythonhosted.org/packages/38/f8/d8fddee9ed0d0c0f4a2132c1dfcf0e3e53265055da8df952a53e7eaf178c/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319", size = 12739445, upload-time = "2024-09-20T13:09:17.621Z" }, - { url = "https://files.pythonhosted.org/packages/20/e8/45a05d9c39d2cea61ab175dbe6a2de1d05b679e8de2011da4ee190d7e748/pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8", size = 16359235, upload-time = "2024-09-20T19:02:07.094Z" }, - { url = "https://files.pythonhosted.org/packages/1d/99/617d07a6a5e429ff90c90da64d428516605a1ec7d7bea494235e1c3882de/pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a", size = 14056756, upload-time = "2024-09-20T13:09:20.474Z" }, - { url = "https://files.pythonhosted.org/packages/29/d4/1244ab8edf173a10fd601f7e13b9566c1b525c4f365d6bee918e68381889/pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", size = 11504248, upload-time = "2024-09-20T13:09:23.137Z" }, -] - -[[package]] -name = "pastel" -version = "0.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/76/f1/4594f5e0fcddb6953e5b8fe00da8c317b8b41b547e2b3ae2da7512943c62/pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d", size = 7555, upload-time = "2020-09-16T19:21:12.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/18/a8444036c6dd65ba3624c63b734d3ba95ba63ace513078e1580590075d21/pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364", size = 5955, upload-time = "2020-09-16T19:21:11.409Z" }, -] - -[[package]] -name = "pathvalidate" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, -] - -[[package]] -name = "pdfminer-six" -version = "20250506" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "charset-normalizer" }, - { name = "cryptography" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/78/46/5223d613ac4963e1f7c07b2660fe0e9e770102ec6bda8c038400113fb215/pdfminer_six-20250506.tar.gz", hash = "sha256:b03cc8df09cf3c7aba8246deae52e0bca7ebb112a38895b5e1d4f5dd2b8ca2e7", size = 7387678, upload-time = "2025-05-06T16:17:00.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/73/16/7a432c0101fa87457e75cb12c879e1749c5870a786525e2e0f42871d6462/pdfminer_six-20250506-py3-none-any.whl", hash = "sha256:d81ad173f62e5f841b53a8ba63af1a4a355933cfc0ffabd608e568b9193909e3", size = 5620187, upload-time = "2025-05-06T16:16:58.669Z" }, -] - -[[package]] -name = "pdfplumber" -version = "0.11.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pdfminer-six" }, - { name = "pillow" }, - { name = "pypdfium2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/0d/4135821aa7b1a0b77a29fac881ef0890b46b0b002290d04915ed7acc0043/pdfplumber-0.11.7.tar.gz", hash = "sha256:fa67773e5e599de1624255e9b75d1409297c5e1d7493b386ce63648637c67368", size = 115518, upload-time = "2025-06-12T11:30:49.864Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/e0/52b67d4f00e09e497aec4f71bc44d395605e8ebcea52543242ed34c25ef9/pdfplumber-0.11.7-py3-none-any.whl", hash = "sha256:edd2195cca68bd770da479cf528a737e362968ec2351e62a6c0b71ff612ac25e", size = 60029, upload-time = "2025-06-12T11:30:48.89Z" }, -] - -[[package]] -name = "peewee" -version = "3.18.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/89/76f6f1b744c8608e0d416b588b9d63c2a500ff800065ae610f7c80f532d6/peewee-3.18.2.tar.gz", hash = "sha256:77a54263eb61aff2ea72f63d2eeb91b140c25c1884148e28e4c0f7c4f64996a0", size = 949220, upload-time = "2025-07-08T12:52:03.941Z" } - -[[package]] -name = "pgvector" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/43/9a0fb552ab4fd980680c2037962e331820f67585df740bedc4a2b50faf20/pgvector-0.4.1.tar.gz", hash = "sha256:83d3a1c044ff0c2f1e95d13dfb625beb0b65506cfec0941bfe81fd0ad44f4003", size = 30646, upload-time = "2025-04-26T18:56:37.151Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/21/b5735d5982892c878ff3d01bb06e018c43fc204428361ee9fc25a1b2125c/pgvector-0.4.1-py3-none-any.whl", hash = "sha256:34bb4e99e1b13d08a2fe82dda9f860f15ddcd0166fbb25bffe15821cbfeb7362", size = 27086, upload-time = "2025-04-26T18:56:35.956Z" }, -] - -[[package]] -name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, -] - -[[package]] -name = "pinecone" -version = "7.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "pinecone-plugin-interface" }, - { name = "python-dateutil" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/9d/07a7f2136ce04cabd21d69c057dc2915867082b0047e6873e424388d4475/pinecone-7.0.1.tar.gz", hash = "sha256:49ff7b0f5be4a2ddec5aaa709758a9f2df56baa58ad46507d081409e246a81ec", size = 207930, upload-time = "2025-05-21T19:39:01.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/88/896221e991077d353e61991b759f46d75f3b4298eb5a4aa6534c1371f4b0/pinecone-7.0.1-py3-none-any.whl", hash = "sha256:ce7b0dab3c9f7d81e75b24c13fcbca4a51371e08021faaecaf0cd9a45ca1be6c", size = 516590, upload-time = "2025-05-21T19:38:59.117Z" }, -] - -[[package]] -name = "pinecone-plugin-interface" -version = "0.0.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f4/fb/e8a4063264953ead9e2b24d9b390152c60f042c951c47f4592e9996e57ff/pinecone_plugin_interface-0.0.7.tar.gz", hash = "sha256:b8e6675e41847333aa13923cc44daa3f85676d7157324682dc1640588a982846", size = 3370, upload-time = "2024-06-05T01:57:52.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/1d/a21fdfcd6d022cb64cef5c2a29ee6691c6c103c4566b41646b080b7536a5/pinecone_plugin_interface-0.0.7-py3-none-any.whl", hash = "sha256:875857ad9c9fc8bbc074dbe780d187a2afd21f5bfe0f3b08601924a61ef1bba8", size = 6249, upload-time = "2024-06-05T01:57:50.583Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "poethepoet" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pastel" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a5/f2/273fe54a78dc5c6c8dd63db71f5a6ceb95e4648516b5aeaeff4bde804e44/poethepoet-0.37.0.tar.gz", hash = "sha256:73edf458707c674a079baa46802e21455bda3a7f82a408e58c31b9f4fe8e933d", size = 68570, upload-time = "2025-08-11T18:00:29.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/1b/5337af1a6a478d25a3e3c56b9b4b42b0a160314e02f4a0498d5322c8dac4/poethepoet-0.37.0-py3-none-any.whl", hash = "sha256:861790276315abcc8df1b4bd60e28c3d48a06db273edd3092f3c94e1a46e5e22", size = 90062, upload-time = "2025-08-11T18:00:27.595Z" }, -] - -[[package]] -name = "portalocker" -version = "3.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, -] - -[[package]] -name = "propcache" -version = "0.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" }, - { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" }, - { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" }, - { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" }, - { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" }, - { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" }, - { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" }, - { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" }, - { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" }, - { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" }, - { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" }, - { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, -] - -[[package]] -name = "proto-plus" -version = "1.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f4/ac/87285f15f7cce6d4a008f33f1757fb5a13611ea8914eb58c3d0d26243468/proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012", size = 56142, upload-time = "2025-03-10T15:54:38.843Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/6d/280c4c2ce28b1593a19ad5239c8b826871fc6ec275c21afc8e1820108039/proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", size = 50163, upload-time = "2025-03-10T15:54:37.335Z" }, -] - -[[package]] -name = "protobuf" -version = "5.29.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/57/394a763c103e0edf87f0938dafcd918d53b4c011dfc5c8ae80f3b0452dbb/protobuf-5.29.6.tar.gz", hash = "sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723", size = 425623, upload-time = "2026-02-04T22:54:40.584Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/88/9ee58ff7863c479d6f8346686d4636dd4c415b0cbeed7a6a7d0617639c2a/protobuf-5.29.6-cp310-abi3-win32.whl", hash = "sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1", size = 423357, upload-time = "2026-02-04T22:54:25.805Z" }, - { url = "https://files.pythonhosted.org/packages/1c/66/2dc736a4d576847134fb6d80bd995c569b13cdc7b815d669050bf0ce2d2c/protobuf-5.29.6-cp310-abi3-win_amd64.whl", hash = "sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda", size = 435175, upload-time = "2026-02-04T22:54:28.592Z" }, - { url = "https://files.pythonhosted.org/packages/06/db/49b05966fd208ae3f44dcd33837b6243b4915c57561d730a43f881f24dea/protobuf-5.29.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269", size = 418619, upload-time = "2026-02-04T22:54:30.266Z" }, - { url = "https://files.pythonhosted.org/packages/b7/d7/48cbf6b0c3c39761e47a99cb483405f0fde2be22cf00d71ef316ce52b458/protobuf-5.29.6-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6", size = 320284, upload-time = "2026-02-04T22:54:31.782Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dd/cadd6ec43069247d91f6345fa7a0d2858bef6af366dbd7ba8f05d2c77d3b/protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9", size = 320478, upload-time = "2026-02-04T22:54:32.909Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cb/e3065b447186cb70aa65acc70c86baf482d82bf75625bf5a2c4f6919c6a3/protobuf-5.29.6-py3-none-any.whl", hash = "sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86", size = 173126, upload-time = "2026-02-04T22:54:39.462Z" }, -] - -[[package]] -name = "psycopg2-binary" -version = "2.9.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764, upload-time = "2024-10-16T11:24:58.126Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/7d/465cc9795cf76f6d329efdafca74693714556ea3891813701ac1fee87545/psycopg2_binary-2.9.10-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0", size = 3044771, upload-time = "2024-10-16T11:20:35.234Z" }, - { url = "https://files.pythonhosted.org/packages/8b/31/6d225b7b641a1a2148e3ed65e1aa74fc86ba3fee850545e27be9e1de893d/psycopg2_binary-2.9.10-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a", size = 3275336, upload-time = "2024-10-16T11:20:38.742Z" }, - { url = "https://files.pythonhosted.org/packages/30/b7/a68c2b4bff1cbb1728e3ec864b2d92327c77ad52edcd27922535a8366f68/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539", size = 2851637, upload-time = "2024-10-16T11:20:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b1/cfedc0e0e6f9ad61f8657fd173b2f831ce261c02a08c0b09c652b127d813/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526", size = 3082097, upload-time = "2024-10-16T11:20:46.185Z" }, - { url = "https://files.pythonhosted.org/packages/18/ed/0a8e4153c9b769f59c02fb5e7914f20f0b2483a19dae7bf2db54b743d0d0/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1", size = 3264776, upload-time = "2024-10-16T11:20:50.879Z" }, - { url = "https://files.pythonhosted.org/packages/10/db/d09da68c6a0cdab41566b74e0a6068a425f077169bed0946559b7348ebe9/psycopg2_binary-2.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e", size = 3020968, upload-time = "2024-10-16T11:20:56.819Z" }, - { url = "https://files.pythonhosted.org/packages/94/28/4d6f8c255f0dfffb410db2b3f9ac5218d959a66c715c34cac31081e19b95/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f", size = 2872334, upload-time = "2024-10-16T11:21:02.411Z" }, - { url = "https://files.pythonhosted.org/packages/05/f7/20d7bf796593c4fea95e12119d6cc384ff1f6141a24fbb7df5a668d29d29/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00", size = 2822722, upload-time = "2024-10-16T11:21:09.01Z" }, - { url = "https://files.pythonhosted.org/packages/4d/e4/0c407ae919ef626dbdb32835a03b6737013c3cc7240169843965cada2bdf/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5", size = 2920132, upload-time = "2024-10-16T11:21:16.339Z" }, - { url = "https://files.pythonhosted.org/packages/2d/70/aa69c9f69cf09a01da224909ff6ce8b68faeef476f00f7ec377e8f03be70/psycopg2_binary-2.9.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47", size = 2959312, upload-time = "2024-10-16T11:21:25.584Z" }, - { url = "https://files.pythonhosted.org/packages/d3/bd/213e59854fafe87ba47814bf413ace0dcee33a89c8c8c814faca6bc7cf3c/psycopg2_binary-2.9.10-cp312-cp312-win32.whl", hash = "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64", size = 1025191, upload-time = "2024-10-16T11:21:29.912Z" }, - { url = "https://files.pythonhosted.org/packages/92/29/06261ea000e2dc1e22907dbbc483a1093665509ea586b29b8986a0e56733/psycopg2_binary-2.9.10-cp312-cp312-win_amd64.whl", hash = "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0", size = 1164031, upload-time = "2024-10-16T11:21:34.211Z" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pycparser" -version = "2.23" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, -] - -[[package]] -name = "pydantic" -version = "2.12.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, -] - -[[package]] -name = "pydantic-core" -version = "2.41.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, - { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, - { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, - { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, - { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, - { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, - { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, - { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, - { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, - { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, - { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, - { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, - { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, - { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, - { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pymilvus" -version = "2.5.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, - { name = "pandas" }, - { name = "protobuf" }, - { name = "python-dotenv" }, - { name = "setuptools" }, - { name = "ujson" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/e2/5613bc7b2af0ccd760177ca4255243c284cfc0f2cba3f10ff63325c4ca34/pymilvus-2.5.16.tar.gz", hash = "sha256:65f56b81806bc217cca3cf29b70a27d053dea4b1ffada910cf63a38f96381618", size = 1280614, upload-time = "2025-09-19T07:02:14.747Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/09/b67a55abee0a53ea50ba0de0cba6e1c0f7ca7ce2c15ffd6f40c059c25e88/pymilvus-2.5.16-py3-none-any.whl", hash = "sha256:76258a324f19c60fee247467e11cd7d6f35a64d2a9c753f5d7b1a5fa15dd6c8a", size = 243272, upload-time = "2025-09-19T07:02:12.443Z" }, -] - -[[package]] -name = "pypdf" -version = "6.13.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/17/18/9947cc201af9ccf76720fd3347bf4f70eb882ce3fcf4cb05f7443e4cf871/pypdf-6.13.3.tar.gz", hash = "sha256:f3cb822769725f1bac658c406cfc9460399043f3750c2d3e4650e0a85eacabd7", size = 6484063, upload-time = "2026-06-17T15:22:00.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/56/2967e621598987905fb8cdfadd8f8de6b5c68c9351f0523c4df8409f28f1/pypdf-6.13.3-py3-none-any.whl", hash = "sha256:c6e3f86afb625791510b02ad5480e94b63970bb957df75d44657c282ecc52224", size = 347288, upload-time = "2026-06-17T15:21:59.512Z" }, -] - -[[package]] -name = "pypdfium2" -version = "4.30.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/14/838b3ba247a0ba92e4df5d23f2bea9478edcfd72b78a39d6ca36ccd84ad2/pypdfium2-4.30.0.tar.gz", hash = "sha256:48b5b7e5566665bc1015b9d69c1ebabe21f6aee468b509531c3c8318eeee2e16", size = 140239, upload-time = "2024-05-09T18:33:17.552Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/9a/c8ff5cc352c1b60b0b97642ae734f51edbab6e28b45b4fcdfe5306ee3c83/pypdfium2-4.30.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:b33ceded0b6ff5b2b93bc1fe0ad4b71aa6b7e7bd5875f1ca0cdfb6ba6ac01aab", size = 2837254, upload-time = "2024-05-09T18:32:48.653Z" }, - { url = "https://files.pythonhosted.org/packages/21/8b/27d4d5409f3c76b985f4ee4afe147b606594411e15ac4dc1c3363c9a9810/pypdfium2-4.30.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4e55689f4b06e2d2406203e771f78789bd4f190731b5d57383d05cf611d829de", size = 2707624, upload-time = "2024-05-09T18:32:51.458Z" }, - { url = "https://files.pythonhosted.org/packages/11/63/28a73ca17c24b41a205d658e177d68e198d7dde65a8c99c821d231b6ee3d/pypdfium2-4.30.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e6e50f5ce7f65a40a33d7c9edc39f23140c57e37144c2d6d9e9262a2a854854", size = 2793126, upload-time = "2024-05-09T18:32:53.581Z" }, - { url = "https://files.pythonhosted.org/packages/d1/96/53b3ebf0955edbd02ac6da16a818ecc65c939e98fdeb4e0958362bd385c8/pypdfium2-4.30.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3d0dd3ecaffd0b6dbda3da663220e705cb563918249bda26058c6036752ba3a2", size = 2591077, upload-time = "2024-05-09T18:32:55.99Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/0394e56e7cab8b5b21f744d988400948ef71a9a892cbeb0b200d324ab2c7/pypdfium2-4.30.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc3bf29b0db8c76cdfaac1ec1cde8edf211a7de7390fbf8934ad2aa9b4d6dfad", size = 2864431, upload-time = "2024-05-09T18:32:57.911Z" }, - { url = "https://files.pythonhosted.org/packages/65/cd/3f1edf20a0ef4a212a5e20a5900e64942c5a374473671ac0780eaa08ea80/pypdfium2-4.30.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1f78d2189e0ddf9ac2b7a9b9bd4f0c66f54d1389ff6c17e9fd9dc034d06eb3f", size = 2812008, upload-time = "2024-05-09T18:32:59.886Z" }, - { url = "https://files.pythonhosted.org/packages/c8/91/2d517db61845698f41a2a974de90762e50faeb529201c6b3574935969045/pypdfium2-4.30.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:5eda3641a2da7a7a0b2f4dbd71d706401a656fea521b6b6faa0675b15d31a163", size = 6181543, upload-time = "2024-05-09T18:33:02.597Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c4/ed1315143a7a84b2c7616569dfb472473968d628f17c231c39e29ae9d780/pypdfium2-4.30.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:0dfa61421b5eb68e1188b0b2231e7ba35735aef2d867d86e48ee6cab6975195e", size = 6175911, upload-time = "2024-05-09T18:33:05.376Z" }, - { url = "https://files.pythonhosted.org/packages/7a/c4/9e62d03f414e0e3051c56d5943c3bf42aa9608ede4e19dc96438364e9e03/pypdfium2-4.30.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:f33bd79e7a09d5f7acca3b0b69ff6c8a488869a7fab48fdf400fec6e20b9c8be", size = 6267430, upload-time = "2024-05-09T18:33:08.067Z" }, - { url = "https://files.pythonhosted.org/packages/90/47/eda4904f715fb98561e34012826e883816945934a851745570521ec89520/pypdfium2-4.30.0-py3-none-win32.whl", hash = "sha256:ee2410f15d576d976c2ab2558c93d392a25fb9f6635e8dd0a8a3a5241b275e0e", size = 2775951, upload-time = "2024-05-09T18:33:10.567Z" }, - { url = "https://files.pythonhosted.org/packages/25/bd/56d9ec6b9f0fc4e0d95288759f3179f0fcd34b1a1526b75673d2f6d5196f/pypdfium2-4.30.0-py3-none-win_amd64.whl", hash = "sha256:90dbb2ac07be53219f56be09961eb95cf2473f834d01a42d901d13ccfad64b4c", size = 2892098, upload-time = "2024-05-09T18:33:13.107Z" }, - { url = "https://files.pythonhosted.org/packages/be/7a/097801205b991bc3115e8af1edb850d30aeaf0118520b016354cf5ccd3f6/pypdfium2-4.30.0-py3-none-win_arm64.whl", hash = "sha256:119b2969a6d6b1e8d55e99caaf05290294f2d0fe49c12a3f17102d01c441bd29", size = 2752118, upload-time = "2024-05-09T18:33:15.489Z" }, -] - -[[package]] -name = "pytablewriter" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dataproperty" }, - { name = "mbstrdecoder" }, - { name = "pathvalidate" }, - { name = "setuptools" }, - { name = "tabledata" }, - { name = "tcolorpy" }, - { name = "typepy", extra = ["datetime"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f6/a1/617730f290f04d347103ab40bf67d317df6691b14746f6e1ea039fb57062/pytablewriter-1.2.1.tar.gz", hash = "sha256:7bd0f4f397e070e3b8a34edcf1b9257ccbb18305493d8350a5dbc9957fced959", size = 619241, upload-time = "2025-01-01T15:37:00.04Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/21/4c/c199512f01c845dfe5a7840ab3aae6c60463b5dc2a775be72502dfd9170a/pytablewriter-1.2.1-py3-none-any.whl", hash = "sha256:e906ff7ff5151d70a5f66e0f7b75642a7f2dce8d893c265b79cc9cf6bc04ddb4", size = 91083, upload-time = "2025-01-01T15:36:55.63Z" }, -] - -[[package]] -name = "pytest" -version = "9.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, -] - -[[package]] -name = "pytest-asyncio" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, -] - -[[package]] -name = "pytest-md-report" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytablewriter" }, - { name = "pytest" }, - { name = "tcolorpy" }, - { name = "typepy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/2e/14711f6af84b1f637167b52aef179e307a580dfb54f7da8b0c06c3125453/pytest_md_report-0.8.0.tar.gz", hash = "sha256:c8e3b7f1f91a0e8e7d1b946e1b224f4f39187da0df2f812731361a436a17f472", size = 289649, upload-time = "2026-05-04T04:30:34.384Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/47/c847f9095e466e5933570d3b603eee78389b3bfc534d017b43f6cc62fa1a/pytest_md_report-0.8.0-py3-none-any.whl", hash = "sha256:d2ba54b4be2071ea91bf3a17b215f70093fd5b8148356cb33f9a7a9ac53f177a", size = 17185, upload-time = "2026-05-04T04:30:32.556Z" }, -] - -[[package]] -name = "pytest-mock" -version = "3.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/28/67172c96ba684058a4d24ffe144d64783d2a270d0af0d9e792737bddc75c/pytest_mock-3.14.1.tar.gz", hash = "sha256:159e9edac4c451ce77a5cdb9fc5d1100708d2dd4ba3c3df572f14097351af80e", size = 33241, upload-time = "2025-05-26T13:58:45.167Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/05/77b60e520511c53d1c1ca75f1930c7dd8e971d0c4379b7f4b3f9644685ba/pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0", size = 9923, upload-time = "2025-05-26T13:58:43.487Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - -[[package]] -name = "python-magic" -version = "0.4.27" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/db/0b3e28ac047452d079d375ec6798bf76a036a08182dbb39ed38116a49130/python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", size = 14677, upload-time = "2022-06-07T20:16:59.508Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/73/9f872cb81fc5c3bb48f7227872c28975f998f3e7c2b1c16e95e6432bbb90/python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3", size = 13840, upload-time = "2022-06-07T20:16:57.763Z" }, -] - -[[package]] -name = "pytz" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, -] - -[[package]] -name = "pywin32" -version = "311" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, - { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, -] - -[[package]] -name = "qdrant-client" -version = "1.16.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "httpx", extra = ["http2"] }, - { name = "numpy" }, - { name = "portalocker" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ca/7d/3cd10e26ae97b35cf856ca1dc67576e42414ae39502c51165bb36bb1dff8/qdrant_client-1.16.2.tar.gz", hash = "sha256:ca4ef5f9be7b5eadeec89a085d96d5c723585a391eb8b2be8192919ab63185f0", size = 331112, upload-time = "2025-12-12T10:58:30.866Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/13/8ce16f808297e16968269de44a14f4fef19b64d9766be1d6ba5ba78b579d/qdrant_client-1.16.2-py3-none-any.whl", hash = "sha256:442c7ef32ae0f005e88b5d3c0783c63d4912b97ae756eb5e052523be682f17d3", size = 377186, upload-time = "2025-12-12T10:58:29.282Z" }, -] - -[[package]] -name = "redis" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355, upload-time = "2024-12-06T09:50:41.956Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502, upload-time = "2024-12-06T09:50:39.656Z" }, -] - -[[package]] -name = "referencing" -version = "0.36.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, -] - -[[package]] -name = "regex" -version = "2025.9.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/49/d3/eaa0d28aba6ad1827ad1e716d9a93e1ba963ada61887498297d3da715133/regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4", size = 400917, upload-time = "2025-09-19T00:38:35.79Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/99/05859d87a66ae7098222d65748f11ef7f2dff51bfd7482a4e2256c90d72b/regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e", size = 486335, upload-time = "2025-09-19T00:36:03.661Z" }, - { url = "https://files.pythonhosted.org/packages/97/7e/d43d4e8b978890932cf7b0957fce58c5b08c66f32698f695b0c2c24a48bf/regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a", size = 289720, upload-time = "2025-09-19T00:36:05.471Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3b/ff80886089eb5dcf7e0d2040d9aaed539e25a94300403814bb24cc775058/regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab", size = 287257, upload-time = "2025-09-19T00:36:07.072Z" }, - { url = "https://files.pythonhosted.org/packages/ee/66/243edf49dd8720cba8d5245dd4d6adcb03a1defab7238598c0c97cf549b8/regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5", size = 797463, upload-time = "2025-09-19T00:36:08.399Z" }, - { url = "https://files.pythonhosted.org/packages/df/71/c9d25a1142c70432e68bb03211d4a82299cd1c1fbc41db9409a394374ef5/regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742", size = 862670, upload-time = "2025-09-19T00:36:10.101Z" }, - { url = "https://files.pythonhosted.org/packages/f8/8f/329b1efc3a64375a294e3a92d43372bf1a351aa418e83c21f2f01cf6ec41/regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425", size = 910881, upload-time = "2025-09-19T00:36:12.223Z" }, - { url = "https://files.pythonhosted.org/packages/35/9e/a91b50332a9750519320ed30ec378b74c996f6befe282cfa6bb6cea7e9fd/regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352", size = 802011, upload-time = "2025-09-19T00:36:13.901Z" }, - { url = "https://files.pythonhosted.org/packages/a4/1d/6be3b8d7856b6e0d7ee7f942f437d0a76e0d5622983abbb6d21e21ab9a17/regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d", size = 786668, upload-time = "2025-09-19T00:36:15.391Z" }, - { url = "https://files.pythonhosted.org/packages/cb/ce/4a60e53df58bd157c5156a1736d3636f9910bdcc271d067b32b7fcd0c3a8/regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56", size = 856578, upload-time = "2025-09-19T00:36:16.845Z" }, - { url = "https://files.pythonhosted.org/packages/86/e8/162c91bfe7217253afccde112868afb239f94703de6580fb235058d506a6/regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e", size = 849017, upload-time = "2025-09-19T00:36:18.597Z" }, - { url = "https://files.pythonhosted.org/packages/35/34/42b165bc45289646ea0959a1bc7531733e90b47c56a72067adfe6b3251f6/regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282", size = 788150, upload-time = "2025-09-19T00:36:20.464Z" }, - { url = "https://files.pythonhosted.org/packages/79/5d/cdd13b1f3c53afa7191593a7ad2ee24092a5a46417725ffff7f64be8342d/regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459", size = 264536, upload-time = "2025-09-19T00:36:21.922Z" }, - { url = "https://files.pythonhosted.org/packages/e0/f5/4a7770c9a522e7d2dc1fa3ffc83ab2ab33b0b22b447e62cffef186805302/regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77", size = 275501, upload-time = "2025-09-19T00:36:23.4Z" }, - { url = "https://files.pythonhosted.org/packages/df/05/9ce3e110e70d225ecbed455b966003a3afda5e58e8aec2964042363a18f4/regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5", size = 268601, upload-time = "2025-09-19T00:36:25.092Z" }, -] - -[[package]] -name = "requests" -version = "2.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, -] - -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - -[[package]] -name = "rpds-py" -version = "0.27.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e9/dd/2c0cbe774744272b0ae725f44032c77bdcab6e8bcf544bffa3b6e70c8dba/rpds_py-0.27.1.tar.gz", hash = "sha256:26a1c73171d10b7acccbded82bf6a586ab8203601e565badc74bbbf8bc5a10f8", size = 27479, upload-time = "2025-08-27T12:16:36.024Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/fe/38de28dee5df58b8198c743fe2bea0c785c6d40941b9950bac4cdb71a014/rpds_py-0.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ae2775c1973e3c30316892737b91f9283f9908e3cc7625b9331271eaaed7dc90", size = 361887, upload-time = "2025-08-27T12:13:10.233Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/4b6c7eedc7dd90986bf0fab6ea2a091ec11c01b15f8ba0a14d3f80450468/rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2643400120f55c8a96f7c9d858f7be0c88d383cd4653ae2cf0d0c88f668073e5", size = 345795, upload-time = "2025-08-27T12:13:11.65Z" }, - { url = "https://files.pythonhosted.org/packages/6f/0e/e650e1b81922847a09cca820237b0edee69416a01268b7754d506ade11ad/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16323f674c089b0360674a4abd28d5042947d54ba620f72514d69be4ff64845e", size = 385121, upload-time = "2025-08-27T12:13:13.008Z" }, - { url = "https://files.pythonhosted.org/packages/1b/ea/b306067a712988e2bff00dcc7c8f31d26c29b6d5931b461aa4b60a013e33/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a1f4814b65eacac94a00fc9a526e3fdafd78e439469644032032d0d63de4881", size = 398976, upload-time = "2025-08-27T12:13:14.368Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0a/26dc43c8840cb8fe239fe12dbc8d8de40f2365e838f3d395835dde72f0e5/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ba32c16b064267b22f1850a34051121d423b6f7338a12b9459550eb2096e7ec", size = 525953, upload-time = "2025-08-27T12:13:15.774Z" }, - { url = "https://files.pythonhosted.org/packages/22/14/c85e8127b573aaf3a0cbd7fbb8c9c99e735a4a02180c84da2a463b766e9e/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5c20f33fd10485b80f65e800bbe5f6785af510b9f4056c5a3c612ebc83ba6cb", size = 407915, upload-time = "2025-08-27T12:13:17.379Z" }, - { url = "https://files.pythonhosted.org/packages/ed/7b/8f4fee9ba1fb5ec856eb22d725a4efa3deb47f769597c809e03578b0f9d9/rpds_py-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466bfe65bd932da36ff279ddd92de56b042f2266d752719beb97b08526268ec5", size = 386883, upload-time = "2025-08-27T12:13:18.704Z" }, - { url = "https://files.pythonhosted.org/packages/86/47/28fa6d60f8b74fcdceba81b272f8d9836ac0340570f68f5df6b41838547b/rpds_py-0.27.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:41e532bbdcb57c92ba3be62c42e9f096431b4cf478da9bc3bc6ce5c38ab7ba7a", size = 405699, upload-time = "2025-08-27T12:13:20.089Z" }, - { url = "https://files.pythonhosted.org/packages/d0/fd/c5987b5e054548df56953a21fe2ebed51fc1ec7c8f24fd41c067b68c4a0a/rpds_py-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f149826d742b406579466283769a8ea448eed82a789af0ed17b0cd5770433444", size = 423713, upload-time = "2025-08-27T12:13:21.436Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ba/3c4978b54a73ed19a7d74531be37a8bcc542d917c770e14d372b8daea186/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80c60cfb5310677bd67cb1e85a1e8eb52e12529545441b43e6f14d90b878775a", size = 562324, upload-time = "2025-08-27T12:13:22.789Z" }, - { url = "https://files.pythonhosted.org/packages/b5/6c/6943a91768fec16db09a42b08644b960cff540c66aab89b74be6d4a144ba/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7ee6521b9baf06085f62ba9c7a3e5becffbc32480d2f1b351559c001c38ce4c1", size = 593646, upload-time = "2025-08-27T12:13:24.122Z" }, - { url = "https://files.pythonhosted.org/packages/11/73/9d7a8f4be5f4396f011a6bb7a19fe26303a0dac9064462f5651ced2f572f/rpds_py-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a512c8263249a9d68cac08b05dd59d2b3f2061d99b322813cbcc14c3c7421998", size = 558137, upload-time = "2025-08-27T12:13:25.557Z" }, - { url = "https://files.pythonhosted.org/packages/6e/96/6772cbfa0e2485bcceef8071de7821f81aeac8bb45fbfd5542a3e8108165/rpds_py-0.27.1-cp312-cp312-win32.whl", hash = "sha256:819064fa048ba01b6dadc5116f3ac48610435ac9a0058bbde98e569f9e785c39", size = 221343, upload-time = "2025-08-27T12:13:26.967Z" }, - { url = "https://files.pythonhosted.org/packages/67/b6/c82f0faa9af1c6a64669f73a17ee0eeef25aff30bb9a1c318509efe45d84/rpds_py-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:d9199717881f13c32c4046a15f024971a3b78ad4ea029e8da6b86e5aa9cf4594", size = 232497, upload-time = "2025-08-27T12:13:28.326Z" }, - { url = "https://files.pythonhosted.org/packages/e1/96/2817b44bd2ed11aebacc9251da03689d56109b9aba5e311297b6902136e2/rpds_py-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:33aa65b97826a0e885ef6e278fbd934e98cdcfed80b63946025f01e2f5b29502", size = 222790, upload-time = "2025-08-27T12:13:29.71Z" }, -] - -[[package]] -name = "rsa" -version = "4.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, -] - -[[package]] -name = "s3fs" -version = "2024.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiobotocore" }, - { name = "aiohttp" }, - { name = "fsspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/75/65/4b4c868cff76c036d11dc75dd91e5696dbf16ce626514166f35d5f4a930f/s3fs-2024.10.0.tar.gz", hash = "sha256:58b8c3650f8b99dbedf361543da3533aac8707035a104db5d80b094617ad4a3f", size = 75916, upload-time = "2024-10-21T01:45:49.967Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/44/bb9ff095ae7b1b6908480f683b6ca6b71c2105d343a5e5cb25334b01f5fa/s3fs-2024.10.0-py3-none-any.whl", hash = "sha256:7a2025d60d5b1a6025726b3a5e292a8e5aa713abc3b16fd1f81735181f7bb282", size = 29855, upload-time = "2024-10-21T01:45:47.905Z" }, -] - -[package.optional-dependencies] -boto3 = [ - { name = "aiobotocore", extra = ["boto3"] }, -] - -[[package]] -name = "s3transfer" -version = "0.10.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/0a/1cdbabf9edd0ea7747efdf6c9ab4e7061b085aa7f9bfc36bb1601563b069/s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7", size = 145287, upload-time = "2024-11-20T21:06:05.981Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/05/7957af15543b8c9799209506df4660cba7afc4cf94bfb60513827e96bed6/s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e", size = 83175, upload-time = "2024-11-20T21:06:03.961Z" }, -] - -[[package]] -name = "setuptools" -version = "80.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, -] - -[[package]] -name = "singleton-decorator" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/98/a8b5c919bee1152a9a1afd82014431f8db5882699754de50d1b3aba4d136/singleton-decorator-1.0.0.tar.gz", hash = "sha256:1a90ad8a8a738be591c9c167fdd677c5d4a43d1bc6b1c128227be1c5e03bee07", size = 2791, upload-time = "2017-08-10T19:52:45.903Z" } - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - -[[package]] -name = "soupsieve" -version = "2.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/e6/21ccce3262dd4889aa3332e5a119a3491a95e8f60939870a3a035aabac0d/soupsieve-2.8.tar.gz", hash = "sha256:e2dd4a40a628cb5f28f6d4b0db8800b8f581b65bb380b97de22ba5ca8d72572f", size = 103472, upload-time = "2025-08-27T15:39:51.78Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/a0/bb38d3b76b8cae341dad93a2dd83ab7462e6dbcdd84d43f54ee60a8dc167/soupsieve-2.8-py3-none-any.whl", hash = "sha256:0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c", size = 36679, upload-time = "2025-08-27T15:39:50.179Z" }, -] - -[[package]] -name = "sqlalchemy" -version = "2.0.43" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/bc/d59b5d97d27229b0e009bd9098cd81af71c2fa5549c580a0a67b9bed0496/sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417", size = 9762949, upload-time = "2025-08-11T14:24:58.438Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/db/20c78f1081446095450bdc6ee6cc10045fce67a8e003a5876b6eaafc5cc4/sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24", size = 2134891, upload-time = "2025-08-11T15:51:13.019Z" }, - { url = "https://files.pythonhosted.org/packages/45/0a/3d89034ae62b200b4396f0f95319f7d86e9945ee64d2343dcad857150fa2/sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83", size = 2123061, upload-time = "2025-08-11T15:51:14.319Z" }, - { url = "https://files.pythonhosted.org/packages/cb/10/2711f7ff1805919221ad5bee205971254845c069ee2e7036847103ca1e4c/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9", size = 3320384, upload-time = "2025-08-11T15:52:35.088Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0e/3d155e264d2ed2778484006ef04647bc63f55b3e2d12e6a4f787747b5900/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48", size = 3329648, upload-time = "2025-08-11T15:56:34.153Z" }, - { url = "https://files.pythonhosted.org/packages/5b/81/635100fb19725c931622c673900da5efb1595c96ff5b441e07e3dd61f2be/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687", size = 3258030, upload-time = "2025-08-11T15:52:36.933Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ed/a99302716d62b4965fded12520c1cbb189f99b17a6d8cf77611d21442e47/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe", size = 3294469, upload-time = "2025-08-11T15:56:35.553Z" }, - { url = "https://files.pythonhosted.org/packages/5d/a2/3a11b06715149bf3310b55a98b5c1e84a42cfb949a7b800bc75cb4e33abc/sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d", size = 2098906, upload-time = "2025-08-11T15:55:00.645Z" }, - { url = "https://files.pythonhosted.org/packages/bc/09/405c915a974814b90aa591280623adc6ad6b322f61fd5cff80aeaef216c9/sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a", size = 2126260, upload-time = "2025-08-11T15:55:02.965Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d9/13bdde6521f322861fab67473cec4b1cc8999f3871953531cf61945fad92/sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc", size = 1924759, upload-time = "2025-08-11T15:39:53.024Z" }, -] - -[package.optional-dependencies] -asyncio = [ - { name = "greenlet" }, -] - -[[package]] -name = "striprtf" -version = "0.0.26" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/20/3d419008265346452d09e5dadfd5d045b64b40d8fc31af40588e6c76997a/striprtf-0.0.26.tar.gz", hash = "sha256:fdb2bba7ac440072d1c41eab50d8d74ae88f60a8b6575c6e2c7805dc462093aa", size = 6258, upload-time = "2023-07-20T14:30:36.29Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/cf/0fea4f4ba3fc2772ac2419278aa9f6964124d4302117d61bc055758e000c/striprtf-0.0.26-py3-none-any.whl", hash = "sha256:8c8f9d32083cdc2e8bfb149455aa1cc5a4e0a035893bedc75db8b73becb3a1bb", size = 6914, upload-time = "2023-07-20T14:30:35.338Z" }, -] - -[[package]] -name = "tabledata" -version = "1.3.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dataproperty" }, - { name = "typepy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/35/171c8977162f1163368406deddde4c59673b62bd0cb2f34948a02effb075/tabledata-1.3.4.tar.gz", hash = "sha256:e9649cab129d718f3bff4150083b77f8a78c30f6634a30caf692b10fdc60cb97", size = 25074, upload-time = "2024-12-31T14:12:31.198Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/64/fa4160151976ee4b2cf0c1217a99443ffaeb991956feddfeac9eee9952f8/tabledata-1.3.4-py3-none-any.whl", hash = "sha256:1f56e433bfdeb89f4487abfa48c4603a3b07c5d3a3c7e05ff73dd018c24bd0d4", size = 11820, upload-time = "2024-12-31T14:12:28.584Z" }, -] - -[[package]] -name = "tcolorpy" -version = "0.1.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/cc/44f2d81d8f9093aad81c3467a5bf5718d2b5f786e887b6e4adcfc17ec6b9/tcolorpy-0.1.7.tar.gz", hash = "sha256:0fbf6bf238890bbc2e32662aa25736769a29bf6d880328f310c910a327632614", size = 299437, upload-time = "2024-12-29T15:24:23.847Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/a2/ed023f2edd1e011b4d99b6727bce8253842d66c3fbf9ed0a26fc09a92571/tcolorpy-0.1.7-py3-none-any.whl", hash = "sha256:26a59d52027e175a37e0aba72efc99dda43f074db71f55b316d3de37d3251378", size = 8096, upload-time = "2024-12-29T15:24:21.33Z" }, -] - -[[package]] -name = "tenacity" -version = "9.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, -] - -[[package]] -name = "tiktoken" -version = "0.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "regex" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728, upload-time = "2025-10-06T20:21:52.756Z" }, - { url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049, upload-time = "2025-10-06T20:21:53.782Z" }, - { url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008, upload-time = "2025-10-06T20:21:54.832Z" }, - { url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665, upload-time = "2025-10-06T20:21:56.129Z" }, - { url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230, upload-time = "2025-10-06T20:21:57.546Z" }, - { url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688, upload-time = "2025-10-06T20:21:58.619Z" }, - { url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694, upload-time = "2025-10-06T20:21:59.876Z" }, -] - -[[package]] -name = "tokenizers" -version = "0.22.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, - { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, - { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, - { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, - { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, - { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, - { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, - { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, - { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, - { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, - { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, - { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, - { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, - { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, - { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, -] - -[[package]] -name = "tqdm" -version = "4.67.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, -] - -[[package]] -name = "typepy" -version = "1.3.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mbstrdecoder" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/59/4c39942077d7de285f762a91024dbda731be693591732977358f77d120fb/typepy-1.3.4.tar.gz", hash = "sha256:89c1f66de6c6133209c43a94d23431d320ba03ef5db18f241091ea594035d9de", size = 39558, upload-time = "2024-12-29T09:18:15.774Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/31/e393c3830bdedd01735bd195c85ac3034b6bcaf6c18142bab60a4047ca36/typepy-1.3.4-py3-none-any.whl", hash = "sha256:d5ed3e0c7f49521bff0603dd08cf8d453371cf68d65a29d3d0038552ccc46e2e", size = 31449, upload-time = "2024-12-29T09:18:13.135Z" }, -] - -[package.optional-dependencies] -datetime = [ - { name = "packaging" }, - { name = "python-dateutil" }, - { name = "pytz" }, -] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] - -[[package]] -name = "typing-inspect" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] - -[[package]] -name = "tzdata" -version = "2025.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, -] - -[[package]] -name = "ujson" -version = "5.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/78/937198ea8708182dd1edbf0237bf255a96feab3f511691ad08b84da98e5d/ujson-5.12.1.tar.gz", hash = "sha256:5b7e96406c301a1366534479a7352ec40ec68bb327c0c119091635acd5925e35", size = 7164538, upload-time = "2026-05-05T22:05:01.354Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/40/dbb8e2fe6ee33769602fba203dacaa3963b6599f0d0aefdf2b8811af5f70/ujson-5.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:10f44bd08ae52ee23ca6e8b472692e5da1768af2d53ff1bad6f40b532e0bc7ee", size = 57951, upload-time = "2026-05-05T22:03:31.606Z" }, - { url = "https://files.pythonhosted.org/packages/8d/db/627472e6b4ac34148ea52e6d3d15f6f366fc21c72fe7d6c7d3729d4b3ac5/ujson-5.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cc6ea753b7303fa5629fa9ac9257ea4b001c4d72583b2bb36ff1855a07db49f", size = 55562, upload-time = "2026-05-05T22:03:32.853Z" }, - { url = "https://files.pythonhosted.org/packages/be/59/1248c966da197ae7d2673542444a2d9a1ff7c46e3ec2a302c3caf902b922/ujson-5.12.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:289f13095764d03734adfa10107da9b530ceb64dc1b02a5f507588d978d5b7df", size = 59448, upload-time = "2026-05-05T22:03:34.143Z" }, - { url = "https://files.pythonhosted.org/packages/d5/d7/60c1ca71a09c0654c3edca1192a18fc55e6cc06107be86d7d3f2b39fb29b/ujson-5.12.1-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:427893168d074e59214b0ee058337c57f5bb80175cdd5b4799a9c931aae22022", size = 61608, upload-time = "2026-05-05T22:03:35.386Z" }, - { url = "https://files.pythonhosted.org/packages/d5/0a/c619525576219bfc50084100117481b1a732a16716a3878355570995de4e/ujson-5.12.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7a81724d5d90a2da7155d15d8b156ce57eaed7cdd622df813f36a8e612fd4c8", size = 59113, upload-time = "2026-05-05T22:03:37.555Z" }, - { url = "https://files.pythonhosted.org/packages/18/4d/79c1674036085e8dfdb77f8d87c1fd2896e97e6affd117c5e8ecc40f0ae4/ujson-5.12.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3a6efff7dc6515416366819de4a1bc449b77107c5b48508b101fd40f7f8bec08", size = 1038914, upload-time = "2026-05-05T22:03:38.954Z" }, - { url = "https://files.pythonhosted.org/packages/94/b1/9409bba17189ee282b6314cdf0ecdcc72e3d38cd565c870c0227d0494569/ujson-5.12.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:77a71fe53427a0cf49d56eafd801d9f7e203b784b7f99cc717783fd6f6f7b732", size = 1198408, upload-time = "2026-05-05T22:03:40.943Z" }, - { url = "https://files.pythonhosted.org/packages/4b/ad/fafbce7ac59f1a10a83892d0a34add23cc06492308e1330493aab707dc20/ujson-5.12.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea3bed53d2ea8e5642e814a9e41f3e29420a8067874ba03ace8c0462e160490c", size = 1091451, upload-time = "2026-05-05T22:03:42.739Z" }, - { url = "https://files.pythonhosted.org/packages/5a/1f/76fc9d5b1dcb9eb73ed45fd56e5114391bd30808eb1cea7f8bc5c9a64324/ujson-5.12.1-cp312-cp312-win32.whl", hash = "sha256:758e5c8fbe4e6d483041e03b307b01fb5d2f2dd4452d4d4b927ab902e188939e", size = 41049, upload-time = "2026-05-05T22:03:44.341Z" }, - { url = "https://files.pythonhosted.org/packages/35/2a/7ce3b6fda10d05b79a245db03405734b521ba3da6c377f173b018dce6d4e/ujson-5.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:f6074d3d3267ba1914c624b6e1fa3d8152648ff36b0ab77ddf83b92db488c30d", size = 45330, upload-time = "2026-05-05T22:03:45.828Z" }, - { url = "https://files.pythonhosted.org/packages/d7/66/5a37bba7a2e2ab36ae467521c4511e6593ad74c869f62ec4ba6330f3f71e/ujson-5.12.1-cp312-cp312-win_arm64.whl", hash = "sha256:7642a41520ac1b2bc25ea282b66b8da522cc43424442e6fb5e039be4d4f96530", size = 39828, upload-time = "2026-05-05T22:03:47.123Z" }, - { url = "https://files.pythonhosted.org/packages/6d/26/c9d0479236b3f5690d6a8bb45f708aabc2c91ca80d275eba24b1e9e464ab/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c419bf42ae40963fc27f70c59e24e9a97f5cf168dbce2c572f3c0ce3595912", size = 56153, upload-time = "2026-05-05T22:04:40.326Z" }, - { url = "https://files.pythonhosted.org/packages/ee/c8/785f4e132500aff2f1fd2bd4a4b86fe396a5519f830a098358c90ebb92ee/ujson-5.12.1-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0be2b4f2f547b9f0f3d902640e410e5a2fc851576cbe033c88445a23e3e7aef1", size = 57352, upload-time = "2026-05-05T22:04:42.005Z" }, - { url = "https://files.pythonhosted.org/packages/8f/13/b688a905653871b10b4ff0403c2ff562c17a0bd50be0d44324f3c85ca48f/ujson-5.12.1-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:4ea0c490c702c20495e97345acfcf0c2f3153e658ef537ff111929c48b89e10a", size = 45988, upload-time = "2026-05-05T22:04:43.36Z" }, -] - -[[package]] -name = "unstract-core" -version = "0.0.1" -source = { editable = "../unstract/core" } -dependencies = [ - { name = "httpx" }, - { name = "kombu" }, - { name = "redis" }, - { name = "requests" }, -] - -[package.metadata] -requires-dist = [ - { name = "flask", marker = "extra == 'flask'", specifier = "~=3.1.3" }, - { name = "httpx", specifier = ">=0.27.0" }, - { name = "kombu", specifier = "~=5.5.3" }, - { name = "redis", specifier = "~=5.2.1" }, - { name = "requests", specifier = "==2.33.0" }, -] -provides-extras = ["flask"] - -[[package]] -name = "unstract-flags" -version = "0.0.1" -source = { editable = "../unstract/flags" } -dependencies = [ - { name = "grpcio" }, - { name = "grpcio-tools" }, - { name = "protobuf" }, -] - -[package.metadata] -requires-dist = [ - { name = "grpcio", specifier = ">=1.60.0" }, - { name = "grpcio-tools", specifier = ">=1.60.0" }, - { name = "protobuf", specifier = ">=4.25.0" }, -] - -[[package]] -name = "unstract-prompt-service" -version = "0.0.1" -source = { editable = "." } -dependencies = [ - { name = "flask" }, - { name = "json-repair" }, - { name = "llama-index" }, - { name = "nltk" }, - { name = "peewee" }, - { name = "python-dotenv" }, - { name = "redis" }, - { name = "requests" }, - { name = "unstract-core" }, - { name = "unstract-flags" }, - { name = "unstract-sdk1", extra = ["aws", "azure", "gcs"] }, -] - -[package.dev-dependencies] -deploy = [ - { name = "gunicorn" }, - { name = "opentelemetry-distro" }, - { name = "opentelemetry-exporter-otlp" }, -] -dev = [ - { name = "debugpy" }, - { name = "poethepoet" }, -] -test = [ - { name = "flask-wtf" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-md-report" }, - { name = "pytest-mock" }, - { name = "python-dotenv" }, -] - -[package.metadata] -requires-dist = [ - { name = "flask", specifier = "~=3.1" }, - { name = "json-repair", specifier = "~=0.42.0" }, - { name = "llama-index", specifier = ">=0.14.13" }, - { name = "nltk", specifier = "~=3.9" }, - { name = "peewee", specifier = "~=3.16" }, - { name = "python-dotenv", specifier = "==1.2.2" }, - { name = "redis", specifier = ">=5.0.3,<5.3" }, - { name = "requests", specifier = ">=2.33.0,<3.0" }, - { name = "unstract-core", editable = "../unstract/core" }, - { name = "unstract-flags", editable = "../unstract/flags" }, - { name = "unstract-sdk1", extras = ["aws", "gcs", "azure"], editable = "../unstract/sdk1" }, -] - -[package.metadata.requires-dev] -deploy = [ - { name = "gunicorn", specifier = "~=23.0" }, - { name = "opentelemetry-distro" }, - { name = "opentelemetry-exporter-otlp" }, -] -dev = [ - { name = "debugpy", specifier = ">=1.8.14" }, - { name = "poethepoet", specifier = ">=0.33.1" }, -] -test = [ - { name = "flask-wtf", specifier = "~=1.1" }, - { name = "pytest", specifier = "~=9.0.3" }, - { name = "pytest-asyncio", specifier = ">=0.23.0" }, - { name = "pytest-md-report", specifier = ">=0.6.2" }, - { name = "pytest-mock", specifier = "~=3.14.0" }, - { name = "python-dotenv", specifier = "==1.2.2" }, -] - -[[package]] -name = "unstract-sdk1" -source = { editable = "../unstract/sdk1" } -dependencies = [ - { name = "filetype" }, - { name = "httpx" }, - { name = "jsonschema" }, - { name = "litellm" }, - { name = "llama-index" }, - { name = "llama-index-vector-stores-milvus" }, - { name = "llama-index-vector-stores-pinecone" }, - { name = "llama-index-vector-stores-postgres" }, - { name = "llama-index-vector-stores-qdrant" }, - { name = "llama-index-vector-stores-weaviate" }, - { name = "llama-parse" }, - { name = "llmwhisperer-client" }, - { name = "pdfplumber" }, - { name = "python-dotenv" }, - { name = "python-magic" }, - { name = "qdrant-client" }, - { name = "redis" }, - { name = "singleton-decorator" }, - { name = "tiktoken" }, - { name = "unstract-core" }, -] - -[package.optional-dependencies] -aws = [ - { name = "boto3" }, - { name = "s3fs", extra = ["boto3"] }, -] -azure = [ - { name = "adlfs" }, -] -gcs = [ - { name = "gcsfs" }, -] - -[package.metadata] -requires-dist = [ - { name = "adlfs", marker = "extra == 'azure'", specifier = "~=2024.7.0" }, - { name = "boto3", marker = "extra == 'aws'", specifier = "~=1.34.131" }, - { name = "filetype", specifier = "~=1.2.0" }, - { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, - { name = "httpx", specifier = ">=0.25.2" }, - { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, - { name = "llama-index", specifier = ">=0.14.13" }, - { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, - { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, - { name = "llama-index-vector-stores-postgres", specifier = ">=0.7.3" }, - { name = "llama-index-vector-stores-qdrant", specifier = ">=0.9.1" }, - { name = "llama-index-vector-stores-weaviate", specifier = ">=1.4.1" }, - { name = "llama-parse", specifier = ">=0.6.0" }, - { name = "llmwhisperer-client", specifier = ">=2.6.2" }, - { name = "pdfplumber", specifier = ">=0.11.2" }, - { name = "python-dotenv", specifier = "==1.2.2" }, - { name = "python-magic", specifier = "~=0.4.27" }, - { name = "qdrant-client", specifier = ">=1.16.0,<1.17.0" }, - { name = "redis", specifier = ">=5.2.1" }, - { name = "s3fs", extras = ["boto3"], marker = "extra == 'aws'", specifier = "~=2024.10.0" }, - { name = "singleton-decorator", specifier = "~=1.0.0" }, - { name = "tiktoken", specifier = "~=0.12.0" }, - { name = "unstract-core", editable = "../unstract/core" }, -] -provides-extras = ["aws", "azure", "gcs"] - -[package.metadata.requires-dev] -dev = [ - { name = "docutils", specifier = "~=0.20.1" }, - { name = "mypy", specifier = "~=1.2.0" }, - { name = "pre-commit", specifier = "~=3.3.1" }, - { name = "pycln", specifier = ">=2.5.0" }, - { name = "pytest", specifier = ">=9.0.3" }, - { name = "ruff", specifier = ">=0.2.2,<1.0.0" }, - { name = "yamllint", specifier = ">=1.35.1" }, -] -docs = [{ name = "lazydocs", specifier = "~=0.4.8" }] -test = [ - { name = "parameterized", specifier = "==0.9.0" }, - { name = "pytest", specifier = ">=9.0.3" }, - { name = "pytest-asyncio", specifier = ">=0.24.0" }, - { name = "pytest-cov", specifier = ">=6.0.0" }, - { name = "pytest-md-report", specifier = ">=0.6.2" }, - { name = "pytest-mock", specifier = "==3.14.0" }, -] - -[[package]] -name = "urllib3" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, -] - -[[package]] -name = "validators" -version = "0.35.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/66/a435d9ae49850b2f071f7ebd8119dd4e84872b01630d6736761e6e7fd847/validators-0.35.0.tar.gz", hash = "sha256:992d6c48a4e77c81f1b4daba10d16c3a9bb0dbb79b3a19ea847ff0928e70497a", size = 73399, upload-time = "2025-05-01T05:42:06.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/6e/3e955517e22cbdd565f2f8b2e73d52528b14b8bcfdb04f62466b071de847/validators-0.35.0-py3-none-any.whl", hash = "sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd", size = 44712, upload-time = "2025-05-01T05:42:04.203Z" }, -] - -[[package]] -name = "vine" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, -] - -[[package]] -name = "weaviate-client" -version = "4.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "authlib" }, - { name = "deprecation" }, - { name = "grpcio" }, - { name = "httpx" }, - { name = "protobuf" }, - { name = "pydantic" }, - { name = "validators" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/0e/e4582b007427187a9fde55fa575db4b766c81929d2b43a3dd8becce50567/weaviate_client-4.17.0.tar.gz", hash = "sha256:731d58d84b0989df4db399b686357ed285fb95971a492ccca8dec90bb2343c51", size = 769019, upload-time = "2025-09-26T11:20:27.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/c5/2da3a45866da7a935dab8ad07be05dcaee48b3ad4955144583b651929be7/weaviate_client-4.17.0-py3-none-any.whl", hash = "sha256:60e4a355b90537ee1e942ab0b76a94750897a13d9cf13c5a6decbd166d0ca8b5", size = 582763, upload-time = "2025-09-26T11:20:25.864Z" }, -] - -[[package]] -name = "werkzeug" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/61/f1/ee81806690a87dab5f5653c1f146c92bc066d7f4cebc603ef88eb9e13957/werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", size = 864736, upload-time = "2026-02-19T15:17:18.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, -] - -[[package]] -name = "wrapt" -version = "1.17.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, - { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, - { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, - { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, - { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, - { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, - { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, -] - -[[package]] -name = "wtforms" -version = "3.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/01/e4/633d080897e769ed5712dcfad626e55dbd6cf45db0ff4d9884315c6a82da/wtforms-3.2.1.tar.gz", hash = "sha256:df3e6b70f3192e92623128123ec8dca3067df9cfadd43d59681e210cfb8d4682", size = 137801, upload-time = "2024-10-21T11:34:00.108Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/c9/2088fb5645cd289c99ebe0d4cdcc723922a1d8e1beaefb0f6f76dff9b21c/wtforms-3.2.1-py3-none-any.whl", hash = "sha256:583bad77ba1dd7286463f21e11aa3043ca4869d03575921d1a1698d0715e0fd4", size = 152454, upload-time = "2024-10-21T11:33:58.44Z" }, -] - -[[package]] -name = "yarl" -version = "1.20.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" }, - { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" }, - { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" }, - { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" }, - { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" }, - { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" }, - { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" }, - { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" }, - { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" }, - { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" }, - { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" }, - { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" }, - { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" }, - { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" }, - { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, -] - -[[package]] -name = "zipp" -version = "3.23.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, -] diff --git a/pyproject.toml b/pyproject.toml index 17bc80034f..d46632ef40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,7 +178,6 @@ keep-dict-typing = true [tool.pytest.ini_options] python_files = ["tests.py", "test_*.py", "*_tests.py"] -DJANGO_SETTINGS_MODULE = "backend.settings.test_cases" testpaths = ["tests"] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", @@ -186,6 +185,7 @@ markers = [ "unit: marks tests as unit (no external services required)", "e2e: marks tests as end-to-end (require full platform stack)", "critical: marks tests covering a critical path (see tests/critical_paths.yaml)", + "critical_path(path_id): attests coverage of the named critical path(s) when this test passes", ] [tool.coverage.run] diff --git a/run-platform.sh b/run-platform.sh index 40181806f8..7ce14347bc 100755 --- a/run-platform.sh +++ b/run-platform.sh @@ -355,7 +355,7 @@ first_setup=false services=($(VERSION=$opt_version $docker_compose_cmd -f "$script_dir/docker/docker-compose.build.yaml" config --services)) # Add workers manually for env setup services+=("workers") -ignore_services=("tool-structure" "tool-sidecar" "tool-classifier" "tool-text_extractor" "worker-unified") +ignore_services=("tool-sidecar" "tool-classifier" "tool-text_extractor" "worker-unified") current_version="" target_branch="" diff --git a/runner/pyproject.toml b/runner/pyproject.toml index ba489a48ce..669cd348cb 100644 --- a/runner/pyproject.toml +++ b/runner/pyproject.toml @@ -8,7 +8,7 @@ readme = "README.md" license = { text = "MIT" } dependencies = [ - "docker==6.1.3", + "docker==7.1.0", "flask~=3.1.3", "python-dotenv>=1.2.2", "redis~=5.2.1", diff --git a/runner/uv.lock b/runner/uv.lock index 7128e6fdbc..becef7feda 100644 --- a/runner/uv.lock +++ b/runner/uv.lock @@ -146,18 +146,16 @@ wheels = [ [[package]] name = "docker" -version = "6.1.3" +version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, { name = "requests" }, { name = "urllib3" }, - { name = "websocket-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f0/73/f7c9a14e88e769f38cb7fb45aa88dfd795faa8e18aea11bababf6e068d5e/docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20", size = 259301, upload-time = "2023-06-01T14:24:49.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/be/3032490fa33b36ddc8c4b1da3252c6f974e7133f1a50de00c6b85cca203a/docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9", size = 148096, upload-time = "2023-06-01T14:24:47.769Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, ] [[package]] @@ -910,7 +908,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "docker", specifier = "==6.1.3" }, + { name = "docker", specifier = "==7.1.0" }, { name = "flask", specifier = "~=3.1.3" }, { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "redis", specifier = "~=5.2.1" }, @@ -954,15 +952,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, ] -[[package]] -name = "websocket-client" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e6/30/fba0d96b4b5fbf5948ed3f4681f7da2f9f64512e1d303f94b4cc174c24a5/websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da", size = 54648, upload-time = "2024-04-23T22:16:16.976Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" }, -] - [[package]] name = "werkzeug" version = "3.1.6" diff --git a/tests/README.md b/tests/README.md index e5677bffc5..40abddd1bf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -216,11 +216,48 @@ Developers can scope local runs however they like via positional args, `--from-f | `tests/e2e//` | HTTP-level tests against a running platform. | | `tests/e2e/hurl/` | Hurl-based HTTP suites. | -After adding tests, either: -1. Reuse an existing group whose `paths` already cover your file, **or** -2. Add a new group to `groups.yaml` (and, if relevant, a `critical_paths.yaml` entry that lists it in `covered_by`). +### Backend: no registration needed + +1. Write a normal test in `backend//tests/test_.py`. The filename + **must** match `test_*.py`, `*_test.py`, `*_tests.py`, or Django's per-app + `tests.py` — anything else is silently never collected. Prefer `test_*.py` + for consistency. +2. Tier is inferred, not declared. A test that touches the database (subclasses + Django `TestCase`/`APITestCase`, or uses `@pytest.mark.django_db`) is + auto-marked `integration` by `backend/conftest.py` and runs in + `integration-backend` against a rig-provisioned Postgres/Redis. Everything + else runs in `unit-backend`. No marker, no `groups.yaml` edit. +3. Don't stub the environment. The rig runs the whole backend tree in one + pytest session with Django fully loaded — tests that patch `sys.modules`, + assume import order, or assume they run alone will break under collection. + Use `unittest.mock.patch` on real modules; mock only true externals + (LLM SDKs, third-party APIs) — never the ORM or serializers. +4. Seed data per test in `setUp` via the ORM. Schema comes from migrations + (run once per session); each test's writes roll back automatically. +5. Needs credentials CI doesn't have (external DB/SaaS)? Guard with + `self.skipTest(" not set")` in `setUp` — it skips in CI and runs + locally when the env vars are exported. + +Run it locally: -Validate with `python -m tests.rig validate` before pushing. +```bash +tox -e groups -- unit-backend --no-coverage # pure tests +tox -e groups -- integration-backend --no-coverage # DB tests (needs Docker) +``` + +List what a group would run (no DB required): + +```bash +cd backend && uv run --group test pytest . -m integration --collect-only -q +``` + +### Other services / new areas + +Outside backend, markers are manual (`workers` enforces `--strict-markers`). +Either reuse an existing group whose `paths` already cover your file, or add a +group to `groups.yaml` (and, if relevant, list it in a `critical_paths.yaml` +entry's `covered_by`). Validate with `python -m tests.rig validate` before +pushing. --- diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yaml index a513633aaa..a5cbbee0c3 100644 --- a/tests/compose/docker-compose.test.yaml +++ b/tests/compose/docker-compose.test.yaml @@ -11,17 +11,12 @@ services: backend: # Defaults to `latest` so contributors can run without setting - # UNSTRACT_TEST_VERSION. CI will pin it to the built image SHA once the - # image-build wiring lands (see .github/workflows/ci-test-e2e.yaml header). + # UNSTRACT_TEST_VERSION. CI pins it to the commit SHA of the images built + # in the same run (see the e2e job in .github/workflows/ci-test.yaml). image: unstract/backend:${UNSTRACT_TEST_VERSION:-latest} environment: - ENVIRONMENT=test - prompt-service: - image: unstract/prompt-service:${UNSTRACT_TEST_VERSION:-latest} - environment: - - ENVIRONMENT=test - platform-service: image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest} environment: diff --git a/tests/critical_paths.yaml b/tests/critical_paths.yaml index 3a062c94d2..2a2ff8c06b 100644 --- a/tests/critical_paths.yaml +++ b/tests/critical_paths.yaml @@ -3,65 +3,92 @@ # A "critical path" is an end-to-end user or system flow whose failure would # constitute a production incident. The rig reports: # ✅ covered — at least one group in `covered_by` ran green this build -# ⚠️ gap — `covered_by` is empty OR no group covering it ran +# ⚠️ gap — no covering group ran green this build # ❌ regression — a path that was ✅ on the cached main baseline is now not ✅ # +# Only one kind of gap gates --fail-on-critical-gap: +# • in-scope gap — a covering group ran in this tier but not green; fails. +# • out-of-scope gap — covered only by an unrun tier, or no group declared; +# warn-only (a tier can't fail for coverage it can't run). +# +# Only wire `covered_by` to a group that really exercises the path — a bogus +# mapping fails the build when that group breaks, for the wrong reason. +# +# `proof` (default: group): +# • group — covered when any covered_by group runs green. Coarse: survives +# the covering test being deleted. +# • marker — additionally requires ≥1 passing test carrying +# @pytest.mark.critical_path("") in this build. Flip each +# path to marker once its tests are marked. +# # We intentionally do NOT chase 100% coverage. Focus on filling these gaps first. version: 1 paths: - id: auth-login description: "User can log in and obtain a session cookie." - entry: "POST /api/v1/auth/login" - # Honest declaration: e2e-smoke only exercises the session sentinel + - # /health/, never POST /api/v1/auth/login. Track as a gap until a real - # login test exists (likely under tests/e2e/smoke/ or tests/e2e/auth/). - covered_by: [] + covered_by: [e2e-login] + proof: marker - id: adapter-register-llm description: "Register and validate an LLM adapter." - entry: "POST /api/v1/adapter/" - # Honest declaration: unit-backend is currently optional/gated and - # e2e-smoke only hits /health/. Track as a gap until a real adapter test - # exists (likely under tests/e2e/smoke/ or a new tests/e2e/adapters/ group). - covered_by: [] + covered_by: [integration-backend] + proof: marker + + - id: workflow-author + description: "Create a workflow; its source+destination endpoints materialise and are configurable." + covered_by: [integration-backend] + proof: marker - id: workflow-create-execute description: "Create a workflow, configure source+destination, execute, poll, fetch result." - entry: "POST /api/v1/workflow/{id}/execute/" covered_by: [e2e-workflow] + - id: api-deployment-provision + description: "Deploying a workflow as an API mints a usable key and a resolvable endpoint." + covered_by: [integration-backend] + proof: marker + + - id: api-deployment-auth + description: "Unauthenticated or mis-scoped API-deployment calls are rejected before dispatch." + covered_by: [integration-backend] + proof: marker + - id: api-deployment-run description: "Deploy a workflow as an API, POST a document, receive structured JSON." - entry: "POST /deployment/api/{org}/{name}/" covered_by: [e2e-api-deployment] + - id: prompt-studio-author + description: "Create a Prompt Studio project and add a prompt to it." + covered_by: [integration-backend] + proof: marker + - id: prompt-studio-fetch-response description: "Prompt Studio: create project, add prompt, run single-pass, get response." - entry: "POST /api/v1/prompt-studio/prompt-studio-tool/{id}/fetch_response/" covered_by: [e2e-prompt-studio] + - id: connector-register-test + description: "Connector credentials are validated against the live system and stored encrypted." + covered_by: [integration-backend] + proof: marker + - id: pipeline-etl-execute description: "Run an ETL pipeline from source connector to destination." - entry: "POST /api/v1/pipeline/{id}/execute/" covered_by: [] # gap - - id: tool-sandbox-exec - description: "Tool image runs in sandbox container and emits structured output." - entry: "internal: tool-registry → runner → docker run" - covered_by: [unit-runner] + - id: usage-aggregate-read + description: "Per-run token usage aggregates correctly and stays scoped to its organization." + covered_by: [integration-backend] + proof: marker - id: usage-token-tracking description: "Per-execution token usage is recorded and retrievable." - entry: "GET /api/v1/usage/get_token_usage/" covered_by: [] # gap - id: workflow-execution-fan-out description: "Multi-file workflow execution fans out to file-processing workers and rejoins." - entry: "internal: backend → rabbitmq → workers/file_processing" covered_by: [] # gap - id: callback-result-delivery description: "Async results are posted back via the callback worker." - entry: "internal: workers/callback → backend /internal endpoints" covered_by: [] # gap diff --git a/tools/structure/__init__.py b/tests/e2e/auth/__init__.py similarity index 100% rename from tools/structure/__init__.py rename to tests/e2e/auth/__init__.py diff --git a/tests/e2e/auth/test_login.py b/tests/e2e/auth/test_login.py new file mode 100644 index 0000000000..d9f41a7eb9 --- /dev/null +++ b/tests/e2e/auth/test_login.py @@ -0,0 +1,49 @@ +"""E2E: OSS mock-login over real HTTP (covers the ``auth-login`` critical path). + +Self-contained on purpose — it builds its own session because it is the thing +under test. Other e2e tests should take the shared ``authed_session`` fixture. +""" + +from __future__ import annotations + +import pytest +import requests + +from tests.rig.runtime import PlatformEndpoints + +pytestmark = [pytest.mark.e2e, pytest.mark.critical] + + +@pytest.mark.critical_path("auth-login") +def test_login_sets_usable_session(platform: PlatformEndpoints) -> None: + base = platform.backend_url.rstrip("/") + session = requests.Session() + + resp = session.post( + f"{base}/api/v1/login", + data={"username": platform.admin_user, "password": platform.admin_password}, + allow_redirects=False, + timeout=10, + ) + assert resp.status_code == 302, ( + f"expected 302 redirect on login, got {resp.status_code} " + "(200 with HTML means bad credentials)" + ) + assert session.cookies.get("sessionid"), "login did not set a sessionid cookie" + + # The cookie is only meaningful if it authenticates a follow-up request. + who = session.get(f"{base}/api/v1/session", timeout=10) + assert who.status_code in (200, 201), f"/api/v1/session returned {who.status_code}" + assert who.json().get("email"), f"/api/v1/session returned no user: {who.text}" + + +def test_login_rejects_bad_credentials(platform: PlatformEndpoints) -> None: + base = platform.backend_url.rstrip("/") + resp = requests.Session().post( + f"{base}/api/v1/login", + data={"username": "unstract", "password": "wrong-password"}, + allow_redirects=False, + timeout=10, + ) + # Bad creds re-render the login page (200 HTML), never a 302 redirect. + assert resp.status_code != 302, "bad credentials should not yield a login redirect" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b184df9641..d4ba147507 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -11,6 +11,7 @@ import os import pytest +import requests from tests.rig.runtime import PlatformEndpoints @@ -24,3 +25,47 @@ def platform() -> PlatformEndpoints: "UNSTRACT_BACKEND_URL (etc.) yourself." ) return PlatformEndpoints.from_env() + + +@pytest.fixture(scope="session") +def authed_session(platform: PlatformEndpoints) -> requests.Session: + """A logged-in session with the active organization set. + + Standardizes the OSS mock-login flow so tests don't re-implement it: form + POST to /api/v1/login (302 + sessionid), then the org handshake + (GET /organization seeds the csrftoken cookie, POST /organization/{id}/set + with X-CSRFToken) so org-scoped endpoints are reachable. Session-scoped: + logged in once, reused across tests. Tests that exercise login itself + should build their own session instead. + """ + base = platform.backend_url.rstrip("/") + session = requests.Session() + + resp = session.post( + f"{base}/api/v1/login", + data={"username": platform.admin_user, "password": platform.admin_password}, + allow_redirects=False, + timeout=10, + ) + assert resp.status_code == 302, ( + f"login failed: expected 302, got {resp.status_code} " + "(200 with HTML means bad credentials)" + ) + + # GET /organization sets the csrftoken cookie and returns the org id, which + # the following POST needs both as a cookie and echoed in the CSRF header. + orgs = session.get(f"{base}/api/v1/organization", timeout=10) + orgs.raise_for_status() + org_list = orgs.json().get("organizations", []) + assert org_list, ( + f"no organizations returned by /api/v1/organization — is seed data " + f"loaded? response: {orgs.text}" + ) + org_id = org_list[0]["id"] + resp = session.post( + f"{base}/api/v1/organization/{org_id}/set", + headers={"X-CSRFToken": session.cookies.get("csrftoken", "")}, + timeout=10, + ) + resp.raise_for_status() + return session diff --git a/tests/e2e/smoke/test_smoke.py b/tests/e2e/smoke/test_smoke.py index 4e496196fa..8f60e3d6aa 100644 --- a/tests/e2e/smoke/test_smoke.py +++ b/tests/e2e/smoke/test_smoke.py @@ -1,9 +1,8 @@ -"""Single placeholder e2e smoke test. +"""E2E smoke gate: rig plumbing is wired and every service is up. -Exercises the rig's e2e plumbing end-to-end (platform up → URL injection → -pytest discovery → result aggregation → platform down) without depending on -auth state or seed data. Real workflow + API-deployment tests will land in -adjacent files (tests/e2e/workflows/, tests/e2e/api_deployment/, ...). +Depends on no auth state or seed data, so it isolates "is the platform +healthy" from feature tests. The other e2e groups depend on this one, so a +red smoke run stops them before they produce confusing failures. """ from __future__ import annotations @@ -13,6 +12,8 @@ import pytest import requests +from tests.rig.runtime import PlatformEndpoints, health_targets + pytestmark = [pytest.mark.e2e, pytest.mark.critical] @@ -30,6 +31,18 @@ def test_rig_session_sentinel_present() -> None: ) -def test_backend_health(platform) -> None: - response = requests.get(f"{platform.backend_url.rstrip('/')}/health/", timeout=10) - assert response.status_code < 500, f"backend /health returned {response.status_code}" +def test_all_services_healthy(platform: PlatformEndpoints) -> None: + """Every HTTP service in the stack answers its health endpoint with 200. + + This is the gate the rest of the e2e tier depends on — a half-booted stack + should fail here loudly rather than surface as confusing downstream errors. + """ + failures = [] + for name, url in health_targets(platform): + try: + resp = requests.get(url, timeout=10) + if resp.status_code != 200: + failures.append(f"{name} ({url}) -> HTTP {resp.status_code}") + except requests.RequestException as exc: + failures.append(f"{name} ({url}) -> {exc}") + assert not failures, "unhealthy services: " + "; ".join(failures) diff --git a/tests/groups.yaml b/tests/groups.yaml index 1e92d9f3d9..c441b6012d 100644 --- a/tests/groups.yaml +++ b/tests/groups.yaml @@ -32,32 +32,6 @@ groups: uv_sync_group: test coverage_source: src/unstract/sdk1 - unit-runner: - tier: unit - workdir: runner - # Runner co-locates its tests under src/. Pytest recurses from here. - # The project has no `test` uv group, so deps are pip-installed inline. - paths: [src] - pip_install: - - "flask~=3.1.0" - - "docker==6.1.3" - - "redis~=5.2.1" - - "python-dotenv>=1.0.0" - - "kubernetes" - install_editable: true - coverage_source: src/unstract/runner - - unit-prompt-service: - tier: unit - workdir: prompt-service - paths: [src/unstract/prompt_service/tests/unit] - uv_sync_group: test - # The parent conftest pulls in heavy adapter chains (pinecone etc.); unit - # tests must not depend on integration fixtures. - pytest_extra: ["--noconftest"] - markers: "not slow" - coverage_source: src/unstract/prompt_service - unit-platform-service: tier: unit workdir: platform-service @@ -100,38 +74,48 @@ groups: unit-backend: tier: unit workdir: backend - # List paths explicitly. `[.]` recurses into every test_*.py in backend/, - # including vendored fixtures and pluggable-app tests that don't belong - # in the OSS rig — keep this list scoped to the apps actually under test. - paths: - - account_v2/tests - - adapter_processor_v2/tests - - api_deployment_v2/tests - - connector_v2/tests - - dashboard_metrics/tests - - file_management/tests - - project/tests - - prompt_studio/prompt_studio_registry_v2/tests - - tenant_account_v2/tests - - usage_v2/tests - - utils/tests - - workflow_manager/endpoint_v2/tests + # Pure backend tests — no DB. Collect the whole tree and let markers, not a + # hand-kept file list, decide membership: tests needing live infra carry + # `@pytest.mark.integration` (see integration-backend) and are excluded here. + paths: ["."] + markers: "not integration" uv_sync_group: test - env: - DJANGO_SETTINGS_MODULE: backend.settings.test_cases - # Backend ORM imports require a real Postgres; rig provisions it via - # testcontainers or compose when this group is selected. - requires_services: [postgres, redis] + # Anchored: integration-backend reuses the identical Django settings env so + # the two halves of the backend suite can't drift apart. + env: &backend_test_env + DJANGO_SETTINGS_MODULE: backend.settings.test + # Tenancy is row-level, not schema-per-tenant; tests run in public. + DB_SCHEMA: public + # base.py resolves these at import time with no default; supply test-safe + # values here. + DJANGO_SECRET_KEY: test-secret-key-not-for-production + # All-zero Fernet key: valid format, obviously a placeholder. + ENCRYPTION_KEY: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + CELERY_BROKER_BASE_URL: redis://localhost:6379 + CELERY_BROKER_USER: guest + CELERY_BROKER_PASS: guest + INDEXING_FLAG_TTL: "3600" + ENABLE_LOG_HISTORY: "False" + STRUCTURE_TOOL_IMAGE_URL: docker:test + STRUCTURE_TOOL_IMAGE_NAME: test-structure-tool + STRUCTURE_TOOL_IMAGE_TAG: test + SYSTEM_ADMIN_USERNAME: admin + SYSTEM_ADMIN_PASSWORD: admin + SYSTEM_ADMIN_EMAIL: admin@example.com + # ExecutionFileHandler builds execution-dir paths from this at init; only + # constructed, never written to in these tests. + WORKFLOW_EXECUTION_DIR_PREFIX: /tmp/unstract-workflow-exec coverage_source: . - optional: true # gated until backend test_cases settings are complete unit-connectors: tier: unit workdir: unstract/connectors paths: [tests] + # Pure connector tests only. Credential / live-infra tests are marked + # `@pytest.mark.integration` and run in `integration-connectors`. + markers: "not integration" uv_sync_group: test coverage_source: src - optional: true unit-core: tier: unit @@ -140,17 +124,38 @@ groups: # No `test` uv group in unstract/core today; rig still injects pytest plugins. install_editable: true coverage_source: src - optional: true - unit-tool-registry: - tier: unit - workdir: unstract/tool-registry + # ── Integration tier: needs infra but not full platform ──────────────────── + integration-backend: + tier: integration + workdir: backend + # Backend ORM tests — need live infra. The rig provisions the declared + # requires_services via testcontainers and injects their connection env into + # pytest (tests/rig/cli.py:_inject_infra_env). Not optional: these gate the + # integration tier. + # Marker-only selection — exact complement of unit-backend: conftest.py + # auto-marks DB-bound tests `integration`, so new DB tests join this + # group with no edit here. Tests needing external creds skip without them. + paths: ["."] + markers: "integration" + uv_sync_group: test + env: *backend_test_env + requires_services: [postgres, redis, minio] + coverage_source: . + + integration-connectors: + tier: integration + workdir: unstract/connectors paths: [tests] + markers: "integration" + # Most connector integration tests need real third-party credentials + # (Snowflake, GDrive, Box, Dropbox, …) and skip when those are absent. The + # MinIO test actually runs: the rig provisions MinIO via testcontainers and + # injects MINIO_* creds (tests/rig/cli.py). + requires_services: [minio] uv_sync_group: test coverage_source: src - optional: true - # ── Integration tier: needs infra but not full platform ──────────────────── integration-workflow-execution: tier: integration paths: [tests/integration/workflow_execution] @@ -173,6 +178,15 @@ groups: timeout_seconds: 1200 depends_on: [unit-sdk1, unit-workers] + # Not optional: login is the first e2e critical path we gate on. A red login + # test fails the build and (on main) surfaces as an in-scope critical gap. + e2e-login: + tier: e2e + paths: [tests/e2e/auth] + requires_platform: true + critical: true + depends_on: [e2e-smoke] + e2e-workflow: tier: e2e paths: [tests/e2e/workflows] diff --git a/tests/rig/cli.py b/tests/rig/cli.py index 71b9d44de7..4ff333da29 100644 --- a/tests/rig/cli.py +++ b/tests/rig/cli.py @@ -21,6 +21,7 @@ import uuid from functools import lru_cache from pathlib import Path +from urllib.parse import urlsplit from xml.sax import saxutils from tests.rig import critical_paths as cp @@ -31,8 +32,18 @@ GroupDefinition, load_groups, ) -from tests.rig.reporting import GroupResult, parse_junit, write_summary -from tests.rig.runtime import PlatformEndpoints, PlatformRuntime, pick_runtime +from tests.rig.reporting import ( + GroupResult, + parse_junit, + passed_critical_path_ids, + write_summary, +) +from tests.rig.runtime import ( + PlatformEndpoints, + PlatformRuntime, + TestcontainersRuntime, + pick_runtime, +) from tests.rig.selection import resolve # Pytest exit codes that the rig treats as non-failure for aggregation: @@ -43,6 +54,10 @@ # cmd_run ("Empty collection (exit 5) is a failure…") and fails loudly. _NON_FAILING_PYTEST_EXIT_CODES = (0, 5) +# Injected into every group's pytest run (PYTHONPATH + -p) so +# @pytest.mark.critical_path marker args land in junit properties. +_PYTEST_PLUGIN_DIR = REPO_ROOT / "tests" / "rig" / "pytest_plugin" + @lru_cache(maxsize=1) def _rig_session_id() -> str: @@ -162,6 +177,21 @@ def _build_parser() -> argparse.ArgumentParser: pre = sub.add_parser("report", help="Re-aggregate existing reports/.") pre.add_argument("action", choices=["combine"]) pre.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports") + pre.add_argument( + "--baseline", + type=Path, + default=None, + help="Baseline path (default: /previous-summary.json).", + ) + pre.add_argument( + "--update-baseline", + action="store_true", + help=( + "Merge covered paths from all tiers' reports into the baseline in " + "one write (parallel per-tier writes would race). Refused (exit 1) " + "if any non-optional group is red or the baseline is corrupt." + ), + ) pre.set_defaults(func=cmd_report) return p @@ -292,15 +322,21 @@ def cmd_report(args: argparse.Namespace) -> int: result = parse_junit(name, tier, reports_dir) if result is not None: group_results.append(result) - green = _green_group_names(group_results) + green = _coverage_attesting_groups(group_results) + proven, unknown_marker_ids = _marker_proven_paths( + registry, group_results, reports_dir + ) + baseline_path = args.baseline or reports_dir / "previous-summary.json" baseline_corrupt = False try: - baseline = cp.load_baseline(reports_dir / "previous-summary.json") + baseline = cp.load_baseline(baseline_path) except cp.BaselineCorruptError as exc: print(f"[rig] {exc}", file=sys.stderr) baseline = None baseline_corrupt = True - statuses = cp.evaluate(registry, groups_run_green=green, baseline=baseline) + statuses = cp.evaluate( + registry, groups_run_green=green, baseline=baseline, marker_proven=proven + ) write_summary( reports_dir=reports_dir, group_results=group_results, @@ -308,7 +344,30 @@ def cmd_report(args: argparse.Namespace) -> int: baseline_corrupt=baseline_corrupt, ) print(f"Wrote {reports_dir / 'summary.md'}") - return 0 + if unknown_marker_ids: + print( + f"[rig] ❌ @pytest.mark.critical_path references unknown path " + f"id(s): {', '.join(unknown_marker_ids)} " + f"(not in tests/critical_paths.yaml)", + file=sys.stderr, + ) + if args.update_baseline: + red = [ + r.name + for r in group_results + if r.status not in ("pass", "empty") and not manifest.get(r.name).optional + ] + if baseline_corrupt or red: + reason = ( + "corrupt baseline" + if baseline_corrupt + else f"red groups: {', '.join(red)}" + ) + print(f"[rig] ❌ baseline update refused ({reason})", file=sys.stderr) + return 1 + cp.merge_into_baseline(statuses, baseline_path) + print(f"[rig] merged into baseline: {baseline_path}") + return 1 if unknown_marker_ids else 0 def cmd_run(args: argparse.Namespace) -> int: @@ -352,6 +411,11 @@ def cmd_run(args: argparse.Namespace) -> int: reports_dir.mkdir(parents=True, exist_ok=True) needs_platform = any(manifest.get(n).requires_platform for n in runnable) + # A group can declare `requires_services` (stateful infra like Postgres/ + # Redis) without needing the whole platform — provision just that infra via + # testcontainers instead of standing up every compose service. Platform wins + # when both are set. + needs_services = any(manifest.get(n).requires_services for n in runnable) runtime: PlatformRuntime | None = None endpoints: PlatformEndpoints | None = None group_results: list[GroupResult] = [] @@ -364,6 +428,16 @@ def cmd_run(args: argparse.Namespace) -> int: # `up()` is inside the try so a failure here still triggers `down()` # in the finally, cleaning up any partial stack. endpoints = runtime.up() + elif needs_services and not args.dry_run: + # Infra-only: testcontainers Postgres/Redis/etc., no platform + # services. ponytail: up() starts the full infra set even if a run + # only needs Postgres; trim to the requested services if startup + # cost ever matters. + runtime = TestcontainersRuntime() + print( + f"[rig] bringing infra up via runtime={runtime.name} (requires_services)" + ) + endpoints = runtime.up() # TODO(runtime-gate-skip): groups run unconditionally in topo order; # there is no skip-if-a-dependency-failed logic yet. The dep edge to @@ -448,7 +522,19 @@ def cmd_run(args: argparse.Namespace) -> int: if args.coverage and not args.dry_run: combine_and_report(reports_dir) - green = _green_group_names(group_results) + green = _coverage_attesting_groups(group_results) + proven, unknown_marker_ids = _marker_proven_paths( + registry, group_results, reports_dir + ) + if unknown_marker_ids: + print( + f"[rig] ❌ @pytest.mark.critical_path references unknown path " + f"id(s): {', '.join(unknown_marker_ids)} " + f"(not in tests/critical_paths.yaml)", + file=sys.stderr, + ) + if overall_exit == 0: + overall_exit = 1 # A corrupt baseline can't be silently treated as empty (that would turn # the next build into a regression festival once a one-tier baseline gets # written back). But we must still write the per-group summary so the @@ -467,6 +553,7 @@ def cmd_run(args: argparse.Namespace) -> int: groups_run_green=green, baseline=baseline, scope_groups=scope_groups, + marker_proven=proven, ) write_summary( reports_dir=reports_dir, @@ -497,11 +584,25 @@ def cmd_run(args: argparse.Namespace) -> int: if overall_exit == 0: overall_exit = 1 + # Only in-scope gaps gate: a declared covering group ran in this tier but + # not green. Out-of-scope gaps (covered only by other tiers, or undeclared) + # are reported but must not fail a tier for coverage it can't produce. gaps = [s for s in statuses if s.state == "gap"] - if gaps and args.fail_on_critical_gap: + in_scope_gaps = [s for s in gaps if s.in_scope] + out_of_scope_gaps = [s for s in gaps if not s.in_scope] + if out_of_scope_gaps: + ids = ", ".join(s.path.id for s in out_of_scope_gaps) print( - f"\n[rig] ⚠️ {len(gaps)} critical-path gap(s) detected " - f"(fail-on-critical-gap)", + f"[rig] ℹ️ {len(out_of_scope_gaps)} critical-path gap(s) out of scope " + f"for this run (warn-only, not covered by any group in this tier): " + f"{ids}", + file=sys.stderr, + ) + if in_scope_gaps and args.fail_on_critical_gap: + ids = ", ".join(s.path.id for s in in_scope_gaps) + print( + f"\n[rig] ⚠️ {len(in_scope_gaps)} critical-path gap(s) detected " + f"(fail-on-critical-gap): {ids}", file=sys.stderr, ) if overall_exit == 0: @@ -521,8 +622,78 @@ def cmd_run(args: argparse.Namespace) -> int: # ── execution helpers ───────────────────────────────────────────────────────── -def _green_group_names(results: list[GroupResult]) -> set[str]: - return {r.name for r in results if r.status in ("pass", "empty")} +def _db_env_from_postgres_url(url: str) -> dict[str, str]: + """Translate a provisioned Postgres URL into the discrete ``DB_*`` vars + Django reads (``backend/settings/base.py``). + + The rig provisions a throwaway Postgres via testcontainers for groups + declaring ``requires_services: [postgres]``. Without this translation the + backend falls back to the compose hostname ``backend-db-1``, unreachable + from the host-side pytest, and every ``django_db`` test errors on connect. + """ + # e.g. postgresql+psycopg2://user:pass@host:49153/dbname + parts = urlsplit(url) + return { + "DB_HOST": parts.hostname or "localhost", + "DB_PORT": str(parts.port or 5432), + "DB_USER": parts.username or "test", + "DB_PASSWORD": parts.password or "test", + "DB_NAME": parts.path.lstrip("/") or "test", + } + + +def _inject_infra_env( + env: dict[str, str], + group: GroupDefinition, + endpoints: PlatformEndpoints | None, +) -> None: + # Postgres/Redis override so a stale shell value can't shadow the + # provisioned testcontainer; MinIO uses setdefault so a developer's own + # endpoint wins. + if endpoints is None: + return + infra = endpoints.infra + if "postgres" in group.requires_services and infra.postgres_url: + env.update(_db_env_from_postgres_url(infra.postgres_url)) + if "minio" in group.requires_services and infra.minio_endpoint: + # http: this is a local, throwaway testcontainers MinIO with no TLS. + env.setdefault( + "MINIO_ENDPOINT_URL", + f"http://{infra.minio_endpoint}", # NOSONAR + ) + if infra.minio_access_key: + env.setdefault("MINIO_ACCESS_KEY_ID", infra.minio_access_key) + if infra.minio_secret_key: + env.setdefault("MINIO_SECRET_ACCESS_KEY", infra.minio_secret_key) + if "redis" in group.requires_services and infra.redis_host: + redis_port = str(infra.redis_port or 6379) + env["REDIS_HOST"] = infra.redis_host + env["REDIS_PORT"] = redis_port + env["CELERY_BROKER_BASE_URL"] = f"redis://{infra.redis_host}:{redis_port}" + + +def _coverage_attesting_groups(results: list[GroupResult]) -> set[str]: + # "empty" (exit 5) stays non-failing for the build, but a covering group + # that collected zero tests must not attest critical-path coverage — a + # broken marker expression would otherwise report ✅ with zero tests run. + return {r.name for r in results if r.status == "pass"} + + +def _marker_proven_paths( + registry: cp.CriticalPathRegistry, + group_results: list[GroupResult], + reports_dir: Path, +) -> tuple[set[str], list[str]]: + """Union critical-path ids attested by passing marked tests across groups. + + Returns ``(known_ids, unknown_ids)`` — unknown ids (marker typos) must fail + the build or the marked test silently attests nothing. + """ + registry_ids = {p.id for p in registry.paths} + proven: set[str] = set() + for r in group_results: + proven |= passed_critical_path_ids(r.name, reports_dir) + return proven & registry_ids, sorted(proven - registry_ids) def _is_missing_placeholder(group: GroupDefinition) -> bool: @@ -552,6 +723,9 @@ def _execute_group( md_report = group_reports / "report.md" env = _subprocess_env() + env["PYTHONPATH"] = os.pathsep.join( + p for p in (str(_PYTEST_PLUGIN_DIR), env.get("PYTHONPATH")) if p + ) env.update(group.env) if endpoints is not None: env.setdefault("UNSTRACT_BACKEND_URL", endpoints.backend_url) @@ -564,6 +738,7 @@ def _execute_group( # leaked in". `setdefault` would let a leaked sentinel win, which # defeats the purpose — set unconditionally. env["UNSTRACT_RIG_SESSION_ID"] = _rig_session_id() + _inject_infra_env(env, group, endpoints) if coverage and group.coverage_source: env.update(coverage_env(group.name, reports_dir)) @@ -615,6 +790,9 @@ def _execute_group( return parse_junit(group.name, group.tier, reports_dir), exit_code +# Injected into every group's ephemeral `uv run` env via --with, so they +# survive the per-call venv re-sync (unlike `uv pip install`). Includes pytest +# and requests themselves — needed regardless of the workdir project's deps. RIG_PYTEST_PLUGINS = ( "pytest>=8.0.1", "pytest-md-report>=0.6.2", @@ -622,6 +800,7 @@ def _execute_group( "pytest-xdist>=3.5.0", "pytest-cov>=4.1.0", "pytest-mock>=3.12.0", + "requests>=2.31.0", ) @@ -639,13 +818,8 @@ def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None: env=env, check=False, ) - if group.install_editable: - subprocess.run( - ["uv", "pip", "install", "-e", "."], - cwd=workdir, - env=env, - check=False, - ) + # install_editable is handled in _pytest_command via `--with-editable`; + # installing it here would be wiped by `uv run`'s venv re-sync. if group.pip_install: subprocess.run( ["uv", "pip", "install", *group.pip_install], @@ -658,6 +832,20 @@ def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None: # That avoids losing them on the next `uv run` (which re-syncs the venv). +def _pytest_base_cmd(group: GroupDefinition, workdir: Path) -> list[str]: + if not shutil.which("uv"): + return [sys.executable, "-m", "pytest"] + # `uv run` re-syncs the venv each call, wiping anything from `uv pip + # install`. `--with`/`--with-editable` inject plugins + the project into the + # ephemeral run env instead, surviving the sync. + with_args: list[str] = [] + for spec in RIG_PYTEST_PLUGINS: + with_args += ["--with", spec] + if group.install_editable: + with_args += ["--with-editable", str(workdir)] + return ["uv", "run", *with_args, "pytest"] + + def _pytest_command( group: GroupDefinition, *, @@ -671,23 +859,21 @@ def _pytest_command( workers: str, timeout: int, ) -> list[str]: - use_uv = shutil.which("uv") is not None - if use_uv: - # `uv run` re-syncs the project's venv each call, which would wipe any - # plugins added via `uv pip install`. `--with` injects them into the - # ephemeral run environment, surviving the sync. - with_args: list[str] = [] - for spec in RIG_PYTEST_PLUGINS: - with_args += ["--with", spec] - base: list[str] = ["uv", "run", *with_args, "pytest"] - else: - base = [sys.executable, "-m", "pytest"] + base = _pytest_base_cmd(group, workdir) cmd = [ *base, "-v", f"--junitxml={junit}", f"--timeout={timeout}", + # Marker→junit-property plugin (dir added to PYTHONPATH in + # _execute_group). junit_family=legacy: the default xunit2 schema + # rejects per-testcase , which the coverage attestation + # reads. + "-p", + "rig_critical_path", + "-o", + "junit_family=legacy", ] # pytest-md-report does not aggregate worker output reliably under xdist. # Emit markdown only on serial runs; junit + reporting.py's _render_markdown diff --git a/tests/rig/critical_paths.py b/tests/rig/critical_paths.py index b0cbbd0596..e9cb3ccbbb 100644 --- a/tests/rig/critical_paths.py +++ b/tests/rig/critical_paths.py @@ -34,8 +34,12 @@ class BaselineCorruptError(RuntimeError): class CriticalPath: id: str description: str - entry: str covered_by: tuple[str, ...] + # "group": covered when any covered_by group runs green (coarse — survives + # the covering test being deleted). "marker": additionally requires ≥1 + # passing test marked @pytest.mark.critical_path("") in this run. + # Ratchet each path to "marker" as its tests get marked. + proof: Literal["group", "marker"] = "group" @dataclass(frozen=True) @@ -65,6 +69,12 @@ class CriticalPathStatus: state: CriticalPathState covering_groups_run: tuple[str, ...] notes: str = "" + # True when a declared covering group is in this run's scope (or scoping is + # off). An out-of-scope gap (coverage only in an unrun tier, or none + # declared) must not gate under --fail-on-critical-gap. Defaults False so a + # regression that forgets to pass it can only under-gate (spurious warning), + # never over-gate (spurious build block). + in_scope: bool = False def __post_init__(self) -> None: # Make the contradictory states unrepresentable rather than relying on @@ -80,17 +90,23 @@ def load_critical_paths(path: Path | None = None) -> CriticalPathRegistry: raw = yaml.safe_load((path or DEFAULT_REGISTRY).read_text()) if not isinstance(raw, dict) or "paths" not in raw: raise ValueError(f"{path or DEFAULT_REGISTRY}: expected top-level `paths:` list") - return CriticalPathRegistry( - paths=tuple( + paths = [] + for p in raw["paths"]: + proof = str(p.get("proof", "group")) + if proof not in ("group", "marker"): + raise ValueError( + f"critical path {p.get('id')!r}: proof must be 'group' or " + f"'marker', got {proof!r}" + ) + paths.append( CriticalPath( id=str(p["id"]), description=str(p.get("description", "")), - entry=str(p.get("entry", "")), covered_by=tuple(p.get("covered_by") or ()), + proof=proof, ) - for p in raw["paths"] ) - ) + return CriticalPathRegistry(paths=tuple(paths)) def validate_registry_against_manifest( @@ -106,6 +122,11 @@ def validate_registry_against_manifest( f"critical path {path.id!r}: " f"covered_by references unknown group {g!r}" ) + if path.proof == "marker" and not path.covered_by: + errors.append( + f"critical path {path.id!r}: proof 'marker' requires a " + f"non-empty covered_by (marked tests must live in a group)" + ) return errors @@ -115,21 +136,27 @@ def evaluate( groups_run_green: Collection[str], baseline: dict[str, Any] | None, scope_groups: Collection[str] | None = None, + marker_proven: Collection[str] = (), ) -> list[CriticalPathStatus]: """Compute the status for each critical path against this build's results. Args: registry: parsed critical-paths registry. groups_run_green: names of groups that ran AND passed in this build. + Pass only ``status == "pass"`` groups — a group that collected + zero tests ("empty") must not attest coverage. baseline: parsed previous-summary.json from the main-branch cache, or None. Expected shape: ``{"covered_paths": ["auth-login", ...]}``. - scope_groups: collection of every group the caller considered running - this invocation (including dep-expanded deps and skipped - optional placeholders). When a critical path's ``covered_by`` - is fully outside ``scope_groups``, the path is classified as - ``gap`` rather than ``regression`` — running only the unit - tier shouldn't flag e2e-tier paths as regressed. If ``None``, - no scoping is applied (back-compat). + scope_groups: the groups this invocation actually runs (dep-expanded). + When a critical path's ``covered_by`` is fully outside + ``scope_groups``, the path is classified as ``gap`` rather than + ``regression`` — running only the unit tier shouldn't flag + e2e-tier paths as regressed. If ``None``, no scoping is applied + (back-compat). + marker_proven: path ids with at least one passing + ``@pytest.mark.critical_path`` test in this build. Paths with + ``proof: marker`` require membership here on top of a green + covering group. Returns: Statuses in the original registry order. @@ -139,34 +166,52 @@ def evaluate( # when callers pass lists/tuples; the public signature accepts Collection # to leave that choice to them. green = set(groups_run_green) + proven = set(marker_proven) scope = None if scope_groups is None else set(scope_groups) - statuses: list[CriticalPathStatus] = [] - for path in registry.paths: - covering = tuple(g for g in path.covered_by if g in green) - in_scope = scope is None or any(g in scope for g in path.covered_by) - state: CriticalPathState - if covering: - state = "covered" - note = "" - elif path.id in previously_covered and in_scope: - state = "regression" - note = "Was covered on the cached baseline; not covered in this build." - else: - state = "gap" - note = ( - "Out of scope for this invocation." - if not in_scope - else "No group covering this path ran green in this build." - ) - statuses.append( - CriticalPathStatus( - path=path, - state=state, - covering_groups_run=covering, - notes=note, - ) + return [ + _status_for(path, green, proven, previously_covered, scope) + for path in registry.paths + ] + + +def _status_for( + path: CriticalPath, + green: set[str], + proven: set[str], + previously_covered: set[str], + scope: set[str] | None, +) -> CriticalPathStatus: + covering = tuple(g for g in path.covered_by if g in green) + if path.proof == "marker" and path.id not in proven: + covering = () + in_scope = scope is None or any(g in scope for g in path.covered_by) + if covering: + state: CriticalPathState = "covered" + note = "" + elif path.id in previously_covered and in_scope: + state = "regression" + note = "Was covered on the cached baseline; not covered in this build." + else: + state = "gap" + note = _gap_note(path, green, in_scope) + return CriticalPathStatus( + path=path, + state=state, + covering_groups_run=covering, + notes=note, + in_scope=in_scope, + ) + + +def _gap_note(path: CriticalPath, green: set[str], in_scope: bool) -> str: + if not in_scope: + return "Out of scope for this invocation." + if path.proof == "marker" and any(g in green for g in path.covered_by): + return ( + "Covering group ran green but no passing " + "@pytest.mark.critical_path test attested this path." ) - return statuses + return "No group covering this path ran green in this build." def merge_into_baseline(statuses: list[CriticalPathStatus], destination: Path) -> None: diff --git a/tests/rig/pytest_plugin/rig_critical_path.py b/tests/rig/pytest_plugin/rig_critical_path.py new file mode 100644 index 0000000000..2704465b56 --- /dev/null +++ b/tests/rig/pytest_plugin/rig_critical_path.py @@ -0,0 +1,29 @@ +"""Pytest plugin injected by the rig into every group's pytest run. + +Copies ``@pytest.mark.critical_path("")`` marker args into junit +testcase ```` so the rig can attest critical-path coverage from +tests that actually passed, not just from a group's overall exit code. A test +may carry the marker multiple times (or pass several ids) to attest several +paths. + +Lives in its own directory (not ``tests/rig/``) because the rig injects it via +``PYTHONPATH`` + ``-p`` into each group's own venv — putting ``tests/rig`` on +``PYTHONPATH`` would shadow real packages (e.g. ``coverage``). Stdlib-only for +the same reason. +""" + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "critical_path(path_id, ...): attests coverage of declared critical " + "path(s) from tests/critical_paths.yaml when this test passes", + ) + + +# pytest matches hook args by name, so unused params can be omitted. +def pytest_collection_modifyitems(items): + for item in items: + for marker in item.iter_markers("critical_path"): + for path_id in marker.args: + item.user_properties.append(("critical_path", str(path_id))) diff --git a/tests/rig/reporting.py b/tests/rig/reporting.py index 8bff0a00cb..21ef3f6fa9 100644 --- a/tests/rig/reporting.py +++ b/tests/rig/reporting.py @@ -139,6 +139,32 @@ def parse_junit(group_name: str, tier: str, reports_dir: Path) -> GroupResult | ) +def passed_critical_path_ids(group_name: str, reports_dir: Path) -> set[str]: + """Extract critical-path ids attested by *passing* tests in a group's junit. + + The rig's injected pytest plugin copies ``@pytest.mark.critical_path`` args + into per-testcase ````. Only testcases with no failure/error/ + skipped child count — a skipped or failing marked test proves nothing. + Missing or malformed junit yields an empty set (the group-level result + already surfaces that as an error). + """ + junit_path = reports_dir / group_name / "junit.xml" + if not junit_path.exists(): + return set() + try: + root = ET.parse(junit_path).getroot() + except ET.ParseError: + return set() + ids: set[str] = set() + for case in root.iter("testcase"): + if any(case.find(tag) is not None for tag in ("failure", "error", "skipped")): + continue + for prop in case.iter("property"): + if prop.get("name") == "critical_path" and prop.get("value"): + ids.add(prop.get("value")) + return ids + + def _read_exit_code(exit_path: Path) -> int: if not exit_path.exists(): return -1 @@ -244,9 +270,7 @@ def _render_markdown( lines.append("### ❌ Regressions (must be zero)") lines.append("") for s in regressions: - lines.append( - f"- **{s.path.id}** — {s.path.description} (entry: `{s.path.entry}`)" - ) + lines.append(f"- **{s.path.id}** — {s.path.description}") lines.append("") if gaps: lines.append("### ⚠️ Critical paths not yet covered") @@ -255,7 +279,7 @@ def _render_markdown( covers = ", ".join(s.path.covered_by) or "_no groups declared_" lines.append( f"- **{s.path.id}** — {s.path.description} " - f"(entry: `{s.path.entry}`; declared coverage: {covers})" + f"(declared coverage: {covers})" ) lines.append("") if covered: diff --git a/tests/rig/runtime.py b/tests/rig/runtime.py index 97be22d748..2643d9da99 100644 --- a/tests/rig/runtime.py +++ b/tests/rig/runtime.py @@ -51,6 +51,8 @@ class InfraEndpoints: rabbitmq_host: str | None = None rabbitmq_port: int | None = None minio_endpoint: str | None = None + minio_access_key: str | None = None + minio_secret_key: str | None = None def __post_init__(self) -> None: for host, port, label in ( @@ -205,6 +207,10 @@ def up(self) -> PlatformEndpoints: minio_endpoint=( f"{minio.get_container_host_ip()}:{minio.get_exposed_port(9000)}" ), + # Default testcontainers MinIO root creds; surfaced so the + # rig can inject them into connector integration tests. + minio_access_key=getattr(minio, "access_key", "minioadmin"), + minio_secret_key=getattr(minio, "secret_key", "minioadmin"), ), ) except Exception: @@ -256,14 +262,27 @@ def _run(cmd: list[str], *, check: bool = True) -> None: ) from exc +def health_targets(endpoints: PlatformEndpoints) -> list[tuple[str, str]]: + """(service name, health URL) for every HTTP service the e2e tier depends on. + + Single source of truth for the readiness probe and the e2e smoke test. + Paths are service-specific: x2text mounts health under a blueprint prefix, + and there is no standalone prompt-service (folded into workers). The runner + is intentionally absent — container-based execution is being retired in + favour of in-worker execution, so e2e must not depend on it being up. + """ + return [ + ("backend", endpoints.backend_url.rstrip("/") + "/health"), + ("platform-service", endpoints.platform_service_url.rstrip("/") + "/health"), + ("x2text-service", endpoints.x2text_url.rstrip("/") + "/api/v1/x2text/health"), + ] + + def _wait_ready(endpoints: PlatformEndpoints, *, timeout_seconds: int = 300) -> None: - """Poll each service's /health endpoint until all respond or timeout. + """Poll each service's health endpoint until all respond or timeout. - If ``requests`` isn't importable (e.g. running the rig on a bare interpreter - just to list groups), readiness probing is skipped. That's safe because the - only caller, ``ComposeRuntime.up``, is on the e2e path where requests is in - the rig's deps; getting here without requests installed implies a broken - install and the developer will surface it shortly. + Skips probing if ``requests`` isn't importable — the rig may run on a bare + interpreter just to list groups. The e2e path always has requests installed. """ try: import requests @@ -271,13 +290,7 @@ def _wait_ready(endpoints: PlatformEndpoints, *, timeout_seconds: int = 300) -> log.warning("`requests` not installed; skipping platform readiness probe") return - targets = [ - endpoints.backend_url.rstrip("/") + "/health/", - endpoints.prompt_service_url.rstrip("/") + "/health", - endpoints.platform_service_url.rstrip("/") + "/health", - endpoints.runner_url.rstrip("/") + "/health", - endpoints.x2text_url.rstrip("/") + "/health", - ] + targets = [url for _, url in health_targets(endpoints)] deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: if all(_responds(t, requests) for t in targets): diff --git a/tests/rig/tests/test_cli.py b/tests/rig/tests/test_cli.py index a975f43d62..4669488fcf 100644 --- a/tests/rig/tests/test_cli.py +++ b/tests/rig/tests/test_cli.py @@ -216,6 +216,108 @@ def fake_execute_group(group, **kwargs): ) +def _run_gap_scenario( + tmp_path: Path, monkeypatch, *, covered_by: str, fail_on_gap: bool +) -> int: + """Drive cmd_run with a single optional group ``unit-cov`` that runs RED and + one critical path covered by ``covered_by`` (a YAML list literal like + ``[unit-cov]`` or ``[]``). The group is optional so its own red exit never + gates — isolating the critical-gap logic. Returns the overall exit code. + """ + from tests.rig.reporting import GroupResult + + test_dir = Path(__file__).parent + manifest_yaml = ( + "version: 1\n" + "groups:\n" + " unit-cov:\n" + " tier: unit\n" + f" workdir: {test_dir}\n" + " paths: [.]\n" + " optional: true\n" + ) + cp_yaml = ( + "version: 1\n" + "paths:\n" + " - id: p1\n" + " description: ''\n" + f" covered_by: {covered_by}\n" + ) + (tmp_path / "groups.yaml").write_text(manifest_yaml) + (tmp_path / "critical_paths.yaml").write_text(cp_yaml) + + import tests.rig.cli as cli_mod + import tests.rig.critical_paths as cp_mod + import tests.rig.groups as groups_mod + + monkeypatch.setattr(groups_mod, "DEFAULT_MANIFEST", tmp_path / "groups.yaml") + monkeypatch.setattr(cp_mod, "DEFAULT_REGISTRY", tmp_path / "critical_paths.yaml") + + def fake_execute_group(group, **kwargs): + # The covering group runs red, so it never counts as green coverage. + result = GroupResult( + name=group.name, + tier=group.tier, + exit_code=1, + passed=0, + failed=1, + errors=0, + skipped=0, + duration_seconds=0.01, + ) + return result, 1 + + monkeypatch.setattr(cli_mod, "_execute_group", fake_execute_group) + + argv = [ + "run", + "unit-cov", + "--no-coverage", + "--no-parallel", + "--reports-dir", + str(tmp_path / "reports"), + "--baseline", + str(tmp_path / "reports" / "previous-summary.json"), + ] + if fail_on_gap: + argv.append("--fail-on-critical-gap") + args = cli_mod._build_parser().parse_args(argv) + return cli_mod.cmd_run(args) + + +def test_fail_on_critical_gap_gates_on_in_scope_gap(tmp_path: Path, monkeypatch) -> None: + """A critical path covered by an in-tier group that ran red is an IN-SCOPE + gap: --fail-on-critical-gap must fail the build on it (real coverage is + gone). Without the flag, it's reported but doesn't gate. + """ + assert ( + _run_gap_scenario( + tmp_path, monkeypatch, covered_by="[unit-cov]", fail_on_gap=True + ) + == 1 + ) + assert ( + _run_gap_scenario( + tmp_path, monkeypatch, covered_by="[unit-cov]", fail_on_gap=False + ) + == 0 + ) + + +def test_fail_on_critical_gap_ignores_out_of_scope_gap( + tmp_path: Path, monkeypatch +) -> None: + """A path with no declared coverage (or coverage only in another tier) is an + OUT-OF-SCOPE gap: --fail-on-critical-gap must NOT fail this tier on it. + This is the fix for the perma-red `main`: e2e-only and not-yet-covered paths + can't fail the unit/integration tiers. + """ + assert ( + _run_gap_scenario(tmp_path, monkeypatch, covered_by="[]", fail_on_gap=True) + == 0 + ) + + def test_cmd_run_teardown_failure_does_not_mask_up_failure( tmp_path: Path, monkeypatch ) -> None: @@ -465,3 +567,46 @@ def test_cmd_report_re_aggregates_existing_junit(tmp_path: Path, monkeypatch) -> for artifact in ("summary.md", "summary.json", "combined-test-report.md"): assert (reports_dir / artifact).exists(), f"missing {artifact}" assert "unit-x" in (reports_dir / "summary.md").read_text() + + +def test_db_env_from_postgres_url_maps_discrete_vars() -> None: + """The provisioned-Postgres URL (testcontainers, with a `+driver` scheme + and a random host port) must translate into the discrete DB_* vars Django + reads — otherwise integration-backend falls back to `backend-db-1` and + every django_db test errors on connect. + """ + import tests.rig.cli as cli_mod + + env = cli_mod._db_env_from_postgres_url( + "postgresql+psycopg2://tcuser:tcpass@127.0.0.1:49231/testdb" + ) + assert env == { + "DB_HOST": "127.0.0.1", + "DB_PORT": "49231", + "DB_USER": "tcuser", + "DB_PASSWORD": "tcpass", # NOSONAR - test placeholder, not a real credential + "DB_NAME": "testdb", + } + + +def test_inject_infra_env_wires_provisioned_redis() -> None: + """A group declaring `requires_services: [redis]` must get REDIS_HOST/PORT + + the Celery broker URL rewritten to the provisioned endpoint — otherwise + Redis-backed tests silently hit the localhost default and bypass the + testcontainer. + """ + import tests.rig.cli as cli_mod + from tests.rig.groups import GroupDefinition + from tests.rig.runtime import InfraEndpoints, PlatformEndpoints + + endpoints = PlatformEndpoints.from_env( + infra=InfraEndpoints(redis_host="redis.internal", redis_port=49999) + ) + group = GroupDefinition( + name="g", tier="integration", paths=("tests",), requires_services=("redis",) + ) + env: dict[str, str] = {} + cli_mod._inject_infra_env(env, group, endpoints) + assert env["REDIS_HOST"] == "redis.internal" + assert env["REDIS_PORT"] == "49999" + assert env["CELERY_BROKER_BASE_URL"] == "redis://redis.internal:49999" diff --git a/tests/rig/tests/test_critical_paths.py b/tests/rig/tests/test_critical_paths.py index b1798d3c1b..214c958d62 100644 --- a/tests/rig/tests/test_critical_paths.py +++ b/tests/rig/tests/test_critical_paths.py @@ -19,7 +19,7 @@ def _registry(*ids_and_covers: tuple[str, tuple[str, ...]]) -> CriticalPathRegistry: return CriticalPathRegistry( paths=tuple( - CriticalPath(id=i, description="", entry="", covered_by=c) + CriticalPath(id=i, description="", covered_by=c) for i, c in ids_and_covers ) ) @@ -97,7 +97,7 @@ def test_critical_path_status_rejects_contradictions() -> None: """Make the contradictory states unrepresentable.""" from tests.rig.critical_paths import CriticalPath, CriticalPathStatus - path = CriticalPath(id="p", description="", entry="", covered_by=("g",)) + path = CriticalPath(id="p", description="", covered_by=("g",)) with pytest.raises(ValueError, match="covered.*non-empty"): CriticalPathStatus(path=path, state="covered", covering_groups_run=()) with pytest.raises(ValueError, match="empty covering_groups_run"): @@ -158,6 +158,43 @@ def test_scope_demotes_out_of_scope_regressions_to_gaps() -> None: assert by_id["straddle-path"].state == "regression" # partially in scope +def test_in_scope_flag_distinguishes_gap_flavours() -> None: + """The ``in_scope`` flag on a status is what lets --fail-on-critical-gap + gate only on coverage that this tier was actually responsible for. An + out-of-scope gap (e2e path during the unit tier, or a path with no declared + coverage) must report ``in_scope=False``; an in-scope gap (a declared + in-tier group that didn't run green) must report ``in_scope=True``. + """ + registry = _registry( + ("in-scope", ("unit-g",)), # declared group is in scope, but not green + ("e2e-only", ("e2e-g",)), # declared group is out of scope this run + ("undeclared", ()), # no declared coverage anywhere + ) + statuses = evaluate( + registry, + groups_run_green=set(), # nothing passed → all three are gaps + baseline=None, + scope_groups={"unit-g"}, + ) + by_id = {s.path.id: s for s in statuses} + assert all(s.state == "gap" for s in statuses) + assert by_id["in-scope"].in_scope is True + assert by_id["e2e-only"].in_scope is False + assert by_id["undeclared"].in_scope is False + + +def test_covered_path_is_in_scope() -> None: + registry = _registry(("p1", ("g1",))) + statuses = evaluate( + registry, + groups_run_green={"g1"}, + baseline=None, + scope_groups={"g1"}, + ) + assert statuses[0].state == "covered" + assert statuses[0].in_scope is True + + def test_scope_none_preserves_legacy_behavior() -> None: """scope_groups=None disables scope-filtering so callers that don't pass it keep the old "everything in baseline counts" semantics. @@ -170,3 +207,98 @@ def test_scope_none_preserves_legacy_behavior() -> None: scope_groups=None, ) assert statuses[0].state == "regression" + + +def _marker_path(path_id: str, covers: tuple[str, ...]) -> CriticalPath: + return CriticalPath( + id=path_id, description="", covered_by=covers, proof="marker" + ) + + +def test_marker_proof_covered_when_group_green_and_proven() -> None: + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven={"p1"} + ) + assert statuses[0].state == "covered" + assert statuses[0].covering_groups_run == ("g1",) + + +def test_marker_proof_gap_when_group_green_but_unproven() -> None: + """The whole point of proof=marker: a green covering group alone (e.g. the + marked test was deleted and the rest of the group still passes) must not + attest coverage. + """ + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven=set() + ) + assert statuses[0].state == "gap" + assert "critical_path" in statuses[0].notes + + +def test_marker_proof_regression_when_baseline_covered_but_unproven() -> None: + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, + groups_run_green={"g1"}, + baseline={"covered_paths": ["p1"]}, + marker_proven=set(), + ) + assert statuses[0].state == "regression" + + +def test_marker_proof_requires_green_group_too() -> None: + """A passing marked test inside an overall-red group proves nothing — + coverage needs both the proof and a green covering group. + """ + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green=set(), baseline=None, marker_proven={"p1"} + ) + assert statuses[0].state == "gap" + + +def test_marker_proof_does_not_affect_group_proof_paths() -> None: + registry = _registry(("p1", ("g1",))) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven=set() + ) + assert statuses[0].state == "covered" + + +def test_load_rejects_unknown_proof(tmp_path: Path) -> None: + from tests.rig.critical_paths import load_critical_paths + + reg = tmp_path / "critical_paths.yaml" + reg.write_text( + "paths:\n - id: p1\n covered_by: [g1]\n proof: junit\n" + ) + with pytest.raises(ValueError, match="proof"): + load_critical_paths(reg) + + +def test_load_parses_proof_field(tmp_path: Path) -> None: + from tests.rig.critical_paths import load_critical_paths + + reg = tmp_path / "critical_paths.yaml" + reg.write_text( + "paths:\n" + " - id: p1\n covered_by: [g1]\n proof: marker\n" + " - id: p2\n covered_by: [g1]\n" + ) + loaded = load_critical_paths(reg) + assert loaded.by_id("p1").proof == "marker" + assert loaded.by_id("p2").proof == "group" + + +def test_validate_rejects_marker_proof_without_covered_by() -> None: + from tests.rig.critical_paths import validate_registry_against_manifest + from tests.rig.groups import GroupDefinition, GroupManifest + + manifest = GroupManifest( + groups={"g1": GroupDefinition(name="g1", tier="unit", paths=())} + ) + registry = CriticalPathRegistry(paths=(_marker_path("p1", ()),)) + errors = validate_registry_against_manifest(registry, manifest) + assert errors and "marker" in errors[0] diff --git a/tests/rig/tests/test_reporting.py b/tests/rig/tests/test_reporting.py index dca890267b..4913100dc0 100644 --- a/tests/rig/tests/test_reporting.py +++ b/tests/rig/tests/test_reporting.py @@ -104,3 +104,43 @@ def test_status_icon_round_trips() -> None: assert pass_result.status_icon == "✅" assert fail_result.status_icon == "❌" assert empty_result.status_icon == "⚪" + + +def test_passed_critical_path_ids_collects_only_passing_marked_tests( + tmp_path: Path, +) -> None: + from tests.rig.reporting import passed_critical_path_ids + + _write_junit( + tmp_path / "g1", + """ + + + + + + + + + + + + + + + + + +""", + exit_code=1, + ) + ids = passed_critical_path_ids("g1", tmp_path) + assert ids == {"p-pass", "p-second"} + + +def test_passed_critical_path_ids_missing_or_malformed_junit(tmp_path: Path) -> None: + from tests.rig.reporting import passed_critical_path_ids + + assert passed_critical_path_ids("absent", tmp_path) == set() + _write_junit(tmp_path / "broken", "=3.0.0 - -# Set shell options for better error handling -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -# Install project dependencies separately from OpenTelemetry -# This allows for better caching when only project dependencies change -RUN uv pip install --system -r requirements.txt && \ - opentelemetry-bootstrap -a requirements | uv pip install --system --requirement - && \ - pip uninstall -y opentelemetry-instrumentation-openai-v2 && \ - pip install opentelemetry-instrumentation-openai - -# Copy the source code after installing all dependencies -# This ensures that changes to the source code don't invalidate the dependency layers -COPY tools/structure/src ${APP_HOME}/src/ -WORKDIR ${APP_HOME}/src - -ENTRYPOINT ["opentelemetry-instrument", "python", "main.py"] diff --git a/tools/structure/README.md b/tools/structure/README.md deleted file mode 100644 index 458f55895c..0000000000 --- a/tools/structure/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# Structure Tool - -This is a helper tool that needs to be used along with `Prompt Studio`. It helps process input files based on the prompts that have been designed and exported as a tool from `Prompt Studio`. - -## Required environment variables - -| Variable | Description | -| -------------------------- |-----------------------------------------------------------------------| -| `PLATFORM_SERVICE_HOST` | The host in which the platform service is running | -| `PLATFORM_SERVICE_PORT` | The port in which the service is listening | -| `PLATFORM_SERVICE_API_KEY` | The API key for the platform | -| `EXECUTION_DATA_DIR` | The directory in the filesystem which has contents for tool execution | -| `PROMPT_HOST` | The host in which the prompt service is running | -| `PROMPT_PORT` | The port in which the prompt service is listening | -| `PROMPT_PORT` | The port in which the prompt service is listening | -| `X2TEXT_HOST` | The host where the x2text service is running | -| `X2TEXT_PORT` | The port where the x2text service is listening | - -## Testing the tool locally - -### Setting up a dev environment - -Setup a virtual environment and activate it - -```commandline -python -m venv .venv -source .venv/bin/activate -``` - -Install the dependencies for the tool - -```commandline -pip install -r requirements.txt -``` - -To use the local development version of the [unstract-sdk](https://pypi.org/project/unstract-sdk/) install it from the local repository. -Replace the path with the path to your local repository - -```commandline -pip install -e ~/path_to_repo/sdks/. -``` - -### Tool execution preparation - -Load the environment variables for the tool. -Make a copy of the `sample.env` file and name it `.env`. Fill in the required values. -They get loaded with [python-dotenv](https://pypi.org/project/python-dotenv/) through the SDK. - -Update the tool's `data_dir` marked by the `EXECUTION_DATA_DIR` env. This has to be done before each tool execution since the tool updates the `INFILE` and `METADATA.json`. - -### Run SPEC command - -Represents the JSON schema for the runtime configurable `settings` of a tool - -```commandline -python main.py --command SPEC -``` - -### Run PROPERTIES command - -Describes some metadata for the tool such as its `version`, `description`, `inputs` and `outputs` - -```commandline -python main.py --command PROPERTIES -``` - -### Run ICON command - -Returns the SVG icon for the tool, used by Unstract's frontend - -```commandline -python main.py --command ICON -``` - -### Run VARIABLES command - -Represents the runtime variables or envs that will be used by the tool - -```commandline -python main.py --command VARIABLES -``` - -### Run RUN command - -The schema of the JSON required for settings can be found by running the [SPEC](#run-spec-command) command. Alternatively if you have access to the code base, it is located in the `config` folder as `spec.json`. - -```commandline -python main.py \ - --command RUN \ - --settings '{ - "prompt_registry_id": "" - }' \ - --log-level DEBUG - -``` - -## Testing the tool from its docker image - -Build the tool docker image from the folder containing the `Dockerfile` with - -```commandline -docker build -t unstract/tool-structure:0.0.1 . -``` - -Make sure the directory pointed by `EXECUTION_DATA_DIR` has the required information for the tool to run and -necessary services like the `platform-service` is up. -To test the tool from its docker image, run the following command - -```commandline -docker run -it \ - --network unstract-network \ - --env-file .env \ - -v "$(pwd)"/data_dir:/app/data_dir \ - unstract/tool-structure:0.0.1 \ - --command RUN \ - --settings '{ - "prompt_registry_id": "" - }' \ - --log-level DEBUG - -``` diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt deleted file mode 100644 index aba611c0e5..0000000000 --- a/tools/structure/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Add your dependencies here - -# Required for all unstract tools -# aws alone is needed here -# because tools use transient temporary storage. --e file:/unstract/core --e file:/unstract/sdk1[aws] --e file:/unstract/flags -json-repair>=0.25.0 diff --git a/tools/structure/sample.env b/tools/structure/sample.env deleted file mode 100644 index 0e5f82c242..0000000000 --- a/tools/structure/sample.env +++ /dev/null @@ -1,16 +0,0 @@ -PLATFORM_SERVICE_HOST=http://unstract-platform-service -PLATFORM_SERVICE_PORT=3001 -PLATFORM_SERVICE_API_KEY= -EXECUTION_DATA_DIR=../data_dir -PROMPT_HOST=http://unstract-prompt-service -PROMPT_PORT=3003 - -X2TEXT_HOST=http://unstract-x2text-service -X2TEXT_PORT=3004 - -# File System Configuration for Workflow Execution -# Directory path for execution data storage -# (e.g., bucket/execution/org_id/workflow_id/execution_id) -EXECUTION_DATA_DIR= -# Storage provider for Workflow Execution (e.g., minio, S3) -WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider":"minio","credentials"={"endpoint_url":"http://localhost:9000","key":"","secret":""}}' diff --git a/tools/structure/src/config/icon.svg b/tools/structure/src/config/icon.svg deleted file mode 100644 index e3a507bafa..0000000000 --- a/tools/structure/src/config/icon.svg +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json deleted file mode 100644 index c8697e7307..0000000000 --- a/tools/structure/src/config/properties.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "schemaVersion": "0.0.1", - "displayName": "Structure Tool", - "functionName": "structure_tool", - "toolVersion": "0.0.101", - "description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio", - "input": { - "description": "File that needs to be indexed and parsed for answers" - }, - "output": { - "description": "A JSON file containing the parsed answers for the prompts" - }, - "result": { - "type": "JSON", - "description": "A JSON containing the parsed answers for the prompts", - "schema": {} - }, - "ioCompatibility": { - "api": { - "sourceSupport": true, - "destinationSupport": true, - "additionalArgs": { - "sync": true - } - }, - "file": { - "sourceSupport": true, - "destinationSupport": true, - "additionalArgs": {} - }, - "db": { - "destinationSupport": true, - "additionalArgs": {} - } - }, - "restrictions": { - "maxFileSize": "200MB", - "allowedFileTypes": [ - "*" - ] - } -} diff --git a/tools/structure/src/config/runtime_variables.json b/tools/structure/src/config/runtime_variables.json deleted file mode 100644 index 3fc0936224..0000000000 --- a/tools/structure/src/config/runtime_variables.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "title": "Runtime Variables", - "description": "Runtime Variables for Structure Tool ", - "type": "object" -} diff --git a/tools/structure/src/config/spec.json b/tools/structure/src/config/spec.json deleted file mode 100644 index cd73e39ec5..0000000000 --- a/tools/structure/src/config/spec.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "title": "Structure Tool", - "description": "Helper tool used to export tools from Prompt Studio", - "type": "object", - "required": [ - ], - "properties": { - } -} diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py deleted file mode 100644 index 8da6a5701a..0000000000 --- a/tools/structure/src/constants.py +++ /dev/null @@ -1,108 +0,0 @@ -class SettingsKeys: - TOOL_INSTANCE_ID = "tool_instance_id" - PROMPT_REGISTRY_ID = "prompt_registry_id" - PROMPT_HOST = "PROMPT_HOST" - PROMPT_PORT = "PROMPT_PORT" - TOOL_METADATA = "tool_metadata" - TOOL_ID = "tool_id" - OUTPUTS = "outputs" - NAME = "name" - ACTIVE = "active" - PROMPT = "prompt" - CHUNK_SIZE = "chunk-size" - VECTOR_DB = "vector-db" - EMBEDDING = "embedding" - X2TEXT_ADAPTER = "x2text_adapter" - CHUNK_OVERLAP = "chunk-overlap" - LLM = "llm" - RETRIEVAL_STRATEGY = "retrieval-strategy" - SIMPLE = "simple" - TYPE = "type" - NUMBER = "number" - EMAIL = "email" - DATE = "date" - BOOLEAN = "boolean" - JSON = "json" - PREAMBLE = "preamble" - SIMILARITY_TOP_K = "similarity-top-k" - PROMPT_TOKENS = "prompt_tokens" - COMPLETION_TOKENS = "completion_tokens" - TOTAL_TOKENS = "total_tokens" - RESPONSE = "response" - POSTAMBLE = "postamble" - GRAMMAR = "grammar" - WORD = "word" - SYNONYMS = "synonyms" - OUTPUTS = "outputs" - SECTION = "section" - DEFAULT = "default" - AUTHOR = "author" - ICON = "icon" - TOOL_ID = "tool_id" - # PDF_TO_TEXT_CONVERTER = "pdf-to-text-converters" - REINDEX = "reindex" - STRUCTURE_OUTPUT = "structure_output" - TOOL_SETTINGS = "tool_settings" - ENABLE_SINGLE_PASS_EXTRACTION = "enable_single_pass_extraction" - CHALLENGE_LLM = "challenge_llm" - ENABLE_CHALLENGE = "enable_challenge" - SINGLE_PASS_EXTRACTION_MODE = "single_pass_extraction_mode" - CHALLENGE_LLM_ADAPTER_ID = "challenge_llm_adapter_id" - SUMMARIZE_AS_SOURCE = "summarize_as_source" - SUMMARIZE_PROMPT = "summarize_prompt" - CONTEXT = "context" - ERROR = "error" - LLM_ADAPTER_INSTANCE_ID = "llm_adapter_instance_id" - RUN_ID = "run_id" - PROMPT_KEYS = "prompt_keys" - DATA = "data" - EXTRACT = "EXTRACT" - SUMMARIZE = "SUMMARIZE" - STATUS = "status" - OK = "OK" - FILE_NAME = "file_name" - FILE_HASH = "file_hash" - ENABLE_HIGHLIGHT = "enable_highlight" - ENABLE_WORD_CONFIDENCE = "enable_word_confidence" - NAME = "name" - INCLUDE_METADATA = "include_metadata" - TABLE_SETTINGS = "table_settings" - INPUT_FILE = "input_file" - METADATA = "metadata" - EPILOGUE = "epilogue" - HIGHLIGHT_DATA = "highlight_data" - CONFIDENCE_DATA = "confidence_data" - FILE_PATH = "file_path" - EXECUTION_SOURCE = "execution_source" - TOOL = "tool" - METRICS = "metrics" - INDEXING = "indexing" - EXECUTION_ID = "execution_id" - IS_DIRECTORY_MODE = "is_directory_mode" - LLM_PROFILE_ID = "llm_profile_id" - CUSTOM_DATA = "custom_data" - OUTPUT = "output" # For API deployment response format compatibility - - -class IndexingConstants: - TOOL_ID = "tool_id" - EMBEDDING_INSTANCE_ID = "embedding_instance_id" - VECTOR_DB_INSTANCE_ID = "vector_db_instance_id" - X2TEXT_INSTANCE_ID = "x2text_instance_id" - FILE_PATH = "file_path" - CHUNK_SIZE = "chunk_size" - CHUNK_OVERLAP = "chunk_overlap" - REINDEX = "reindex" - FILE_HASH = "file_hash" - OUTPUT_FILE_PATH = "output_file_path" - ENABLE_HIGHLIGHT = "enable_highlight" - USAGE_KWARGS = "usage_kwargs" - PROCESS_TEXT = "process_text" - EXTRACTED_TEXT = "extracted_text" - TAGS = "tags" - EXECUTION_SOURCE = "execution_source" - DOC_ID = "doc_id" - TOOL_EXECUTION_METATADA = "tool_execution_metadata" - EXECUTION_DATA_DIR = "execution_data_dir" - RUN_ID = "run_id" - EXECUTION_ID = "execution_id" diff --git a/tools/structure/src/helpers.py b/tools/structure/src/helpers.py deleted file mode 100644 index 2cb0d9bf72..0000000000 --- a/tools/structure/src/helpers.py +++ /dev/null @@ -1,134 +0,0 @@ -import datetime -import logging -from collections.abc import Callable -from typing import Any - -from constants import IndexingConstants as IKeys -from constants import SettingsKeys # type: ignore [attr-defined] - -from unstract.sdk1.prompt import PromptTool -from unstract.sdk1.tool.base import BaseTool - -logger = logging.getLogger(__name__) - - -class StructureToolHelper: - @staticmethod - def dynamic_extraction( - file_path: str, - enable_highlight: bool, - usage_kwargs: dict[str, Any], - run_id: str, - tool_settings: dict[str, Any], - extract_file_path: str, - tool: BaseTool, - execution_run_data_folder: str, - ) -> str: - x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER] - payload = { - IKeys.X2TEXT_INSTANCE_ID: x2text, - IKeys.FILE_PATH: file_path, - IKeys.ENABLE_HIGHLIGHT: enable_highlight, - IKeys.USAGE_KWARGS: usage_kwargs.copy(), - IKeys.RUN_ID: run_id, - IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL, - IKeys.OUTPUT_FILE_PATH: str(extract_file_path), - IKeys.TAGS: tool.tags, - IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata, - IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder), - } - - logger.info(f"Prompt service payload for text extraction:\n{payload}") - - prompt_tool = PromptTool( - tool=tool, - prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST), - prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT), - request_id=run_id, - ) - return prompt_tool.extract(payload=payload) - - @staticmethod - def dynamic_indexing( - file_path: str, - tool_settings: dict[str, Any], - run_id: str, - tool: BaseTool, - execution_run_data_folder: str, - reindex: bool, - usage_kwargs: dict[str, Any], - enable_highlight: bool, - chunk_size: int, - chunk_overlap: int, - file_hash: str | None = None, - tool_id: str = None, - extracted_text: str = None, - ) -> str: - x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER] - - payload = { - IKeys.TOOL_ID: tool_id, - IKeys.EMBEDDING_INSTANCE_ID: tool_settings[SettingsKeys.EMBEDDING], - IKeys.VECTOR_DB_INSTANCE_ID: tool_settings[SettingsKeys.VECTOR_DB], - IKeys.X2TEXT_INSTANCE_ID: x2text, - IKeys.FILE_HASH: file_hash, - IKeys.CHUNK_SIZE: chunk_size, - IKeys.CHUNK_OVERLAP: chunk_overlap, - IKeys.REINDEX: reindex, - IKeys.FILE_PATH: str(file_path), - IKeys.ENABLE_HIGHLIGHT: enable_highlight, - IKeys.USAGE_KWARGS: usage_kwargs.copy(), - IKeys.RUN_ID: run_id, - IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL, - IKeys.TAGS: tool.tags, - IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata, - IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder), - IKeys.EXTRACTED_TEXT: extracted_text, - } - - sensitive_keys = [IKeys.EXTRACTED_TEXT] - payload_to_log = {k: v for k, v in payload.items() if k not in sensitive_keys} - logger.info(f"Prompt service payload for indexing:\n{payload_to_log}") - responder = PromptTool( - tool=tool, - prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST), - prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT), - request_id=run_id, - ) - return responder.index(payload=payload) - - @staticmethod - def handle_profile_overrides( - tool: BaseTool, - llm_profile_to_override: dict, - llm_profile_id: str, - tool_metadata: dict, - apply_profile_overrides_func: Callable[[dict, dict], list[str]], - ) -> None: - """Handle profile overrides and logging. - - Args: - tool: The tool instance for logging - llm_profile_to_override: The profile data to apply, or None if no profile - llm_profile_id: The profile ID for logging purposes - tool_metadata: The tool metadata dictionary to modify - apply_profile_overrides_func: Function to apply profile overrides - """ - if llm_profile_to_override: - tool.stream_log( - f"Applying profile overrides from profile: {llm_profile_to_override.get('profile_name', llm_profile_id)}" - ) - changes = apply_profile_overrides_func(tool_metadata, llm_profile_to_override) - if changes: - tool.stream_log("Profile overrides applied successfully. Changes made:") - for change in changes: - tool.stream_log(f" - {change}") - else: - tool.stream_log( - "Profile overrides applied - no changes needed (values already matched)" - ) - - @staticmethod - def elapsed_time(start_time) -> float: - """Returns the elapsed time since the process was started.""" - return (datetime.datetime.now() - start_time).total_seconds() diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py deleted file mode 100644 index f68143a6c8..0000000000 --- a/tools/structure/src/main.py +++ /dev/null @@ -1,772 +0,0 @@ -import datetime -import json -import logging -import os -import sys -from pathlib import Path -from typing import Any - -from constants import SettingsKeys # type: ignore [attr-defined] -from helpers import StructureToolHelper as STHelper -from utils import json_to_markdown, repair_json_with_best_structure - -from unstract.sdk1.constants import LogState, MetadataKey, ToolEnv, UsageKwargs -from unstract.sdk1.platform import PlatformHelper -from unstract.sdk1.prompt import PromptTool -from unstract.sdk1.tool.base import BaseTool -from unstract.sdk1.tool.entrypoint import ToolEntrypoint - -logger = logging.getLogger(__name__) - -PAID_FEATURE_MSG = ( - "It is a cloud / enterprise feature. If you have purchased a plan and still " - "face this issue, please contact support" -) - - -class StructureTool(BaseTool): - def _apply_profile_overrides( - self, tool_metadata: dict, profile_data: dict - ) -> list[str]: - """Apply profile overrides to tool metadata. - - Args: - tool_metadata: The tool metadata dictionary to modify - profile_data: The profile data containing override values - - Returns: - List of change descriptions - """ - changes = [] - - # Mapping between profile keys and tool metadata keys - profile_to_tool_mapping = { - "chunk_overlap": "chunk-overlap", - "chunk_size": "chunk-size", - "embedding_model_id": "embedding", - "llm_id": "llm", - "similarity_top_k": "similarity-top-k", - "vector_store_id": "vector-db", - "x2text_id": "x2text_adapter", - "retrieval_strategy": "retrieval-strategy", - } - - # Override tool_settings section - if "tool_settings" in tool_metadata: - tool_changes = self._override_section( - tool_metadata["tool_settings"], - profile_data, - profile_to_tool_mapping, - section_name="tool_settings", - ) - changes.extend(tool_changes) - - # Override each output in outputs section - if "outputs" in tool_metadata: - for i, output in enumerate(tool_metadata["outputs"]): - output_name = output.get("name", f"output_{i}") - output_changes = self._override_section( - output, - profile_data, - profile_to_tool_mapping, - section_name=f"output[{output_name}]", - ) - changes.extend(output_changes) - - return changes - - def _override_section( - self, - section: dict, - profile_data: dict, - mapping: dict, - section_name: str = "section", - ) -> list[str]: - """Override values in a section using profile data. - - Args: - section: The section dictionary to modify - profile_data: The profile data containing override values - mapping: Mapping between profile keys and section keys - section_name: Name of the section for logging purposes - - Returns: - List of change descriptions - """ - changes = [] - for profile_key, section_key in mapping.items(): - if profile_key in profile_data and section_key in section: - old_value = section[section_key] - new_value = profile_data[profile_key] - if old_value != new_value: # Only track actual changes - section[section_key] = new_value - change_desc = ( - f"{section_name}.{section_key}: {old_value} -> {new_value}" - ) - changes.append(change_desc) - self.stream_log(f"Overrode {change_desc}") - return changes - - def _should_skip_extraction_for_smart_table( - self, input_file: str, outputs: list[dict[str, Any]] - ) -> bool: - """Check if extraction and indexing should be skipped for smart table extraction. - - Args: - input_file: Path to the input file - outputs: List of output configurations - - Returns: - True if extraction/indexing should be skipped, False otherwise - """ - # Check if any output has table_settings with valid JSON prompt - for output in outputs: - if SettingsKeys.TABLE_SETTINGS in output: - prompt = output.get(SettingsKeys.PROMPT, "") - if prompt and isinstance(prompt, str): - try: - # Try to parse the prompt as JSON - schema_data = repair_json_with_best_structure(prompt) - # If it's a valid dict (schema object), skip extraction - if schema_data and isinstance(schema_data, dict): - return True - except Exception as e: - logger.warning( - "Failed to parse prompt as JSON for smart table extraction: %s", - str(e), - ) - continue - return False - - def validate(self, input_file: str, settings: dict[str, Any]) -> None: - enable_challenge: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False) - challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "") - if enable_challenge and not challenge_llm: - raise ValueError("Challenge LLM is not set after enabling Challenge") - - def _is_agentic_project(self, tool_metadata: dict[str, Any]) -> bool: - """Check if metadata indicates an agentic project. - - Agentic projects have project_id and json_schema fields. - Prompt studio projects have tool_id and outputs fields. - """ - return "project_id" in tool_metadata and "json_schema" in tool_metadata - - def run( - self, - settings: dict[str, Any], - input_file: str, - output_dir: str, - ) -> None: - prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID] - is_challenge_enabled: bool = settings.get(SettingsKeys.ENABLE_CHALLENGE, False) - is_summarization_enabled: bool = settings.get( - SettingsKeys.SUMMARIZE_AS_SOURCE, False - ) - is_single_pass_enabled: bool = settings.get( - SettingsKeys.SINGLE_PASS_EXTRACTION_MODE, False - ) - challenge_llm: str = settings.get(SettingsKeys.CHALLENGE_LLM_ADAPTER_ID, "") - is_highlight_enabled: bool = settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False) - responder: PromptTool = PromptTool( - tool=self, - prompt_port=self.get_env_or_die(SettingsKeys.PROMPT_PORT), - prompt_host=self.get_env_or_die(SettingsKeys.PROMPT_HOST), - request_id=self.file_execution_id, - ) - self.stream_log(f"Fetching exported tool with UUID '{prompt_registry_id}'") - - platform_helper: PlatformHelper = PlatformHelper( - tool=self, - platform_port=self.get_env_or_die(ToolEnv.PLATFORM_PORT), - platform_host=self.get_env_or_die(ToolEnv.PLATFORM_HOST), - request_id=self.file_execution_id, - ) - - # Try to fetch as prompt studio tool first - tool_metadata = None - is_agentic = False - exported_tool = None - - try: - exported_tool = platform_helper.get_prompt_studio_tool( - prompt_registry_id=prompt_registry_id - ) - except Exception as e: - # If prompt studio lookup fails, try as agentic project - self.stream_log( - f"Not found as prompt studio project, trying agentic registry: {e}" - ) - - if exported_tool and SettingsKeys.TOOL_METADATA in exported_tool: - tool_metadata = exported_tool[SettingsKeys.TOOL_METADATA] - is_agentic = False - # Explicitly mark metadata for clarity - tool_metadata["is_agentic"] = False - else: - # Try agentic registry as fallback - try: - agentic_tool = platform_helper.get_agentic_studio_tool( - agentic_registry_id=prompt_registry_id - ) - if not agentic_tool or SettingsKeys.TOOL_METADATA not in agentic_tool: - self.stream_error_and_exit( - f"Error fetching project: Registry returned empty response for {prompt_registry_id}" - ) - tool_metadata = agentic_tool[SettingsKeys.TOOL_METADATA] - is_agentic = True - # Explicitly mark metadata for clarity - tool_metadata["is_agentic"] = True - self.stream_log( - f"Retrieved agentic project: {tool_metadata.get('name', prompt_registry_id)}" - ) - except Exception as agentic_error: - self.stream_error_and_exit( - f"Error fetching project from both registries " - f"for ID '{prompt_registry_id}': {agentic_error}" - ) - - # Route to appropriate extraction method - if is_agentic: - return self._run_agentic_extraction( - tool_metadata=tool_metadata, - input_file=input_file, - output_dir=output_dir, - settings=settings, - responder=responder, - platform_helper=platform_helper, - ) - - # Continue with regular prompt studio extraction - llm_profile_id = self.get_exec_metadata.get(SettingsKeys.LLM_PROFILE_ID) - llm_profile_to_override = None - try: - if llm_profile_id: - llm_profile = platform_helper.get_llm_profile(llm_profile_id) - llm_profile_to_override = llm_profile - - # Apply profile overrides if available - STHelper.handle_profile_overrides( - self, - llm_profile_to_override, - llm_profile_id, - tool_metadata, - self._apply_profile_overrides, - ) - except Exception as e: - self.stream_error_and_exit(f"Error fetching prompt studio project: {e}") - ps_project_name = tool_metadata.get("name", prompt_registry_id) - # Count only the active (enabled) prompts - total_prompt_count = len(tool_metadata[SettingsKeys.OUTPUTS]) - self.stream_log( - f"Retrieved prompt studio exported tool '{ps_project_name}' having " - f"'{total_prompt_count}' prompts" - ) - - active_prompt_count = len( - [ - output - for output in tool_metadata[SettingsKeys.OUTPUTS] - if output.get("active", False) - ] - ) - # Update GUI - input_log = f"## Loaded '{ps_project_name}'\n{json_to_markdown(tool_metadata)}\n" - output_log = ( - f"## Processing '{self.source_file_name}'\nThis might take a while and " - "involve...\n- Extracting text\n- Indexing\n- Retrieving answers " - f"for '{active_prompt_count}' prompts" - ) - self.stream_update(input_log, state=LogState.INPUT_UPDATE) - self.stream_update(output_log, state=LogState.OUTPUT_UPDATE) - - file_hash = self.get_exec_metadata.get(MetadataKey.SOURCE_HASH) - tool_id = tool_metadata[SettingsKeys.TOOL_ID] - tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS] - outputs = tool_metadata[SettingsKeys.OUTPUTS] - tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llm - tool_settings[SettingsKeys.ENABLE_CHALLENGE] = is_challenge_enabled - tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = is_single_pass_enabled - tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = is_summarization_enabled - tool_settings[SettingsKeys.ENABLE_HIGHLIGHT] = is_highlight_enabled - _, file_name = os.path.split(input_file) - if is_summarization_enabled: - file_name = SettingsKeys.SUMMARIZE - tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR)) - execution_run_data_folder = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR)) - - extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT) - # Resolve and pass log events ID - payload = { - SettingsKeys.RUN_ID: self.file_execution_id, - SettingsKeys.EXECUTION_ID: self.execution_id, - SettingsKeys.TOOL_SETTINGS: tool_settings, - SettingsKeys.OUTPUTS: outputs, - SettingsKeys.TOOL_ID: tool_id, - SettingsKeys.FILE_HASH: file_hash, - SettingsKeys.FILE_NAME: file_name, - SettingsKeys.FILE_PATH: extracted_input_file, - SettingsKeys.EXECUTION_SOURCE: SettingsKeys.TOOL, - } - - custom_data = self.get_exec_metadata.get(SettingsKeys.CUSTOM_DATA, {}) - payload["custom_data"] = custom_data - - # Check if we should skip extraction and indexing for Excel table extraction with valid JSON - skip_extraction_and_indexing = self._should_skip_extraction_for_smart_table( - input_file, outputs - ) - - extracted_text = "" - usage_kwargs: dict[Any, Any] = dict() - if skip_extraction_and_indexing: - self.stream_log( - "Skipping extraction and indexing for Excel table with valid JSON schema" - ) - else: - self.stream_log(f"Extracting document '{self.source_file_name}'") - usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id - usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name - usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id - extracted_text = STHelper.dynamic_extraction( - file_path=input_file, - enable_highlight=is_highlight_enabled, - usage_kwargs=usage_kwargs, - run_id=self.file_execution_id, - tool_settings=tool_settings, - extract_file_path=tool_data_dir / SettingsKeys.EXTRACT, - tool=self, - execution_run_data_folder=str(execution_run_data_folder), - ) - - index_metrics = {} - if is_summarization_enabled: - summarize_file_path, summarize_file_hash = self._summarize( - tool_settings=tool_settings, - tool_data_dir=tool_data_dir, - responder=responder, - outputs=outputs, - usage_kwargs=usage_kwargs, - ) - payload[SettingsKeys.FILE_HASH] = summarize_file_hash - payload[SettingsKeys.FILE_PATH] = summarize_file_path - elif skip_extraction_and_indexing: - # Use source file directly for Excel with valid JSON - payload[SettingsKeys.FILE_PATH] = input_file - pass - elif not is_single_pass_enabled: - # Track seen parameter combinations to avoid duplicate indexing - seen_params = set() - - for output in outputs: - # Get current parameter combination - chunk_size = output[SettingsKeys.CHUNK_SIZE] - chunk_overlap = output[SettingsKeys.CHUNK_OVERLAP] - vector_db = tool_settings[SettingsKeys.VECTOR_DB] - embedding = tool_settings[SettingsKeys.EMBEDDING] - x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER] - - # Create a unique key for this parameter combination - param_key = ( - f"chunk_size={chunk_size}_" - f"chunk_overlap={chunk_overlap}_" - f"vector_db={vector_db}_" - f"embedding={embedding}_" - f"x2text={x2text}" - ) - - # Only process if we haven't seen this combination yet and chunk_size is not zero - if chunk_size != 0 and param_key not in seen_params: - seen_params.add(param_key) - - indexing_start_time = datetime.datetime.now() - self.stream_log( - f"Indexing document with: chunk_size={chunk_size}, " - f"chunk_overlap={chunk_overlap}, vector_db={vector_db}, " - f"embedding={embedding}, x2text={x2text}" - ) - - STHelper.dynamic_indexing( - tool_settings=tool_settings, - run_id=self.file_execution_id, - file_path=tool_data_dir / SettingsKeys.EXTRACT, - tool=self, - execution_run_data_folder=str(execution_run_data_folder), - chunk_overlap=chunk_overlap, - reindex=True, - usage_kwargs=usage_kwargs, - enable_highlight=is_highlight_enabled, - chunk_size=chunk_size, - tool_id=tool_metadata[SettingsKeys.TOOL_ID], - file_hash=file_hash, - extracted_text=extracted_text, - ) - - index_metrics[output[SettingsKeys.NAME]] = { - SettingsKeys.INDEXING: { - "time_taken(s)": STHelper.elapsed_time( - start_time=indexing_start_time - ) - } - } - - if is_single_pass_enabled: - self.stream_log("Fetching response for single pass extraction...") - structured_output = responder.single_pass_extraction( - payload=payload, - ) - else: - for output in outputs: - if SettingsKeys.TABLE_SETTINGS in output: - table_settings = output[SettingsKeys.TABLE_SETTINGS] - is_directory_mode: bool = table_settings.get( - SettingsKeys.IS_DIRECTORY_MODE, False - ) - # Use source file directly for Excel with valid JSON, otherwise use extracted file - if skip_extraction_and_indexing: - table_settings[SettingsKeys.INPUT_FILE] = input_file - payload[SettingsKeys.FILE_PATH] = input_file - else: - table_settings[SettingsKeys.INPUT_FILE] = extracted_input_file - table_settings[SettingsKeys.IS_DIRECTORY_MODE] = is_directory_mode - self.stream_log(f"Performing table extraction with: {table_settings}") - output.update({SettingsKeys.TABLE_SETTINGS: table_settings}) - - self.stream_log(f"Fetching responses for '{len(outputs)}' prompt(s)...") - structured_output = responder.answer_prompt( - payload=payload, - ) - - # HACK: Replacing actual file's name instead of INFILE - # Ensure metadata section exists - if SettingsKeys.METADATA not in structured_output: - structured_output[SettingsKeys.METADATA] = {} - self.stream_log("Created metadata section in structured_output") - - structured_output[SettingsKeys.METADATA][SettingsKeys.FILE_NAME] = ( - self.source_file_name - ) - - # Add extracted text for HITL raw view - if extracted_text: - structured_output[SettingsKeys.METADATA]["extracted_text"] = extracted_text - self.stream_log( - f"Added text extracted from the document to metadata (length: {len(extracted_text)} characters)" - ) - else: - self.stream_log( - "No text is extracted from the document to add to the metadata" - ) - if merged_metrics := self._merge_metrics( - structured_output.get(SettingsKeys.METRICS, {}), index_metrics - ): - structured_output[SettingsKeys.METRICS] = merged_metrics - # Update GUI - output_log = ( - f"## Result\n**NOTE:** In case of a deployed pipeline, the result would " - "be a JSON. This has been rendered for readability here\n" - f"{json_to_markdown(structured_output)}\n" - ) - self.stream_update(output_log, state=LogState.OUTPUT_UPDATE) - - try: - self.stream_log( - "Writing prompt studio project's output to workflow's storage" - ) - output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.json" - self.workflow_filestorage.json_dump(path=output_path, data=structured_output) - self.stream_log( - "Prompt studio project's output written successfully to workflow's storage" - ) - except OSError as e: - self.stream_error_and_exit(f"Error creating output file: {e}") - except json.JSONDecodeError as e: - self.stream_error_and_exit(f"Error encoding JSON: {e}") - self.write_tool_result(data=structured_output) - - def _remove_source_refs(self, data: Any) -> Any: - """Recursively remove _source_refs from data structure. - - Args: - data: Data structure (dict, list, or primitive) to clean - - Returns: - Cleaned data structure without _source_refs fields - """ - if isinstance(data, dict): - return { - key: self._remove_source_refs(value) - for key, value in data.items() - if key != "_source_refs" - } - elif isinstance(data, list): - return [self._remove_source_refs(item) for item in data] - else: - return data - - def _merge_metrics(self, metrics1: dict, metrics2: dict) -> dict: - """Intelligently merge two metrics dictionaries. - - For keys that exist in both dictionaries with dictionary values, merge the dictionaries. - For keys that exist in only one dictionary or have non-dictionary values, use the value as-is. - - Args: - metrics1 (dict): First metrics dictionary - metrics2 (dict): Second metrics dictionary - - Returns: - dict: Merged metrics dictionary - """ - merged_metrics = {} - - # Get all unique keys from both dictionaries - all_keys = set(metrics1) | set(metrics2) - - for key in all_keys: - # If key exists in both dictionaries and both values are dictionaries, merge them - if ( - key in metrics1 - and key in metrics2 - and isinstance(metrics1[key], dict) - and isinstance(metrics2[key], dict) - ): - merged_metrics[key] = {**metrics1[key], **metrics2[key]} - # Otherwise just take the value from whichever dictionary has it - elif key in metrics1: - merged_metrics[key] = metrics1[key] - else: - merged_metrics[key] = metrics2[key] - - return merged_metrics - - def _run_agentic_extraction( - self, - tool_metadata: dict[str, Any], - input_file: str, - output_dir: str, - settings: dict[str, Any], - responder: PromptTool, - platform_helper: PlatformHelper, - ) -> None: - """Execute agentic extraction pipeline. - - Args: - tool_metadata: Complete agentic project metadata - input_file: Path to input document - output_dir: Directory for output files - settings: Workflow settings (contains adapter overrides) - responder: PromptTool instance for API calls - platform_helper: PlatformHelper instance for platform API calls - """ - from unstract.sdk1.x2txt import X2Text - - # Extract agentic-specific metadata - project_id = tool_metadata.get("project_id") - project_name = tool_metadata.get("name", project_id) - json_schema = tool_metadata.get("json_schema", {}) - prompt_text = tool_metadata.get("prompt_text", "") - prompt_version = tool_metadata.get("prompt_version", 1) - schema_version = tool_metadata.get("schema_version", 1) - adapter_config = tool_metadata.get("adapter_config", {}) - - self.stream_log( - f"Executing agentic extraction for project '{project_name}' " - f"(schema v{schema_version}, prompt v{prompt_version})" - ) - - # Get adapter IDs from settings (workflow UI overrides) - # or fall back to exported defaults - extractor_llm = settings.get( - "extractor_llm_adapter_id", adapter_config.get("extractor_llm") - ) - llmwhisperer = settings.get( - "llmwhisperer_adapter_id", adapter_config.get("llmwhisperer") - ) - enable_highlight = settings.get( - SettingsKeys.ENABLE_HIGHLIGHT, tool_metadata.get("enable_highlight", False) - ) - - # Get platform details for organization_id - platform_details = platform_helper.get_platform_details() - organization_id = ( - platform_details.get("organization_id") if platform_details else None - ) - - if not organization_id: - self.stream_error_and_exit("Failed to get organization_id from platform") - - # Update GUI - input_log = ( - f"## Loaded Agentic Project '{project_name}'\n" - f"- **Project ID**: {project_id}\n" - f"- **Schema Version**: {schema_version}\n" - f"- **Prompt Version**: {prompt_version}\n" - f"- **Extractor LLM**: {extractor_llm}\n" - ) - output_log = ( - f"## Processing '{self.source_file_name}'\n" - "Executing agentic extraction pipeline...\n" - "- Extracting document text\n" - "- Running LLM extraction\n" - ) - self.stream_update(input_log, state=LogState.INPUT_UPDATE) - self.stream_update(output_log, state=LogState.OUTPUT_UPDATE) - - try: - # Step 1: Extract text from document using X2Text/LLMWhisperer - self.stream_log("Extracting text from document...") - x2text = X2Text(tool=self, adapter_instance_id=llmwhisperer) - - extraction_result = x2text.process( - input_file_path=input_file, - enable_highlight=enable_highlight, - fs=self.workflow_filestorage, - ) - - document_text = extraction_result.extracted_text - line_metadata = ( - extraction_result.extraction_metadata.line_metadata - if extraction_result.extraction_metadata - else None - ) - - self.stream_log(f"Extracted {len(document_text)} characters of text") - - # Step 2: Build extraction payload for /agentic/extract endpoint - payload = { - "document_id": self.file_execution_id, # Use run ID as document ID - "prompt_text": prompt_text, - "document_text": document_text, - "schema": json_schema, - "organization_id": organization_id, - "adapter_instance_id": extractor_llm, - "include_source_refs": enable_highlight, - } - - # Add line_metadata if available for highlighting - if line_metadata and enable_highlight: - payload["line_metadata"] = line_metadata - self.stream_log( - f"Including {len(line_metadata)} line metadata entries for highlighting" - ) - - # Step 3: Call agentic extraction endpoint - self.stream_log("Calling agentic extraction endpoint...") - extraction_response = responder.agentic_extraction(payload=payload) - - # Step 4: Process response from agentic extraction - extracted_data = extraction_response.get(SettingsKeys.OUTPUT, {}) - - # Remove _source_refs from extracted data - try: - extracted_data = self._remove_source_refs(extracted_data) - except Exception as e: - self.stream_log( - f"Warning: Failed to remove _source_refs: {e}. " - "Proceeding with original data." - ) - - # Build final structured output in prompt studio format - structured_output = { - SettingsKeys.OUTPUT: extracted_data, - SettingsKeys.METADATA: { - **extraction_response.get(SettingsKeys.METADATA, {}), - SettingsKeys.FILE_NAME: self.source_file_name, - "project_id": project_id, - "schema_version": schema_version, - "prompt_version": prompt_version, - "document_id": self.file_execution_id, - }, - } - output_log = ( - f"## Agentic Extraction Complete\n" - f"Successfully extracted data from '{self.source_file_name}'\n" - f"{json_to_markdown(structured_output)}\n" - ) - self.stream_update(output_log, state=LogState.OUTPUT_UPDATE) - - # Write output to file - self.stream_log("Writing agentic extraction output to workflow storage") - output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.json" - self.workflow_filestorage.json_dump(path=output_path, data=structured_output) - self.stream_log("Output written successfully to workflow storage") - - # Write tool result - self.write_tool_result(data=structured_output) - - except Exception as e: - self.stream_error_and_exit(f"Error during agentic extraction: {e}") - - def _summarize( - self, - tool_settings: dict[str, Any], - tool_data_dir: Path, - responder: PromptTool, - outputs: dict[str, Any], - usage_kwargs: dict[Any, Any] = {}, - ) -> tuple[str, str]: - """Summarizes the context of the file. - - Args: - tool_settings (dict[str, Any]): Settings for the tool. - tool_data_dir (Path): Directory where tool data is stored. - responder (PromptTool): Instance of a tool used to generate the summary. - outputs (dict[str, Any]): Dictionary containing prompt details. - usage_kwargs (dict[Any, Any]): Used to capture usage metrics. - - Returns: - tuple[str, str]: Tuple containing the path to the summarized file and its hash. - """ - llm_adapter_instance_id: str = tool_settings[SettingsKeys.LLM] - embedding_instance_id: str = tool_settings[SettingsKeys.EMBEDDING] - vector_db_instance_id: str = tool_settings[SettingsKeys.VECTOR_DB] - x2text_instance_id: str = tool_settings[SettingsKeys.X2TEXT_ADAPTER] - summarize_prompt: str = tool_settings[SettingsKeys.SUMMARIZE_PROMPT] - run_id: str = usage_kwargs.get(UsageKwargs.RUN_ID) - extract_file_path = tool_data_dir / SettingsKeys.EXTRACT - summarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZE - - summarized_context = "" - self.stream_log( - f"Checking if summarized context exists at '{summarize_file_path}'..." - ) - if self.workflow_filestorage.exists(summarize_file_path): - summarized_context = self.workflow_filestorage.read( - path=summarize_file_path, mode="r" - ) - if not summarized_context: - context = "" - context = self.workflow_filestorage.read(path=extract_file_path, mode="r") - prompt_keys = [] - for output in outputs: - prompt_keys.append(output[SettingsKeys.NAME]) - output[SettingsKeys.EMBEDDING] = embedding_instance_id - output[SettingsKeys.VECTOR_DB] = vector_db_instance_id - output[SettingsKeys.X2TEXT_ADAPTER] = x2text_instance_id - output[SettingsKeys.CHUNK_SIZE] = 0 - output[SettingsKeys.CHUNK_OVERLAP] = 0 - self.stream_log("Summarized context not found, summarizing...") - payload = { - SettingsKeys.RUN_ID: run_id, - SettingsKeys.LLM_ADAPTER_INSTANCE_ID: llm_adapter_instance_id, - SettingsKeys.SUMMARIZE_PROMPT: summarize_prompt, - SettingsKeys.CONTEXT: context, - SettingsKeys.PROMPT_KEYS: prompt_keys, - } - structure_output = responder.summarize(payload=payload) - summarized_context = structure_output.get(SettingsKeys.DATA, "") - self.stream_log(f"Writing summarized context to '{summarize_file_path}'") - self.workflow_filestorage.write( - path=summarize_file_path, mode="w", data=summarized_context - ) - - summarize_file_hash: str = self.workflow_filestorage.get_hash_from_file( - path=summarize_file_path - ) - return str(summarize_file_path), summarize_file_hash - - -if __name__ == "__main__": - args = sys.argv[1:] - tool = StructureTool.from_tool_args(args=args) - ToolEntrypoint.launch(tool=tool, args=args) diff --git a/tools/structure/src/utils.py b/tools/structure/src/utils.py deleted file mode 100644 index a374918aba..0000000000 --- a/tools/structure/src/utils.py +++ /dev/null @@ -1,100 +0,0 @@ -from typing import Any - -from json_repair import repair_json - - -def json_to_markdown(data: Any, level: int = 0, parent_key: str = "") -> str: - markdown = "" - indent = " " * level # Increase indentation by 2 for nested levels - - if isinstance(data, dict): - for key, value in data.items(): - if isinstance(value, (dict, list)): - # If value is a dict or list, make it expandable - markdown += f"{indent}- **{key}**:\n" - markdown += json_to_markdown(value, level + 1, key) - else: - markdown += f"{indent}- **{key}**: {value}\n" - elif isinstance(data, list): - for index, item in enumerate(data, 1): - # Use parent key for list item naming - # Fall back to "Item" if parent_key is empty - # TODO: Determine child key using parent key for all plural combinations - item_label = ( - f"{parent_key[:-1] if parent_key.endswith('s') else parent_key} {index}" - if parent_key - else f"Item {index}" - ) - markdown += f"{indent}- **{item_label}**\n" - markdown += json_to_markdown(item, level + 1, parent_key) - else: - markdown += f"{indent}- {data}\n" - - return markdown - - -def repair_json_with_best_structure(json_str: str) -> Any: - """Repair and parse a potentially malformed JSON string with optimal structure detection. - - This function attempts to repair and parse a JSON string using two different strategies - and returns the result that produces the most useful data structure. It handles cases - where the input might be incomplete, malformed, or ambiguous JSON. - - The function tries two parsing approaches: - 1. Parse the JSON string as-is - 2. Parse the JSON string wrapped in array brackets [...] - - It then intelligently selects the best result based on the following logic: - - If both results are strings (failed to parse as objects), return the as-is result - - If one result is a string and the other is an object/array, return the object/array - - If wrapping produces a single-element list that equals the as-is result, return as-is - - If as-is produces an object/array and wrapping produces multiple elements, prefer wrapped - - Otherwise, prefer the as-is result - - Args: - json_str: A string containing potentially malformed JSON data. Can be a complete - JSON object, array, or partial JSON that needs repair. - - Returns: - The parsed JSON structure (dict, list, str, or other JSON-compatible type) that - represents the most meaningful interpretation of the input string. The return type - depends on the input and which parsing strategy produces the better result. - - Example: - >>> repair_json_with_best_structure('{"name": "John", "age": 30}') - {'name': 'John', 'age': 30} - - >>> repair_json_with_best_structure('{"incomplete": "object"') - {'incomplete': 'object'} - - >>> repair_json_with_best_structure('{"a": 1}{"b": 2}') - [{'a': 1}, {'b': 2}] - - Note: - This function is specifically designed for the structure-tool and uses the - json_repair library's repair_json function with return_objects=True and - ensure_ascii=False parameters. - """ - parsed_as_is = repair_json(json_str=json_str, return_objects=True, ensure_ascii=False) - parsed_with_wrap = repair_json( - "[" + json_str + "]", return_objects=True, ensure_ascii=False - ) - - if all(isinstance(x, str) for x in (parsed_as_is, parsed_with_wrap)): - return parsed_as_is - - if isinstance(parsed_as_is, str): - return parsed_with_wrap - if isinstance(parsed_with_wrap, str): - return parsed_as_is - - if isinstance(parsed_with_wrap, list) and len(parsed_with_wrap) == 1: - if parsed_with_wrap[0] == parsed_as_is: - return parsed_as_is - - if isinstance(parsed_as_is, (dict, list)): - if isinstance(parsed_with_wrap, list) and len(parsed_with_wrap) > 1: - return parsed_with_wrap - return parsed_as_is - - return parsed_with_wrap diff --git a/tox.ini b/tox.ini index 318539029e..798d4a96c3 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ # tox -e groups -- --from-file .test-selection --no-coverage # tox -e groups -- all # everything (equivalent to unit,integration,e2e) # -# Pre-existing service envs (runner, sdk1, prompt-service) remain as thin +# Pre-existing service envs (runner, sdk1) remain as thin # aliases so existing CI commands continue to work during the migration. env_list = unit, integration, e2e requires = tox-uv>=0.2.0 @@ -42,6 +42,9 @@ passenv = GITHUB_* DOCKER_HOST UNSTRACT_* + # ComposeRuntime's `docker compose up` resolves ${VERSION} image tags in + # docker/docker-compose.yaml from the caller's environment. + VERSION changedir = {toxinidir} [testenv:unit] @@ -69,11 +72,5 @@ commands = python -m tests.rig {posargs:list-groups} # These mirror the pre-rig envs so existing scripts / CI snippets keep working # during the migration. They delegate to the corresponding rig group. -[testenv:runner] -commands = python -m tests.rig run unit-runner {posargs} - [testenv:sdk1] commands = python -m tests.rig run unit-sdk1 {posargs} - -[testenv:prompt-service] -commands = python -m tests.rig run unit-prompt-service {posargs} diff --git a/unstract/connectors/pyproject.toml b/unstract/connectors/pyproject.toml index fa3afe95dd..2e79f8cec6 100644 --- a/unstract/connectors/pyproject.toml +++ b/unstract/connectors/pyproject.toml @@ -64,3 +64,7 @@ unstract-filesystem = { path = "../filesystem", editable = true } [tool.pytest.ini_options] pythonpath = ["src"] +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] +markers = [ + "integration: needs live infra or external credentials; runs in the rig's integration tier, not unit (select with -m integration / exclude with -m 'not integration')", +] diff --git a/unstract/connectors/tests/databases/test_bigquery_db.py b/unstract/connectors/tests/databases/test_bigquery_db.py index 4518fa9a7e..695d28a337 100644 --- a/unstract/connectors/tests/databases/test_bigquery_db.py +++ b/unstract/connectors/tests/databases/test_bigquery_db.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch import google.api_core.exceptions - from unstract.connectors.databases.bigquery.bigquery import BigQuery from unstract.connectors.databases.exceptions import ( BigQueryForbiddenException, diff --git a/unstract/connectors/tests/databases/test_mariadb.py b/unstract/connectors/tests/databases/test_mariadb.py index e6f008af2a..d9847daa4b 100644 --- a/unstract/connectors/tests/databases/test_mariadb.py +++ b/unstract/connectors/tests/databases/test_mariadb.py @@ -4,7 +4,6 @@ from unittest.mock import Mock, patch import pymysql.err as MysqlError - from unstract.connectors.databases.mariadb.mariadb import MariaDB from unstract.connectors.exceptions import ConnectorError @@ -12,7 +11,6 @@ class TestMariaDB(unittest.TestCase): def setUp(self) -> None: """Set up test configuration from environment variables""" - # SSL enabled config for testing SSL scenarios self.mariadb_config_ssl_enabled = { "host": os.getenv("MARIADB_HOST", "localhost"), @@ -86,7 +84,7 @@ def test_authentication_error_handling(self, mock_connect: Any) -> None: error_message = str(context.exception) self.assertIn("Authentication failed", error_message) - self.assertIn("username, password and ssl-settings", error_message) + self.assertIn("username, password and SSL SETTINGS", error_message) self.assertIn("localhost:3306", error_message) self.assertIn("SSL enabled", error_message) diff --git a/unstract/connectors/tests/databases/test_mssql_db.py b/unstract/connectors/tests/databases/test_mssql_db.py deleted file mode 100644 index c78340af91..0000000000 --- a/unstract/connectors/tests/databases/test_mssql_db.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from unstract.connectors.databases.mssql.mssql import MSSQL - - -class TestMSSQL(unittest.TestCase): - def test_user_name_and_password(self): - mssql = MSSQL( - { - "user": "sa", - "password": "Ascon@123", - "server": "localhost", - "port": "1433", - "database": "testdb", - } - ) - query = "SELECT * FROM Employees" - cursor = mssql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_mysql_db.py b/unstract/connectors/tests/databases/test_mysql_db.py deleted file mode 100644 index aa47cd90cc..0000000000 --- a/unstract/connectors/tests/databases/test_mysql_db.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from unstract.connectors.databases.mysql.mysql import MySQL - - -class TestMySQLDB(unittest.TestCase): - def test_user_name_and_password(self): - mysql = MySQL( - { - "user": "visitran", - "password": "mysqlpass", - "host": "localhost", - "port": "3307", - "database": "sakila", - } - ) - query = "SELECT * FROM category" - cursor = mysql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_postgresql_db.py b/unstract/connectors/tests/databases/test_postgresql_db.py deleted file mode 100644 index 96fceddd59..0000000000 --- a/unstract/connectors/tests/databases/test_postgresql_db.py +++ /dev/null @@ -1,50 +0,0 @@ -import unittest - -from unstract.connectors.databases.postgresql.postgresql import PostgreSQL - - -class TestPostgreSqlDB(unittest.TestCase): - def test_user_name_and_password(self): - psql = PostgreSQL( - { - "user": "test", - "password": "ascon", - "host": "localhost", - "port": "5432", - "database": "test7", - "schema": "public", - } - ) - query = "SELECT * FROM account_user LIMIT 3" - cursor = psql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - def test_connection_url(self): - connection_url = ( - "postgres://iamali003:FeQhupi41INg@ep-crimson-wind-434055" - ".us-east-2.aws.neon.tech/neondb" - ) - psql = PostgreSQL( - { - "connection_url": connection_url, - } - ) - query = "SELECT * FROM users LIMIT 3" - cursor = psql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_redshift_db.py b/unstract/connectors/tests/databases/test_redshift_db.py deleted file mode 100644 index 0b1300bfab..0000000000 --- a/unstract/connectors/tests/databases/test_redshift_db.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest - -from unstract.connectors.databases.redshift.redshift import Redshift - - -class TestRedshift(unittest.TestCase): - def test_user_name_and_password(self): - redshift = Redshift( - { - "user": "awsuser", - "password": "PASSWORD", - "host": "redshift-cluster-1.redshift.amazonaws.com", - "port": "5439", - "database": "dev", - } - ) - query = ( - "SELECT userid, username, firstname, lastname, city, state, email," - "phone, likesports, liketheatre, likeconcerts, likejazz," - "likeclassical, likeopera, likerock, likevegas, likebroadway," - "likemusicals FROM users limit 10" - ) - cursor = redshift.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_snowflake_db.py b/unstract/connectors/tests/databases/test_snowflake_db.py deleted file mode 100644 index a87bb32733..0000000000 --- a/unstract/connectors/tests/databases/test_snowflake_db.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest - -from unstract.connectors.databases.snowflake.snowflake import SnowflakeDB - - -class TestSnowflakeDB(unittest.TestCase): - def test_something(self): - sf = SnowflakeDB( - { - "user": "arun", - "password": "PASSWORD", - "account": "JX91721.ap-south-1", - "database": "RESUME_COLLECTION", - "schema": "PUBLIC", - "warehouse": "COMPUTE_WH", - "role": "", - } - ) - # engine = sf.get_engine() - # try: - # with engine.connect() as connection: - # md = sqlalchemy.MetaData() - # table = sqlalchemy.Table( - # 'RESUME', md, autoload=True, autoload_with=engine) - # columns = table.c - # for c in columns: - # print(c.name, c.type) - # # connection.execute("select current_version()") - # except Exception as e: - # print(e) - # - # engine.dispose() - - cursor = sf.get_engine().cursor() - results = cursor.execute("describe table RESUME") - for c in results: - print(c) - - self.assertIsNotNone(results) # add assertion here - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/filesystems/test_box_fs.py b/unstract/connectors/tests/filesystems/test_box_fs.py index cc062824bc..1aabdd8a81 100644 --- a/unstract/connectors/tests/filesystems/test_box_fs.py +++ b/unstract/connectors/tests/filesystems/test_box_fs.py @@ -1,17 +1,24 @@ import os import unittest +import pytest from unstract.connectors.filesystems.box import BoxFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestBoxFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("TEST_BOX_APP_SETTINGS"), + "Integration test requires TEST_BOX_APP_SETTINGS", + ) def test_basic(self): box_app_settings = os.environ.get("TEST_BOX_APP_SETTINGS") box_fs = BoxFS(settings={"box_app_settings": box_app_settings}) file_path = "/" try: files = box_fs.get_fsspec_fs().ls(file_path) - print(files) self.assertIsNotNone(files) except Exception as e: self.fail(f"TestBoxFS.test_basic failed: {e}") diff --git a/unstract/connectors/tests/filesystems/test_google_drive_fs.py b/unstract/connectors/tests/filesystems/test_google_drive_fs.py index 26b1ac59a8..b3e1b5a3fa 100644 --- a/unstract/connectors/tests/filesystems/test_google_drive_fs.py +++ b/unstract/connectors/tests/filesystems/test_google_drive_fs.py @@ -1,9 +1,19 @@ +import os import unittest +import pytest from unstract.connectors.filesystems.google_drive.google_drive import GoogleDriveFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestGoogleDriveFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("GDRIVE_GOOGLE_SERVICE_ACCOUNT") + and os.environ.get("GDRIVE_GOOGLE_PROJECT_ID"), + "Integration test requires GDRIVE_GOOGLE_SERVICE_ACCOUNT and GDRIVE_GOOGLE_PROJECT_ID", + ) def test_basic(self): self.assertEqual(GoogleDriveFS.requires_oauth(), True) drive = GoogleDriveFS( @@ -14,7 +24,7 @@ def test_basic(self): } ) - print(drive.get_fsspec_fs().ls("")) + self.assertIsNotNone(drive.get_fsspec_fs().ls("")) if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_http_fs.py b/unstract/connectors/tests/filesystems/test_http_fs.py index 4548d99543..0fb0f27940 100644 --- a/unstract/connectors/tests/filesystems/test_http_fs.py +++ b/unstract/connectors/tests/filesystems/test_http_fs.py @@ -1,24 +1,28 @@ +import os import unittest +import pytest from unstract.connectors.filesystems.http.http import HttpFS +# Live-HTTP test — integration tier only, so `unit-connectors` (-m "not +# integration") never selects it even when HTTP_FS_TEST_URL is set. +pytestmark = pytest.mark.integration + class TestHttpFS(unittest.TestCase): - # Run a local HTTP server with - # `python -m http.server -b localhost 8080` + # Needs a reachable HTTP server. Start one locally, e.g. + # python -m http.server -b localhost 8080 + # then run with HTTP_FS_TEST_URL=http://localhost:8080/. Skip-guarded so it + # never hits a hard-coded live URL during a plain unit run. + @unittest.skipUnless( + os.environ.get("HTTP_FS_TEST_URL"), + "Integration test requires a reachable HTTP server via HTTP_FS_TEST_URL", + ) def test_basic(self): self.assertEqual(HttpFS.can_write(), False) - # Assuming that the server is run locally - # url = "http://localhost:8080/" - url = "https://filesystem-spec.readthedocs.io/" - http_fs = HttpFS(settings={"base_url": url}) - file_path = "/" - try: - # print(http_fs.get_fsspec_fs().ls(file_path)) - files = http_fs.get_fsspec_fs().ls(file_path) - self.assertIsNotNone(files) - except Exception as e: - self.fail(f"TestHttpFS.test_basic failed: {e}") + http_fs = HttpFS(settings={"base_url": os.environ["HTTP_FS_TEST_URL"]}) + files = http_fs.get_fsspec_fs().ls("/") + self.assertIsNotNone(files) if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.py index 9da9a1dcf0..9c8c4b6e25 100644 --- a/unstract/connectors/tests/filesystems/test_miniofs.py +++ b/unstract/connectors/tests/filesystems/test_miniofs.py @@ -3,10 +3,10 @@ import unittest from unittest.mock import AsyncMock, patch +import pytest from botocore.exceptions import ClientError from s3fs.core import S3FileSystem from s3fs.errors import translate_boto_error - from unstract.connectors.filesystems.minio.exceptions import s3_error_code from unstract.connectors.filesystems.minio.minio import ( MinioFS, @@ -16,38 +16,31 @@ class TestMinoFS(unittest.TestCase): - @unittest.skip("") - def test_s3(self) -> None: - self.assertEqual(MinioFS.requires_oauth(), False) - access_key = os.environ.get("AWS_ACCESS_KEY_ID") - secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - s3 = MinioFS( - { - "key": access_key, - "secret": secret_key, - "path": "/", - "endpoint_url": "https://s3.amazonaws.com", - } - ) - - print(s3.get_fsspec_fs().ls("unstract-user-storage")) - - # @unittest.skip("Minio is not running") + @pytest.mark.integration + @unittest.skipUnless( + os.environ.get("MINIO_ACCESS_KEY_ID") + and os.environ.get("MINIO_SECRET_ACCESS_KEY"), + "Integration test requires a live MinIO and MINIO_ACCESS_KEY_ID + MINIO_SECRET_ACCESS_KEY", + ) def test_minio(self) -> None: + # Endpoint from the rig's testcontainers MinIO via MINIO_ENDPOINT_URL; + # falls back to the local platform MinIO for manual runs. self.assertEqual(MinioFS.requires_oauth(), False) - access_key = os.environ.get("MINIO_ACCESS_KEY_ID") - secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY") - print(access_key, secret_key) - s3 = MinioFS( + fs = MinioFS( { - "key": access_key, - "secret": secret_key, - "endpoint_url": "http://localhost:9000", - "path": "/minio-test", + "key": os.environ["MINIO_ACCESS_KEY_ID"], + "secret": os.environ["MINIO_SECRET_ACCESS_KEY"], + "endpoint_url": os.environ.get( + "MINIO_ENDPOINT_URL", "http://localhost:9000" + ), + "path": "/", } - ) - - print(s3.get_fsspec_fs().ls("/minio-test")) + ).get_fsspec_fs() + bucket = "rig-minio-test" + if not fs.exists(bucket): + fs.mkdir(bucket) + listed = [b.rstrip("/").split("/")[-1] for b in fs.ls("")] + self.assertIn(bucket, listed) def _translated_error(code: str) -> BaseException: diff --git a/unstract/connectors/tests/filesystems/test_pcs.py b/unstract/connectors/tests/filesystems/test_pcs.py index fa3f49c432..580f08a55d 100644 --- a/unstract/connectors/tests/filesystems/test_pcs.py +++ b/unstract/connectors/tests/filesystems/test_pcs.py @@ -1,10 +1,19 @@ import os import unittest +import pytest from unstract.connectors.filesystems.ucs import UnstractCloudStorage +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestPCS_FS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID") + and os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY"), + "Integration test requires GOOGLE_STORAGE_ACCESS_KEY_ID and GOOGLE_STORAGE_SECRET_ACCESS_KEY", + ) def test_pcs(self) -> None: self.assertEqual(UnstractCloudStorage.requires_oauth(), False) access_key = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID") @@ -18,7 +27,7 @@ def test_pcs(self) -> None: } ) - print(gcs.get_fsspec_fs().ls("unstract-user-storage")) # type:ignore + self.assertIsNotNone(gcs.get_fsspec_fs().ls("unstract-user-storage")) # type:ignore if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_sharepoint_fs.py b/unstract/connectors/tests/filesystems/test_sharepoint_fs.py index d976e73a99..6827ce959a 100644 --- a/unstract/connectors/tests/filesystems/test_sharepoint_fs.py +++ b/unstract/connectors/tests/filesystems/test_sharepoint_fs.py @@ -5,6 +5,8 @@ import unittest from datetime import datetime, timezone +import pytest + logger = logging.getLogger(__name__) @@ -115,14 +117,6 @@ def test_connector_initialization_missing_auth(self): SharePointFS(settings=invalid_settings) self.assertIn("requires authentication", str(context.exception)) - def test_json_schema_has_is_personal(self): - """Test that JSON schema includes is_personal field.""" - from unstract.connectors.filesystems.sharepoint import SharePointFS - - schema = SharePointFS.get_json_schema() - self.assertIn("is_personal", schema) - self.assertIn("Personal Account", schema) - def test_json_schema_has_oneof_pattern(self): """Test that JSON schema uses dependencies/oneOf pattern for dual auth methods.""" import json @@ -243,6 +237,7 @@ def test_get_connector_root_dir(self): self.assertEqual(result, "") +@pytest.mark.integration class TestSharePointFSIntegration(unittest.TestCase): """Integration tests for SharePointFS (require real credentials).""" diff --git a/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py b/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py index df35208d7e..a72bc915e3 100644 --- a/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py +++ b/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py @@ -1,10 +1,18 @@ import os import unittest +import pytest from unstract.connectors.filesystems.zs_dropbox import DropboxFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestDropboxFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("TEST_DROPBOX_ACCESS_TOKEN"), + "Integration test requires TEST_DROPBOX_ACCESS_TOKEN", + ) def test_access_token(self): access_token = os.environ.get("TEST_DROPBOX_ACCESS_TOKEN") settings = {"token": access_token} @@ -12,9 +20,7 @@ def test_access_token(self): # Leave empty for root file_path = "" try: - # print(dropbox_fs.get_fsspec_fs().ls(file_path)) files = dropbox_fs.get_fsspec_fs().ls(file_path) - print(files) self.assertIsNotNone(files) except Exception as e: self.fail(f"TestDropboxFS.test_access_token failed: {e}") diff --git a/unstract/connectors/uv.lock b/unstract/connectors/uv.lock index 23da1b837c..f7a8f7725f 100644 --- a/unstract/connectors/uv.lock +++ b/unstract/connectors/uv.lock @@ -1237,7 +1237,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1253,9 +1253,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -2968,7 +2968,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/unstract/core/tests/account_services/test_pandora_account.py b/unstract/core/tests/account_services/test_pandora_account.py deleted file mode 100644 index 8e74fe3bc4..0000000000 --- a/unstract/core/tests/account_services/test_pandora_account.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from unstract.core.account_services.unstract_account import UnstractAccount - - -class TestUnstractAccount(unittest.TestCase): - def test_provision_blob(self): - account = UnstractAccount("acme", "johndoe") - account.provision_s3_storage() - account.upload_sample_files() - self.assertEqual(True, True) # add assertion here - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/core/tests/test_pubsub_helper.py b/unstract/core/tests/test_pubsub_helper.py deleted file mode 100644 index ae3e102690..0000000000 --- a/unstract/core/tests/test_pubsub_helper.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -from unstract.core.pubsub_helper import LogHelper as Log - - -class PubSubHelperTestCase(unittest.TestCase): - def test_pubsub(self): - ps1 = Log.publish( - project_guid="test", - message=Log.log(stage="COMPILE", message="Compile process started"), - ) - ps2 = Log.publish( - project_guid="test", - message=Log.log(level="ERROR", stage="COMPILE", message="Compile failed"), - ) - self.assertEqual(ps1, True) - self.assertEqual(ps2, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/filesystem/uv.lock b/unstract/filesystem/uv.lock index e8d4525511..2a8cf37718 100644 --- a/unstract/filesystem/uv.lock +++ b/unstract/filesystem/uv.lock @@ -671,7 +671,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -687,9 +687,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -1866,7 +1866,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/unstract/sdk1/pyproject.toml b/unstract/sdk1/pyproject.toml index 950294ea81..70b492f191 100644 --- a/unstract/sdk1/pyproject.toml +++ b/unstract/sdk1/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "python-dotenv==1.2.2", # # Adapter changes "tiktoken~=0.12.0", - "litellm==1.85.1", + "litellm==1.90.3", "llama-index>=0.14.13", "llama-index-vector-stores-postgres>=0.7.3", "llama-index-vector-stores-milvus>=0.9.6", @@ -172,7 +172,7 @@ keep-dict-typing = true [tool.pytest.ini_options] -python_files = ["tests.py", "test_*.py", "*_tests.py"] +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] testpaths = ["tests"] asyncio_mode = "strict" markers = [ diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py index ebd93e3dbd..3c9aee1d60 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py @@ -18,32 +18,47 @@ logger = logging.getLogger(__name__) # Anthropic models that have deprecated sampling parameters (`temperature`, -# `top_p`, `top_k`). The patterns are regex-searched against the model id -# after lowercasing and normalizing `.` / `_` to `-`. The match is anchored at -# the trailing edge so that unrelated future ids (`claude-opus-4-70`, -# `claude-opus-4-75`, `claude-opus-4-7verbose`) do not match. A single entry -# covers every encoding of the id we have observed: -# - Native Anthropic `claude-opus-4-7`, `anthropic/claude-opus-4-7` -# - Bedrock foundation model `anthropic.claude-opus-4-7--v1:0` -# - Bedrock cross-region profile `us.anthropic.claude-opus-4-7-...`, +# `top_p`, `top_k`). Anthropic removed the sampling params starting with Claude +# Opus 4.7, and every model released since — Opus 4.8, Sonnet 5, Fable 5, +# Mythos 5 — rejects them with a 400 (directly, and through the Bedrock / Azure +# AI Foundry / Vertex proxies). Opus 4.6 / Sonnet 4.6 and older still accept +# them, so detection is a narrow allowlist rather than a broad `claude-*` match. +# +# Each stem is compiled into a pattern that is regex-searched against the model +# id after lowercasing and normalizing `.` / `_` to `-`. The match is anchored +# at the trailing edge so unrelated future ids (`claude-sonnet-50`, +# `claude-opus-4-70`, `claude-sonnet-5verbose`) do not match. One stem covers +# every encoding of the id we have observed, e.g. for `claude-sonnet-5`: +# - Native Anthropic `claude-sonnet-5`, `anthropic/claude-sonnet-5` +# - Bedrock foundation model `anthropic.claude-sonnet-5--v1:0` +# - Bedrock cross-region profile `us.anthropic.claude-sonnet-5-...`, # `eu.`, `apac.`, `global.` variants # - Bedrock foundation-model ARN `arn:aws:bedrock:::foundation-model/ -# anthropic.claude-opus-4-7-...` +# anthropic.claude-sonnet-5-...` # - Bedrock inference-profile ARN `arn:aws:bedrock::: -# inference-profile/us.anthropic.claude-opus-4-7-...` -# - Vertex AI `vertex_ai/claude-opus-4-7@` -# - Azure AI Foundry deployments whose name embeds `claude-opus-4-7` +# inference-profile/us.anthropic.claude-sonnet-5-...` +# - Vertex AI `vertex_ai/claude-sonnet-5@` +# - Azure AI Foundry deployments whose name embeds `claude-sonnet-5` # Leading text (route prefixes like `converse/`, `invoke/`, `bedrock/`) passes # through because the regex is anchored only at the trailing edge. -# Add new entries here when Anthropic deprecates sampling on more models. +# Keep this list current — add a stem here when Anthropic deprecates sampling on +# a new model. # Trailing anchor allows: end-of-string, or one of `-`/`:`/`@`/`/` (the # delimiters used in date suffixes, ARN paths, Vertex `@`, and the # `v1:0` tag), or `v` followed by a digit (the version-tag start). A bare -# `v` is intentionally rejected so alpha continuations like `4-7verbose` do -# not silently match. -# See https://docs.claude.com/en/about-claude/models/whats-new-claude-4-7 -_SAMPLING_DEPRECATED_MODEL_PATTERNS: tuple[re.Pattern[str], ...] = ( - re.compile(r"claude-opus-4-7(?=$|[-:@/]|v\d)"), +# `v` is intentionally rejected so alpha continuations like `sonnet-5verbose` +# do not silently match. +# See https://docs.claude.com/en/about-claude/models/overview +_SAMPLING_DEPRECATED_MODEL_STEMS: tuple[str, ...] = ( + "claude-opus-4-7", + "claude-opus-4-8", + "claude-sonnet-5", + "claude-fable-5", + "claude-mythos-5", +) +_SAMPLING_DEPRECATED_MODEL_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(rf"{re.escape(stem)}(?=$|[-:@/]|v\d)") + for stem in _SAMPLING_DEPRECATED_MODEL_STEMS ) _DEPRECATED_SAMPLING_PARAMS: tuple[str, ...] = ("temperature", "top_p", "top_k") # Fields whose value can carry a model id. `model` is universal; `model_id` is @@ -63,7 +78,7 @@ def _looks_like_opaque_aip_arn(value: str | None) -> bool: Bedrock AIP ARNs do not carry the underlying foundation-model id in the string, so the sampling-strip detector cannot decide whether the call is - bound for Claude Opus 4.7. + bound for a sampling-deprecated Claude model (Opus 4.7 and later). """ return bool(value) and _OPAQUE_AIP_ARN_MARKER in value @@ -72,12 +87,14 @@ def _has_deprecated_sampling_params(model: str | None) -> bool: """Return True when the model rejects sampling parameters. Anthropic deprecated `temperature`, `top_p`, and `top_k` starting with - Claude Opus 4.7; sending any of them yields a 400 from Anthropic and from - the providers that proxy it (Bedrock, Azure AI Foundry, Vertex AI). + Claude Opus 4.7, and every model released since (Opus 4.8, Sonnet 5, + Fable 5, Mythos 5) rejects them too; sending any of them yields a 400 from + Anthropic and from the providers that proxy it (Bedrock, Azure AI Foundry, + Vertex AI). See `_SAMPLING_DEPRECATED_MODEL_STEMS` for the covered set. The check normalizes case and `.`/`_` separators to `-`, then regex- searches against the patterns with a trailing-edge boundary, so - `claude-opus-4-70` and `claude-opus-4-7verbose` do not match. This + `claude-sonnet-50` and `claude-sonnet-5verbose` do not match. This catches every format that embeds the model id (foundation model ids, cross-region profiles, foundation-model ARNs, inference-profile ARNs, Vertex `@`-suffixed ids). @@ -89,7 +106,8 @@ def _has_deprecated_sampling_params(model: str | None) -> bool: from the string. Pass the AIP ARN in `model_id` and keep the standard model id in `model`, or the strip won't fire. - Azure AI Foundry deployment names that omit the model id; rename the - deployment to include `claude-opus-4-7` so detection works. + deployment to include the model id (e.g. `claude-sonnet-5`) so detection + works. """ if not model: return False @@ -1641,3 +1659,30 @@ def validate_model(adapter_metadata: dict[str, "Any"]) -> str: if not model.startswith("gemini/"): model = f"gemini/{model}" return model + + +class MistralEmbeddingParameters(BaseEmbeddingParameters): + """See https://docs.litellm.ai/docs/providers/mistral.""" + + api_key: str + embed_batch_size: int | None = None + + @staticmethod + def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]: + metadata_copy = {**adapter_metadata} + metadata_copy["model"] = MistralEmbeddingParameters.validate_model(metadata_copy) + + return MistralEmbeddingParameters(**metadata_copy).model_dump() + + @staticmethod + def validate_model(adapter_metadata: dict[str, "Any"]) -> str: + raw_model = adapter_metadata.get("model") + model = raw_model.strip() if isinstance(raw_model, str) else "" + if not model: + raise ValueError( + "The 'model' field is required for the Mistral embedding adapter. " + "Example: 'mistral-embed'" + ) + if not model.startswith("mistral/"): + model = f"mistral/{model}" + return model diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/__init__.py b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/__init__.py index 18f240f04e..3be59f61c3 100644 --- a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/__init__.py +++ b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/__init__.py @@ -4,6 +4,7 @@ from unstract.sdk1.adapters.embedding1.azure_openai import AzureOpenAIEmbeddingAdapter from unstract.sdk1.adapters.embedding1.bedrock import AWSBedrockEmbeddingAdapter from unstract.sdk1.adapters.embedding1.gemini import GeminiEmbeddingAdapter +from unstract.sdk1.adapters.embedding1.mistral import MistralEmbeddingAdapter from unstract.sdk1.adapters.embedding1.nvidia_build import NvidiaBuildEmbeddingAdapter from unstract.sdk1.adapters.embedding1.ollama import OllamaEmbeddingAdapter from unstract.sdk1.adapters.embedding1.openai import OpenAIEmbeddingAdapter @@ -22,6 +23,7 @@ "AzureOpenAIEmbeddingAdapter", "AWSBedrockEmbeddingAdapter", "GeminiEmbeddingAdapter", + "MistralEmbeddingAdapter", "NvidiaBuildEmbeddingAdapter", "OpenAIEmbeddingAdapter", "OpenAICompatibleEmbeddingAdapter", diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/mistral.py b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/mistral.py new file mode 100644 index 0000000000..3440c1f498 --- /dev/null +++ b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/mistral.py @@ -0,0 +1,40 @@ +from typing import Any + +from unstract.sdk1.adapters.base1 import BaseAdapter, MistralEmbeddingParameters +from unstract.sdk1.adapters.enums import AdapterTypes + + +class MistralEmbeddingAdapter(MistralEmbeddingParameters, BaseAdapter): + @staticmethod + def get_id() -> str: + return "mistral|28a9d784-bc33-46a4-804a-436058e1354f" + + @staticmethod + def get_metadata() -> dict[str, Any]: + return { + "name": "Mistral", + "version": "1.0.0", + "adapter": MistralEmbeddingAdapter, + "description": "Mistral embedding adapter", + "is_active": True, + } + + @staticmethod + def get_name() -> str: + return "Mistral" + + @staticmethod + def get_description() -> str: + return "Mistral embedding adapter" + + @staticmethod + def get_provider() -> str: + return "mistral" + + @staticmethod + def get_icon() -> str: + return "/icons/adapter-icons/Mistral%20AI.png" + + @staticmethod + def get_adapter_type() -> AdapterTypes: + return AdapterTypes.EMBEDDING diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/mistral.json b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/mistral.json new file mode 100644 index 0000000000..9fab2fac89 --- /dev/null +++ b/unstract/sdk1/src/unstract/sdk1/adapters/embedding1/static/mistral.json @@ -0,0 +1,38 @@ +{ + "title": "Mistral Embedding", + "type": "object", + "required": [ + "adapter_name", + "api_key", + "model" + ], + "properties": { + "adapter_name": { + "type": "string", + "title": "Name", + "default": "", + "description": "Provide a unique name for this adapter instance. Example: mistral-emb-1" + }, + "model": { + "type": "string", + "title": "Model", + "default": "mistral-embed", + "description": "Provide the name of the model. The mistral/ prefix will be added automatically if omitted. Recommended: mistral-embed." + }, + "api_key": { + "type": "string", + "title": "API Key", + "default": "", + "format": "password", + "description": "API Key for Mistral embeddings" + }, + "timeout": { + "type": "number", + "minimum": 0, + "multipleOf": 1, + "title": "Timeout", + "default": 240, + "description": "Timeout in seconds" + } + } +} diff --git a/unstract/sdk1/src/unstract/sdk1/patches/litellm_cohere_timeout.py b/unstract/sdk1/src/unstract/sdk1/patches/litellm_cohere_timeout.py index bcbfc97dc3..590445bb8e 100644 --- a/unstract/sdk1/src/unstract/sdk1/patches/litellm_cohere_timeout.py +++ b/unstract/sdk1/src/unstract/sdk1/patches/litellm_cohere_timeout.py @@ -29,10 +29,10 @@ # Only apply the patch on the exact litellm version it was written for. # Any other version (newer or older) skips the patch with a visible # warning so engineers know to verify compatibility. -# Verified against litellm 1.85.1: async_embedding() at -# litellm/llms/cohere/embed/handler.py still does not forward `timeout` -# to client.post() when a pre-constructed client is passed in (the bug). -_PATCHED_LITELLM_VERSION = "1.85.1" +# Verified against litellm 1.90.3: async_embedding() and embedding() at +# litellm/llms/cohere/embed/handler.py still do not forward `timeout` +# to client.post() (the bug); the copied bodies below still match upstream. +_PATCHED_LITELLM_VERSION = "1.90.3" _litellm_version = importlib.metadata.version("litellm") _SKIP_PATCH = Version(_litellm_version) != Version(_PATCHED_LITELLM_VERSION) if _SKIP_PATCH: diff --git a/unstract/sdk1/src/unstract/sdk1/prompt.py b/unstract/sdk1/src/unstract/sdk1/prompt.py deleted file mode 100644 index c648506209..0000000000 --- a/unstract/sdk1/src/unstract/sdk1/prompt.py +++ /dev/null @@ -1,260 +0,0 @@ -import functools -import logging -from collections.abc import Callable -from typing import Any, ParamSpec, TypeVar - -import requests -from requests import ConnectionError, RequestException, Response -from unstract.sdk1.constants import MimeType, RequestHeader, ToolEnv -from unstract.sdk1.platform import PlatformHelper -from unstract.sdk1.tool.base import BaseTool -from unstract.sdk1.utils.common import log_elapsed -from unstract.sdk1.utils.retry_utils import retry_prompt_service_call - -logger = logging.getLogger(__name__) - -P = ParamSpec("P") -R = TypeVar("R") - - -def handle_service_exceptions(context: str) -> Callable[[Callable[P, R]], Callable[P, R]]: - """Decorator to handle exceptions in PromptTool service calls. - - Args: - context (str): Context string describing where the error occurred - Returns: - Callable: Decorated function that handles service exceptions - """ - - def decorator(func: Callable[P, R]) -> Callable[P, R]: - @functools.wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - try: - return func(*args, **kwargs) - except ConnectionError as e: - msg = f"Error while {context}. Unable to connect to prompt service." - logger.error(f"{msg}\n{e}") - args[0].tool.stream_error_and_exit(msg, e) - return None - except RequestException as e: - error_message = str(e) - response = getattr(e, "response", None) - if response is not None: - if ( - MimeType.JSON in response.headers.get("Content-Type", "").lower() - and "error" in response.json() - ): - error_message = response.json()["error"] - elif response.text: - error_message = response.text - msg = f"Error while {context}. {error_message}" - args[0].tool.stream_error_and_exit(msg, e) - return None - except Exception as e: - # Handle any other unexpected exceptions - msg = f"Error while {context}. An unexpected error occurred" - logger.error(f"{msg}: {type(e).__name__}: {str(e)}", exc_info=True) - args[0].tool.stream_error_and_exit(msg, e) - return None - - return wrapper - - return decorator - - -class PromptTool: - """Class to handle prompt service methods for Unstract Tools.""" - - def __init__( - self, - tool: BaseTool, - prompt_host: str, - prompt_port: str, - is_public_call: bool = False, - request_id: str | None = None, - ) -> None: - """Class to interact with prompt-service. - - Args: - tool (AbstractTool): Instance of AbstractTool - prompt_host (str): Host of platform service - prompt_port (str): Port of platform service - is_public_call (bool): Whether the call is public. Defaults to False - """ - self.tool = tool - self.base_url = PlatformHelper.get_platform_base_url(prompt_host, prompt_port) - self.is_public_call = is_public_call - self.request_id = request_id - if not is_public_call: - self.bearer_token = tool.get_env_or_die(ToolEnv.PLATFORM_API_KEY) - - @log_elapsed(operation="ANSWER_PROMPTS") - @handle_service_exceptions("answering prompt(s)") - def answer_prompt( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> dict[str, Any]: - url_path = "answer-prompt" - if self.is_public_call: - url_path = "answer-prompt-public" - return self._call_service( - url_path=url_path, payload=payload, params=params, headers=headers - ) - - @log_elapsed(operation="INDEX") - @handle_service_exceptions("indexing") - def index( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> str: - url_path = "index" - if self.is_public_call: - url_path = "index-public" - prompt_service_response = self._call_service( - url_path=url_path, - payload=payload, - params=params, - headers=headers, - ) - return prompt_service_response.get("doc_id") - - @log_elapsed(operation="EXTRACT") - @handle_service_exceptions("extracting") - def extract( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> dict[str, Any]: - url_path = "extract" - if self.is_public_call: - url_path = "extract-public" - prompt_service_response = self._call_service( - url_path=url_path, - payload=payload, - params=params, - headers=headers, - ) - return prompt_service_response.get("extracted_text") - - @log_elapsed(operation="SINGLE_PASS_EXTRACTION") - @handle_service_exceptions("single pass extraction") - def single_pass_extraction( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> dict[str, Any]: - return self._call_service( - url_path="single-pass-extraction", - payload=payload, - params=params, - headers=headers, - ) - - @log_elapsed(operation="SUMMARIZATION") - @handle_service_exceptions("summarizing") - def summarize( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> dict[str, Any]: - return self._call_service( - url_path="summarize", - payload=payload, - params=params, - headers=headers, - ) - - @log_elapsed(operation="AGENTIC_EXTRACTION") - @handle_service_exceptions("executing agentic extraction") - def agentic_extraction( - self, - payload: dict[str, Any], - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - ) -> dict[str, Any]: - """Execute agentic extraction via prompt service. - - Args: - payload: Extraction payload containing project_id, json_schema, - prompt_text, adapter IDs, etc. - params: Optional query parameters - headers: Optional headers - - Returns: - dict: Extraction results with data, metadata, and metrics - """ - return self._call_service( - url_path="agentic/extract", - payload=payload, - params=params, - headers=headers, - ) - - def _get_headers(self, headers: dict[str, str] | None = None) -> dict[str, str]: - """Get default headers for requests. - - Returns: - dict[str, str]: Default headers including request ID and authorization - """ - request_headers = {RequestHeader.REQUEST_ID: self.request_id} - if self.is_public_call: - return request_headers - request_headers.update( - {RequestHeader.AUTHORIZATION: f"Bearer {self.bearer_token}"} - ) - - if headers: - request_headers.update(headers) - return request_headers - - @retry_prompt_service_call - def _call_service( - self, - url_path: str, - payload: dict[str, Any] | None = None, - params: dict[str, str] | None = None, - headers: dict[str, str] | None = None, - method: str = "POST", - ) -> dict[str, Any]: - """Communicates to prompt service to fetch response for the prompt. - - Only POST calls are made to prompt-service though functionality exists. - This method automatically retries on connection errors with exponential backoff. - - Retry behavior is configurable via environment variables: - - PROMPT_SERVICE_MAX_RETRIES (default: 3) - - PROMPT_SERVICE_BASE_DELAY (default: 1.0s) - - PROMPT_SERVICE_MULTIPLIER (default: 2.0) - - PROMPT_SERVICE_JITTER (default: true) - - Args: - url_path (str): URL path to the service endpoint - payload (dict, optional): Payload to send in the request body - params (dict, optional): Query parameters to include in the request - headers (dict, optional): Headers to include in the request - method (str): HTTP method to use for the request (GET or POST) - - Returns: - dict: Response from the prompt service - """ - url: str = f"{self.base_url}/{url_path}" - req_headers = self._get_headers(headers) - response: Response = Response() - if method.upper() == "POST": - response = requests.post( - url=url, json=payload, params=params, headers=req_headers - ) - elif method.upper() == "GET": - response = requests.get(url=url, params=params, headers=req_headers) - else: - raise ValueError(f"Unsupported HTTP method: {method}") - - response.raise_for_status() - return response.json() diff --git a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py index 0a13331474..1f04026bba 100644 --- a/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py +++ b/unstract/sdk1/src/unstract/sdk1/utils/retry_utils.py @@ -277,7 +277,7 @@ def iter_with_retry[T]( def is_retryable_error(error: Exception) -> bool: """Check if a requests-library HTTP error should trigger a retry. - For retrying internal service calls (platform-service, prompt-service) that + For retrying internal service calls (platform-service) that use the requests library. Distinct from is_retryable_litellm_error() which handles litellm/openai/httpx exceptions with different class hierarchies (e.g. error.status_code vs error.response.status_code). @@ -516,10 +516,3 @@ def create_retry_decorator( # - PLATFORM_SERVICE_MULTIPLIER (default: 2.0) # - PLATFORM_SERVICE_JITTER (default: true) retry_platform_service_call = create_retry_decorator("PLATFORM_SERVICE") - -# Retry configured through below envs. -# - PROMPT_SERVICE_MAX_RETRIES (default: 3) -# - PROMPT_SERVICE_BASE_DELAY (default: 1.0s) -# - PROMPT_SERVICE_MULTIPLIER (default: 2.0) -# - PROMPT_SERVICE_JITTER (default: true) -retry_prompt_service_call = create_retry_decorator("PROMPT_SERVICE") diff --git a/unstract/sdk1/tests/conftest.py b/unstract/sdk1/tests/conftest.py index b3858b7e2f..f01ce56e5e 100644 --- a/unstract/sdk1/tests/conftest.py +++ b/unstract/sdk1/tests/conftest.py @@ -25,10 +25,6 @@ def clean_env(monkeypatch: MonkeyPatch) -> MonkeyPatch: "PLATFORM_SERVICE_BASE_DELAY", "PLATFORM_SERVICE_MULTIPLIER", "PLATFORM_SERVICE_JITTER", - "PROMPT_SERVICE_MAX_RETRIES", - "PROMPT_SERVICE_BASE_DELAY", - "PROMPT_SERVICE_MULTIPLIER", - "PROMPT_SERVICE_JITTER", ] for var in env_vars: monkeypatch.delenv(var, raising=False) diff --git a/unstract/sdk1/tests/test_mistral_embedding.py b/unstract/sdk1/tests/test_mistral_embedding.py new file mode 100644 index 0000000000..6569d0ec78 --- /dev/null +++ b/unstract/sdk1/tests/test_mistral_embedding.py @@ -0,0 +1,139 @@ +import json + +import pytest +from unstract.sdk1.adapters.embedding1.mistral import MistralEmbeddingAdapter +from unstract.sdk1.adapters.enums import AdapterTypes + + +class TestMistralEmbeddingAdapter: + def test_adapter_registration(self) -> None: + from unstract.sdk1.adapters.embedding1 import adapters + + mistral_ids = [k for k in adapters if "mistral" in k.lower()] + assert len(mistral_ids) == 1 + + def test_get_id_format(self) -> None: + adapter_id = MistralEmbeddingAdapter.get_id() + assert adapter_id.startswith("mistral|") + # Standard UUID-4 with hyphens is 36 characters + uuid_part = adapter_id.split("|")[1] + assert len(uuid_part) == 36 + + def test_get_adapter_type(self) -> None: + assert MistralEmbeddingAdapter.get_adapter_type() == AdapterTypes.EMBEDDING + + def test_get_name(self) -> None: + assert MistralEmbeddingAdapter.get_name() == "Mistral" + + def test_get_provider(self) -> None: + assert MistralEmbeddingAdapter.get_provider() == "mistral" + + def test_json_schema_loads(self) -> None: + schema = json.loads(MistralEmbeddingAdapter.get_json_schema()) + assert isinstance(schema, dict) + assert "title" in schema + assert "properties" in schema + assert schema["title"] == "Mistral Embedding" + + def test_json_schema_required_fields(self) -> None: + schema = json.loads(MistralEmbeddingAdapter.get_json_schema()) + assert set(schema["required"]) == {"adapter_name", "api_key", "model"} + + def test_json_schema_omits_batch_size(self) -> None: + # embed_batch_size is an inert client-side hint and is not exposed. + schema = json.loads(MistralEmbeddingAdapter.get_json_schema()) + assert "embed_batch_size" not in schema["properties"] + + def test_json_schema_api_key_password_format(self) -> None: + schema = json.loads(MistralEmbeddingAdapter.get_json_schema()) + assert schema["properties"]["api_key"]["format"] == "password" + + def test_json_schema_model_default(self) -> None: + schema = json.loads(MistralEmbeddingAdapter.get_json_schema()) + assert schema["properties"]["model"]["default"] == "mistral-embed" + + def test_validate_model_adds_prefix(self) -> None: + meta = {"model": "mistral-embed", "api_key": "test"} + result = MistralEmbeddingAdapter.validate_model(meta) + assert result == "mistral/mistral-embed" + + def test_validate_model_idempotent(self) -> None: + meta = {"model": "mistral/mistral-embed", "api_key": "test"} + result = MistralEmbeddingAdapter.validate_model(meta) + assert result == "mistral/mistral-embed" + + def test_validate_model_does_not_mutate_input(self) -> None: + meta = {"model": "mistral-embed", "api_key": "test"} + MistralEmbeddingAdapter.validate_model(meta) + assert meta["model"] == "mistral-embed" + + def test_validate_does_not_mutate_input(self) -> None: + meta = {"model": "mistral-embed", "api_key": "test-key"} + original_model = meta["model"] + MistralEmbeddingAdapter.validate(meta) + assert meta["model"] == original_model + + def test_validate_model_empty_string_raises(self) -> None: + meta = {"model": "", "api_key": "test"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate_model(meta) + + def test_validate_model_whitespace_only_raises(self) -> None: + meta = {"model": " ", "api_key": "test"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate_model(meta) + + def test_validate_model_none_raises(self) -> None: + meta = {"model": None, "api_key": "test"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate_model(meta) + + def test_validate_model_missing_key_raises(self) -> None: + meta = {"api_key": "test"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate_model(meta) + + def test_validate_empty_model_raises(self) -> None: + meta = {"model": "", "api_key": "test-key"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate(meta) + + def test_validate_none_model_raises(self) -> None: + meta = {"model": None, "api_key": "test-key"} + with pytest.raises(ValueError, match="model.*required"): + MistralEmbeddingAdapter.validate(meta) + + def test_validate_missing_api_key_raises(self) -> None: + from pydantic import ValidationError + + meta = {"model": "mistral/mistral-embed"} + with pytest.raises(ValidationError): + MistralEmbeddingAdapter.validate(meta) + + def test_validate_calls_validate_model(self) -> None: + meta = {"model": "mistral-embed", "api_key": "test-key"} + validated = MistralEmbeddingAdapter.validate(meta) + assert validated["model"] == "mistral/mistral-embed" + + def test_validate_strips_extra_fields(self) -> None: + meta = { + "model": "mistral/mistral-embed", + "api_key": "test-key", + "adapter_name": "my-adapter", + "unknown_field": "should_be_dropped", + } + validated = MistralEmbeddingAdapter.validate(meta) + assert "adapter_name" not in validated + assert "unknown_field" not in validated + + def test_validate_includes_base_fields(self) -> None: + meta = {"model": "mistral/mistral-embed", "api_key": "test-key"} + validated = MistralEmbeddingAdapter.validate(meta) + assert "timeout" in validated + assert "max_retries" in validated + + def test_metadata(self) -> None: + metadata = MistralEmbeddingAdapter.get_metadata() + assert metadata["name"] == "Mistral" + assert metadata["is_active"] is True + assert metadata["adapter"] is MistralEmbeddingAdapter diff --git a/unstract/sdk1/tests/test_prompt.py b/unstract/sdk1/tests/test_prompt.py deleted file mode 100644 index 798db9106c..0000000000 --- a/unstract/sdk1/tests/test_prompt.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Integration tests for prompt module with retry logic.""" - -from typing import Any, Self -from unittest.mock import MagicMock, Mock, patch - -import pytest -from pytest import MonkeyPatch -from requests.exceptions import ConnectionError, Timeout -from unstract.sdk1.prompt import PromptTool - - -class TestPromptToolRetry: - """Tests for PromptTool retry functionality.""" - - @pytest.fixture - def mock_tool(self: Self) -> MagicMock: - """Create a mock tool for testing.""" - tool = MagicMock() - tool.get_env_or_die.side_effect = lambda key: { - "PLATFORM_API_KEY": "test-api-key", - }.get(key, "mock-value") - tool.stream_log = MagicMock() - tool.stream_error_and_exit = MagicMock() - return tool - - @pytest.fixture - def prompt_tool(self: Self, mock_tool: MagicMock) -> PromptTool: - """Create a PromptTool instance.""" - return PromptTool( - tool=mock_tool, - prompt_host="http://localhost", - prompt_port="3003", - is_public_call=False, - request_id="test-request-id", - ) - - def test_success_on_first_attempt( - self: Self, prompt_tool: PromptTool, clean_env: MonkeyPatch - ) -> None: - """Test successful service call on first attempt.""" - expected_response = {"result": "success"} - payload = {"prompt": "test"} - - with patch("requests.post") as mock_post: - mock_response = Mock() - mock_response.json.return_value = expected_response - mock_response.raise_for_status = Mock() - mock_post.return_value = mock_response - - result = prompt_tool._call_service("answer-prompt", payload=payload) - - assert result == expected_response - assert mock_post.call_count == 1 - - @pytest.mark.parametrize( - "error_type,error_instance", - [ - ("ConnectionError", ConnectionError("Connection failed")), - ("Timeout", Timeout("Request timed out")), - ], - ) - def test_retry_on_errors( - self: Self, - prompt_tool: PromptTool, - error_type: str, - error_instance: Exception, - clean_env: MonkeyPatch, - ) -> None: - """Test service retries on ConnectionError and Timeout.""" - expected_response = {"result": "success"} - payload = {"prompt": "test"} - - with patch("requests.post") as mock_post: - mock_response = Mock() - mock_response.json.return_value = expected_response - mock_response.raise_for_status = Mock() - - mock_post.side_effect = [ - error_instance, - mock_response, - ] - - result = prompt_tool._call_service("answer-prompt", payload=payload) - - assert result == expected_response - assert mock_post.call_count == 2 - - @pytest.mark.slow - def test_max_retries_exceeded( - self: Self, mock_tool: MagicMock, clean_env: MonkeyPatch - ) -> None: - """Test service call fails after exceeding max retries.""" - prompt_tool = PromptTool( - tool=mock_tool, - prompt_host="http://localhost", - prompt_port="3003", - is_public_call=False, - request_id="test-request-id", - ) - - payload = {"prompt": "test"} - - with patch("requests.post") as mock_post: - mock_post.side_effect = ConnectionError("Persistent failure") - - # Exception handled by decorator - with pytest.raises(ConnectionError): - prompt_tool._call_service("answer-prompt", payload=payload) - - # Default: 3 retries + 1 initial = 4 attempts - assert mock_post.call_count == 4 - - @pytest.mark.parametrize( - "method_name,payload", - [ - ("answer_prompt", {"prompts": ["test"]}), - ("index", {"document": "test"}), - ("extract", {"doc_id": "123"}), - ("summarize", {"text": "test"}), - ], - ) - def test_wrapper_methods_retry( - self: Self, - prompt_tool: PromptTool, - method_name: str, - payload: dict[str, Any], - clean_env: MonkeyPatch, - ) -> None: - """Test that wrapper methods inherit retry behavior.""" - expected_response = { - "answers": ["result"], - "doc_id": "doc-123", - "extracted_text": "text", - "summary": "summary", - } - - with patch("requests.post") as mock_post: - mock_response = Mock() - mock_response.json.return_value = expected_response - mock_response.raise_for_status = Mock() - - mock_post.side_effect = [ - ConnectionError("Transient failure"), - mock_response, - ] - - getattr(prompt_tool, method_name)(payload) - - assert mock_post.call_count == 2 - - @pytest.mark.slow - def test_error_handling_with_retry( - self: Self, mock_tool: MagicMock, clean_env: MonkeyPatch - ) -> None: - """Test error handling decorator works with retry.""" - prompt_tool = PromptTool( - tool=mock_tool, - prompt_host="http://localhost", - prompt_port="3003", - is_public_call=False, - request_id="test-request-id", - ) - - payload = {"prompt": "test"} - - with patch("requests.post") as mock_post: - mock_post.side_effect = ConnectionError("Persistent failure") - - # Error handler should catch after all retries - result = prompt_tool.answer_prompt(payload) - - # handle_service_exceptions decorator calls stream_error_and_exit - assert result is None - prompt_tool.tool.stream_error_and_exit.assert_called() diff --git a/unstract/sdk1/tests/test_sampling_strip.py b/unstract/sdk1/tests/test_sampling_strip.py index 3e3ce8b405..d6d792a3d7 100644 --- a/unstract/sdk1/tests/test_sampling_strip.py +++ b/unstract/sdk1/tests/test_sampling_strip.py @@ -1,4 +1,9 @@ -"""Tests for Claude Opus 4.7 sampling-parameter strip. +"""Tests for the Claude sampling-parameter strip. + +Covers Claude Opus 4.7 and every model released since (Opus 4.8, Sonnet 5, +Fable 5, Mythos 5), all of which reject `temperature`/`top_p`/`top_k`. Sonnet 5 +is the model behind the reported Azure AI Foundry `temperature is deprecated` +failure. Pins the detection regex and the four-adapter wiring against the failure modes that surfaced in PR #1934 review: @@ -68,6 +73,39 @@ def test_has_deprecated_sampling_params_positive(model: str) -> None: assert _has_deprecated_sampling_params(model) +# Every Claude model released after Opus 4.7 also rejects sampling params +# (Opus 4.8, Sonnet 5, Fable 5, Mythos 5). Sonnet 5 is the model behind the +# reported Azure AI Foundry `temperature is deprecated` failure. +POST_47_POSITIVES: list[str] = [ + # Sonnet 5 — native, Azure AI Foundry (prefixed by validate_model), case, + # deployment names embedding the id, separator normalization, version tag + "claude-sonnet-5", + "anthropic/claude-sonnet-5", + "azure_ai/claude-sonnet-5", + "azure_ai/claude-sonnet-5-prod", + "azure_ai/my-claude-sonnet-5-deployment", + "Claude-Sonnet-5", # case + "claude.sonnet.5", # dot separators + "claude_sonnet_5", # underscore separators + "claude-sonnet-5v1", # version tag + "anthropic.claude-sonnet-5-20260101-v1:0", # Bedrock foundation model id + "vertex_ai/claude-sonnet-5@20260101", # Vertex AI + # Opus 4.8 + "claude-opus-4-8", + "anthropic.claude-opus-4-8-20260101-v1:0", + "azure_ai/claude-opus-4-8", + # Fable 5 / Mythos 5 + "claude-fable-5", + "vertex_ai/claude-fable-5@20260101", + "claude-mythos-5", +] + + +@pytest.mark.parametrize("model", POST_47_POSITIVES) +def test_has_deprecated_sampling_params_positive_post_opus_47(model: str) -> None: + assert _has_deprecated_sampling_params(model) + + # ── detection: negatives ──────────────────────────────────────────────────── NEGATIVES: list[str | None] = [ @@ -77,6 +115,18 @@ def test_has_deprecated_sampling_params_positive(model: str) -> None: "claude-sonnet-4-7", "claude-haiku-4-5", "anthropic.claude-3-5-sonnet-20241022-v2:0", + # Sonnet families that still accept sampling params must NOT match the + # `claude-sonnet-5` stem + "claude-sonnet-4-5", + "claude-sonnet-4-6", + # Prefix collisions / alpha continuations for the new stems — lock the + # trailing-edge anchor the same way it is locked for opus-4-7 + "claude-sonnet-50", + "claude-sonnet-59", + "claude-sonnet-5verbose", + "claude-sonnet-5variant", + "claude-opus-4-80", + "claude-fable-50", # Non-Anthropic providers "gpt-4o", "gemini-2.0-flash", @@ -277,6 +327,35 @@ def test_vertex_validate_strips_temperature_for_opus_4_7() -> None: assert param not in result, f"vertex: {param} should be stripped" +@pytest.mark.parametrize( + "name,cls,extra", + ADAPTER_CASES, + ids=lambda v: v if isinstance(v, str) else "", +) +def test_validate_strips_temperature_for_sonnet_5( + name: str, cls: type, extra: dict[str, Any] +) -> None: + """Sonnet 5 rejects temperature just like Opus 4.7. + + The `azure_ai_foundry` case is the exact scenario reported: configuring + Claude Sonnet 5 on Azure AI Foundry 400'd with `temperature is deprecated`. + """ + model = { + "anthropic": "claude-sonnet-5", + "bedrock": "anthropic.claude-sonnet-5-20260101-v1:0", + "azure_ai_foundry": "claude-sonnet-5", + }[name] + result = cls.validate({"model": model, "temperature": 0.5, **extra}) + for param in _DEPRECATED_SAMPLING_PARAMS: + assert param not in result, f"{name}: {param} should be stripped" + + +def test_vertex_validate_strips_temperature_for_sonnet_5() -> None: + result = VertexAILLMParameters.validate(_vertex_metadata("claude-sonnet-5@20260101")) + for param in _DEPRECATED_SAMPLING_PARAMS: + assert param not in result, f"vertex: {param} should be stripped" + + @pytest.mark.parametrize( "name,cls,extra", ADAPTER_CASES, diff --git a/unstract/sdk1/tests/utils/test_retry_utils.py b/unstract/sdk1/tests/utils/test_retry_utils.py index cbf73c731a..227734414a 100644 --- a/unstract/sdk1/tests/utils/test_retry_utils.py +++ b/unstract/sdk1/tests/utils/test_retry_utils.py @@ -12,7 +12,6 @@ create_retry_decorator, is_retryable_error, retry_platform_service_call, - retry_prompt_service_call, retry_with_exponential_backoff, ) @@ -485,10 +484,6 @@ def test_retry_platform_service_call_exists(self) -> None: """Test that retry_platform_service_call decorator exists.""" assert retry_platform_service_call is not None - def test_retry_prompt_service_call_exists(self) -> None: - """Test that retry_prompt_service_call decorator exists.""" - assert retry_prompt_service_call is not None - def test_platform_service_decorator_retries_on_connection_error( self, clean_env: MonkeyPatch ) -> None: @@ -504,21 +499,6 @@ def test_platform_service_decorator_retries_on_connection_error( assert result == "success" assert mock_func.call_count == 2 - def test_prompt_service_decorator_retries_on_timeout( - self, clean_env: MonkeyPatch - ) -> None: - """Test prompt service decorator retries Timeout.""" - mock_func = Mock( - __name__="test_func", side_effect=[Timeout("Timed out"), "success"] - ) - - decorated_func = retry_prompt_service_call(mock_func) - - result = decorated_func() - - assert result == "success" - assert mock_func.call_count == 2 - class TestRetryLogging: """Tests for retry logging behavior.""" diff --git a/unstract/sdk1/uv.lock b/unstract/sdk1/uv.lock index a482fe5b95..5bbecf8c59 100644 --- a/unstract/sdk1/uv.lock +++ b/unstract/sdk1/uv.lock @@ -1103,7 +1103,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1119,9 +1119,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -2785,7 +2785,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/unstract/tool-registry/uv.lock b/unstract/tool-registry/uv.lock index 5bedd8ce5b..84fa292120 100644 --- a/unstract/tool-registry/uv.lock +++ b/unstract/tool-registry/uv.lock @@ -726,7 +726,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -742,9 +742,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -1927,7 +1927,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py index e7b9d46167..3786706cd2 100644 --- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py +++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py @@ -14,8 +14,6 @@ class ToolRuntimeVariable: PLATFORM_HOST = "PLATFORM_SERVICE_HOST" PLATFORM_PORT = "PLATFORM_SERVICE_PORT" PLATFORM_SERVICE_API_KEY = "PLATFORM_SERVICE_API_KEY" - PROMPT_HOST = "PROMPT_HOST" - PROMPT_PORT = "PROMPT_PORT" X2TEXT_HOST = "X2TEXT_HOST" X2TEXT_PORT = "X2TEXT_PORT" ADAPTER_LLMW_POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL" diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py index 6778eafe17..040c9485cb 100644 --- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py +++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py @@ -44,8 +44,6 @@ def __init__( self.platform_service_port = ToolsUtils.get_env( ToolRV.PLATFORM_PORT, raise_exception=True ) - self.prompt_host = ToolsUtils.get_env(ToolRV.PROMPT_HOST, raise_exception=True) - self.prompt_port = ToolsUtils.get_env(ToolRV.PROMPT_PORT, raise_exception=True) self.x2text_host = ToolsUtils.get_env(ToolRV.X2TEXT_HOST, raise_exception=True) self.x2text_port = ToolsUtils.get_env(ToolRV.X2TEXT_PORT, raise_exception=True) self.llmw_poll_interval = ToolsUtils.get_env( @@ -232,8 +230,6 @@ def get_tool_environment_variables(self) -> dict[str, Any]: ToolRV.PLATFORM_HOST: self.platform_service_host, ToolRV.PLATFORM_PORT: self.platform_service_port, ToolRV.PLATFORM_SERVICE_API_KEY: self.platform_service_api_key, - ToolRV.PROMPT_HOST: self.prompt_host, - ToolRV.PROMPT_PORT: self.prompt_port, ToolRV.X2TEXT_HOST: self.x2text_host, ToolRV.X2TEXT_PORT: self.x2text_port, ToolRV.EXECUTION_BY_TOOL: True, diff --git a/unstract/workflow-execution/uv.lock b/unstract/workflow-execution/uv.lock index caa75e7238..8a067e6308 100644 --- a/unstract/workflow-execution/uv.lock +++ b/unstract/workflow-execution/uv.lock @@ -707,7 +707,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -723,9 +723,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -1938,7 +1938,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/uv.lock b/uv.lock index aa17b0c687..35e0045aed 100644 --- a/uv.lock +++ b/uv.lock @@ -1536,7 +1536,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1552,9 +1552,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324, upload-time = "2026-05-21T02:30:38.185Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080, upload-time = "2026-05-21T02:30:35.096Z" }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -3838,7 +3838,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" }, diff --git a/workers/pyproject.toml b/workers/pyproject.toml index b5c3c112aa..da672e0661 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -151,7 +151,7 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] -python_files = ["test_*.py", "*_test.py"] +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] python_classes = ["Test*"] python_functions = ["test_*"] asyncio_mode = "strict" diff --git a/workers/sample.env b/workers/sample.env index 42f9b70003..6a80eb4556 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -321,10 +321,6 @@ NOTIFICATION_QUEUE_NAME=notifications PLATFORM_SERVICE_HOST=http://unstract-platform-service PLATFORM_SERVICE_PORT=3001 -# Prompt Service -PROMPT_HOST=http://unstract-prompt-service -PROMPT_PORT=3003 - # X2Text Service X2TEXT_HOST=http://unstract-x2text-service X2TEXT_PORT=3004 @@ -475,7 +471,6 @@ GOOGLE_OAUTH2_SECRET= # REDIS_HOST=localhost # CACHE_REDIS_HOST=localhost # PLATFORM_SERVICE_HOST=http://localhost -# PROMPT_HOST=http://localhost # X2TEXT_HOST=http://localhost # UNSTRACT_RUNNER_HOST=http://localhost # WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}} diff --git a/workers/tests/test_legacy_executor_scaffold.py b/workers/tests/test_legacy_executor_scaffold.py index 2bce59b2d8..4895e6ffe9 100644 --- a/workers/tests/test_legacy_executor_scaffold.py +++ b/workers/tests/test_legacy_executor_scaffold.py @@ -260,37 +260,6 @@ def test_extraction_error_has_code_and_message(self): assert err.message == "extraction failed" assert err.code == 500 - def test_no_flask_import(self): - """Verify exceptions module does NOT import Flask.""" - import importlib - import sys - - # Reload to re-run the module's imports so a stray Flask import would - # re-populate sys.modules. But reloading rebinds the module's classes - # (LegacyExecutorError etc.) to new objects, which breaks ``isinstance`` - # / ``pytest.raises`` identity for every other test/module that imported - # them earlier. So snapshot the module namespace and restore it in a - # ``finally`` — the exception classes stay identical for the rest of the - # suite. - mod_name = "executor.executors.exceptions" - mod = sys.modules.get(mod_name) - saved = dict(mod.__dict__) if mod is not None else None - try: - if mod is not None: - importlib.reload(mod) - else: - importlib.import_module(mod_name) - - # Check that no flask modules were pulled in - flask_modules = [m for m in sys.modules if m.startswith("flask")] - assert flask_modules == [], ( - f"Flask modules imported: {flask_modules}" - ) - finally: - if saved is not None: - mod.__dict__.clear() - mod.__dict__.update(saved) - def test_custom_data_error_signature(self): from executor.executors.exceptions import CustomDataError diff --git a/workers/uv.lock b/workers/uv.lock index d360e6e11a..7b3af7d16f 100644 --- a/workers/uv.lock +++ b/workers/uv.lock @@ -2050,7 +2050,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.85.1" +version = "1.90.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -2066,9 +2066,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/55/aebffceaa08688a989e9c68b3edc3a520a1f8338eb0346668774bd66ad88/litellm-1.85.1.tar.gz", hash = "sha256:3b8ef0c89ff2736cbd27109f17ff31f1bd0ab59dee9be8cadb28ec3cb167ce0d", size = 15346324 } +sdist = { url = "https://files.pythonhosted.org/packages/fa/c8/18d0c831d97ef6e7a971e1e984aa0a07740adb3f343e4596f21a5e880840/litellm-1.90.3.tar.gz", hash = "sha256:1b15776011745ea4f90259d9bd2f269835a8541cd7948751bec726b1d21647fa", size = 14820738, upload-time = "2026-07-03T19:53:20.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/a0/263a13c2253201aa11563a69d9a87f3510030aa765a16f57fc40ceefcdf5/litellm-1.85.1-py3-none-any.whl", hash = "sha256:c89eb5dfd18cce3d40b59e79c74f7f645bc7814a417c6ab25e53c786f0a6ab7b", size = 16980080 }, + { url = "https://files.pythonhosted.org/packages/c5/ad/b40e8a29ec9b4564c5fd96829c4b22512c24ca76045304b1b521d6ea68c1/litellm-1.90.3-py3-none-any.whl", hash = "sha256:0ba6844865c0637719b3e76f43cf1089e3bd578a456b7be6a7f64afbbc259472", size = 16613824, upload-time = "2026-07-03T19:53:17.393Z" }, ] [[package]] @@ -4841,7 +4841,7 @@ requires-dist = [ { name = "gcsfs", marker = "extra == 'gcs'", specifier = "~=2024.10.0" }, { name = "httpx", specifier = ">=0.25.2" }, { name = "jsonschema" }, - { name = "litellm", specifier = "==1.85.1" }, + { name = "litellm", specifier = "==1.90.3" }, { name = "llama-index", specifier = ">=0.14.13" }, { name = "llama-index-vector-stores-milvus", specifier = ">=0.9.6" }, { name = "llama-index-vector-stores-pinecone", specifier = ">=0.7.1" },