Skip to content

Commit 10b738a

Browse files
QuantumGhostQuantumGhost
authored
feat: Persist Variables for Enhanced Debugging Workflow (langgenius#20699)
This pull request introduces a feature aimed at improving the debugging experience during workflow editing. With the addition of variable persistence, the system will automatically retain the output variables from previously executed nodes. These persisted variables can then be reused when debugging subsequent nodes, eliminating the need for repetitive manual input. By streamlining this aspect of the workflow, the feature minimizes user errors and significantly reduces debugging effort, offering a smoother and more efficient experience. Key highlights of this change: - Automatic persistence of output variables for executed nodes. - Reuse of persisted variables to simplify input steps for nodes requiring them (e.g., `code`, `template`, `variable_assigner`). - Enhanced debugging experience with reduced friction. Closes langgenius#19735.
1 parent 3113350 commit 10b738a

106 files changed

Lines changed: 6068 additions & 761 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/api-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,15 @@ jobs:
8383
compose-file: |
8484
docker/docker-compose.middleware.yaml
8585
services: |
86+
db
87+
redis
8688
sandbox
8789
ssrf_proxy
8890
91+
- name: setup test config
92+
run: |
93+
cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
94+
8995
- name: Run Workflow
9096
run: uv run --project api bash dev/pytest/pytest_workflow.sh
9197

‎.github/workflows/vdb-tests.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@ jobs:
8484
elasticsearch
8585
oceanbase
8686
87+
- name: setup test config
88+
run: |
89+
echo $(pwd)
90+
ls -lah .
91+
cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
92+
8793
- name: Check VDB Ready (TiDB)
8894
run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
8995

‎api/.ruff.toml‎

Lines changed: 51 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
exclude = [
2-
"migrations/*",
3-
]
1+
exclude = ["migrations/*"]
42
line-length = 120
53

64
[format]
@@ -9,34 +7,34 @@ quote-style = "double"
97
[lint]
108
preview = false
119
select = [
12-
"B", # flake8-bugbear rules
13-
"C4", # flake8-comprehensions
14-
"E", # pycodestyle E rules
15-
"F", # pyflakes rules
16-
"FURB", # refurb rules
17-
"I", # isort rules
18-
"N", # pep8-naming
19-
"PT", # flake8-pytest-style rules
10+
"B", # flake8-bugbear rules
11+
"C4", # flake8-comprehensions
12+
"E", # pycodestyle E rules
13+
"F", # pyflakes rules
14+
"FURB", # refurb rules
15+
"I", # isort rules
16+
"N", # pep8-naming
17+
"PT", # flake8-pytest-style rules
2018
"PLC0208", # iteration-over-set
2119
"PLC0414", # useless-import-alias
2220
"PLE0604", # invalid-all-object
2321
"PLE0605", # invalid-all-format
2422
"PLR0402", # manual-from-import
2523
"PLR1711", # useless-return
2624
"PLR1714", # repeated-equality-comparison
27-
"RUF013", # implicit-optional
28-
"RUF019", # unnecessary-key-check
29-
"RUF100", # unused-noqa
30-
"RUF101", # redirected-noqa
31-
"RUF200", # invalid-pyproject-toml
32-
"RUF022", # unsorted-dunder-all
33-
"S506", # unsafe-yaml-load
34-
"SIM", # flake8-simplify rules
35-
"TRY400", # error-instead-of-exception
36-
"TRY401", # verbose-log-message
37-
"UP", # pyupgrade rules
38-
"W191", # tab-indentation
39-
"W605", # invalid-escape-sequence
25+
"RUF013", # implicit-optional
26+
"RUF019", # unnecessary-key-check
27+
"RUF100", # unused-noqa
28+
"RUF101", # redirected-noqa
29+
"RUF200", # invalid-pyproject-toml
30+
"RUF022", # unsorted-dunder-all
31+
"S506", # unsafe-yaml-load
32+
"SIM", # flake8-simplify rules
33+
"TRY400", # error-instead-of-exception
34+
"TRY401", # verbose-log-message
35+
"UP", # pyupgrade rules
36+
"W191", # tab-indentation
37+
"W605", # invalid-escape-sequence
4038
# security related linting rules
4139
# RCE proctection (sort of)
4240
"S102", # exec-builtin, disallow use of `exec`
@@ -47,36 +45,37 @@ select = [
4745
]
4846

4947
ignore = [
50-
"E402", # module-import-not-at-top-of-file
51-
"E711", # none-comparison
52-
"E712", # true-false-comparison
53-
"E721", # type-comparison
54-
"E722", # bare-except
55-
"F821", # undefined-name
56-
"F841", # unused-variable
48+
"E402", # module-import-not-at-top-of-file
49+
"E711", # none-comparison
50+
"E712", # true-false-comparison
51+
"E721", # type-comparison
52+
"E722", # bare-except
53+
"F821", # undefined-name
54+
"F841", # unused-variable
5755
"FURB113", # repeated-append
5856
"FURB152", # math-constant
59-
"UP007", # non-pep604-annotation
60-
"UP032", # f-string
61-
"UP045", # non-pep604-annotation-optional
62-
"B005", # strip-with-multi-characters
63-
"B006", # mutable-argument-default
64-
"B007", # unused-loop-control-variable
65-
"B026", # star-arg-unpacking-after-keyword-arg
66-
"B903", # class-as-data-structure
67-
"B904", # raise-without-from-inside-except
68-
"B905", # zip-without-explicit-strict
69-
"N806", # non-lowercase-variable-in-function
70-
"N815", # mixed-case-variable-in-class-scope
71-
"PT011", # pytest-raises-too-broad
72-
"SIM102", # collapsible-if
73-
"SIM103", # needless-bool
74-
"SIM105", # suppressible-exception
75-
"SIM107", # return-in-try-except-finally
76-
"SIM108", # if-else-block-instead-of-if-exp
77-
"SIM113", # enumerate-for-loop
78-
"SIM117", # multiple-with-statements
79-
"SIM210", # if-expr-with-true-false
57+
"UP007", # non-pep604-annotation
58+
"UP032", # f-string
59+
"UP045", # non-pep604-annotation-optional
60+
"B005", # strip-with-multi-characters
61+
"B006", # mutable-argument-default
62+
"B007", # unused-loop-control-variable
63+
"B026", # star-arg-unpacking-after-keyword-arg
64+
"B903", # class-as-data-structure
65+
"B904", # raise-without-from-inside-except
66+
"B905", # zip-without-explicit-strict
67+
"N806", # non-lowercase-variable-in-function
68+
"N815", # mixed-case-variable-in-class-scope
69+
"PT011", # pytest-raises-too-broad
70+
"SIM102", # collapsible-if
71+
"SIM103", # needless-bool
72+
"SIM105", # suppressible-exception
73+
"SIM107", # return-in-try-except-finally
74+
"SIM108", # if-else-block-instead-of-if-exp
75+
"SIM113", # enumerate-for-loop
76+
"SIM117", # multiple-with-statements
77+
"SIM210", # if-expr-with-true-false
78+
"UP038", # deprecated and not recommended by Ruff, https://docs.astral.sh/ruff/rules/non-pep604-isinstance/
8079
]
8180

8281
[lint.per-file-ignores]

‎api/controllers/console/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
statistic,
6464
workflow,
6565
workflow_app_log,
66+
workflow_draft_variable,
6667
workflow_run,
6768
workflow_statistic,
6869
)

‎api/controllers/console/app/workflow.py‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import json
22
import logging
3+
from collections.abc import Sequence
34
from typing import cast
45

56
from flask import abort, request
@@ -18,10 +19,12 @@
1819
from controllers.console.app.wraps import get_app_model
1920
from controllers.console.wraps import account_initialization_required, setup_required
2021
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
22+
from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
2123
from core.app.apps.base_app_queue_manager import AppQueueManager
2224
from core.app.entities.app_invoke_entities import InvokeFrom
25+
from core.file.models import File
2326
from extensions.ext_database import db
24-
from factories import variable_factory
27+
from factories import file_factory, variable_factory
2528
from fields.workflow_fields import workflow_fields, workflow_pagination_fields
2629
from fields.workflow_run_fields import workflow_run_node_execution_fields
2730
from libs import helper
@@ -30,6 +33,7 @@
3033
from models import App
3134
from models.account import Account
3235
from models.model import AppMode
36+
from models.workflow import Workflow
3337
from services.app_generate_service import AppGenerateService
3438
from services.errors.app import WorkflowHashNotEqualError
3539
from services.errors.llm import InvokeRateLimitError
@@ -38,6 +42,24 @@
3842
logger = logging.getLogger(__name__)
3943

4044

45+
# TODO(QuantumGhost): Refactor existing node run API to handle file parameter parsing
46+
# at the controller level rather than in the workflow logic. This would improve separation
47+
# of concerns and make the code more maintainable.
48+
def _parse_file(workflow: Workflow, files: list[dict] | None = None) -> Sequence[File]:
49+
files = files or []
50+
51+
file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False)
52+
file_objs: Sequence[File] = []
53+
if file_extra_config is None:
54+
return file_objs
55+
file_objs = file_factory.build_from_mappings(
56+
mappings=files,
57+
tenant_id=workflow.tenant_id,
58+
config=file_extra_config,
59+
)
60+
return file_objs
61+
62+
4163
class DraftWorkflowApi(Resource):
4264
@setup_required
4365
@login_required
@@ -402,15 +424,30 @@ def post(self, app_model: App, node_id: str):
402424

403425
parser = reqparse.RequestParser()
404426
parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
427+
parser.add_argument("query", type=str, required=False, location="json", default="")
428+
parser.add_argument("files", type=list, location="json", default=[])
405429
args = parser.parse_args()
406430

407-
inputs = args.get("inputs")
408-
if inputs == None:
431+
user_inputs = args.get("inputs")
432+
if user_inputs is None:
409433
raise ValueError("missing inputs")
410434

435+
workflow_srv = WorkflowService()
436+
# fetch draft workflow by app_model
437+
draft_workflow = workflow_srv.get_draft_workflow(app_model=app_model)
438+
if not draft_workflow:
439+
raise ValueError("Workflow not initialized")
440+
files = _parse_file(draft_workflow, args.get("files"))
411441
workflow_service = WorkflowService()
442+
412443
workflow_node_execution = workflow_service.run_draft_workflow_node(
413-
app_model=app_model, node_id=node_id, user_inputs=inputs, account=current_user
444+
app_model=app_model,
445+
draft_workflow=draft_workflow,
446+
node_id=node_id,
447+
user_inputs=user_inputs,
448+
account=current_user,
449+
query=args.get("query", ""),
450+
files=files,
414451
)
415452

416453
return workflow_node_execution
@@ -731,6 +768,27 @@ def delete(self, app_model: App, workflow_id: str):
731768
return None, 204
732769

733770

771+
class DraftWorkflowNodeLastRunApi(Resource):
772+
@setup_required
773+
@login_required
774+
@account_initialization_required
775+
@get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
776+
@marshal_with(workflow_run_node_execution_fields)
777+
def get(self, app_model: App, node_id: str):
778+
srv = WorkflowService()
779+
workflow = srv.get_draft_workflow(app_model)
780+
if not workflow:
781+
raise NotFound("Workflow not found")
782+
node_exec = srv.get_node_last_run(
783+
app_model=app_model,
784+
workflow=workflow,
785+
node_id=node_id,
786+
)
787+
if node_exec is None:
788+
raise NotFound("last run not found")
789+
return node_exec
790+
791+
734792
api.add_resource(
735793
DraftWorkflowApi,
736794
"/apps/<uuid:app_id>/workflows/draft",
@@ -795,3 +853,7 @@ def delete(self, app_model: App, workflow_id: str):
795853
WorkflowByIdApi,
796854
"/apps/<uuid:app_id>/workflows/<string:workflow_id>",
797855
)
856+
api.add_resource(
857+
DraftWorkflowNodeLastRunApi,
858+
"/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/last-run",
859+
)

0 commit comments

Comments
 (0)