From 7656d514b99dc4644f2cc86d4bceca6203de0b8a Mon Sep 17 00:00:00 2001 From: 99 Date: Mon, 16 Feb 2026 22:38:19 +0800 Subject: [PATCH] refactor(workflow-file): move `core.file` to `core.workflow.file` (#32252) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- api/.importlinter | 29 ------- api/controllers/common/fields.py | 2 +- api/controllers/console/app/app.py | 2 +- api/controllers/console/app/workflow.py | 2 +- .../console/app/workflow_draft_variable.py | 2 +- api/controllers/console/remote_files.py | 2 +- api/controllers/files/upload.py | 2 +- api/controllers/inner_api/plugin/plugin.py | 2 +- api/controllers/web/remote_files.py | 2 +- api/core/agent/base_agent_runner.py | 2 +- api/core/agent/cot_chat_agent_runner.py | 2 +- api/core/agent/fc_agent_runner.py | 2 +- api/core/app/app_config/entities.py | 2 +- .../features/file_upload/manager.py | 2 +- api/core/app/apps/base_app_generator.py | 2 +- api/core/app/apps/base_app_runner.py | 4 +- api/core/app/apps/chat/app_runner.py | 2 +- .../common/workflow_response_converter.py | 2 +- api/core/app/apps/completion/app_runner.py | 2 +- api/core/app/entities/app_invoke_entities.py | 2 +- .../easy_ui_based_generate_task_pipeline.py | 4 +- api/core/app/workflow/file_runtime.py | 47 ++++++++++++ api/core/app/workflow/node_factory.py | 2 +- .../datasource/datasource_file_manager.py | 2 +- .../datasource/utils/message_transformer.py | 2 +- api/core/entities/mcp_provider.py | 2 +- api/core/file/tool_file_parser.py | 12 --- api/core/memory/token_buffer_memory.py | 2 +- api/core/plugin/utils/converter.py | 2 +- api/core/prompt/advanced_prompt_transform.py | 4 +- api/core/prompt/simple_prompt_transform.py | 4 +- .../processor/paragraph_index_processor.py | 2 +- api/core/rag/models/document.py | 2 +- api/core/rag/retrieval/dataset_retrieval.py | 2 +- .../builtin_tool/providers/audio/tools/asr.py | 4 +- api/core/tools/custom_tool/tool.py | 2 +- api/core/tools/tool_engine.py | 4 +- api/core/tools/tool_file_manager.py | 2 +- api/core/tools/utils/message_transformer.py | 2 +- api/core/tools/workflow_as_tool/tool.py | 2 +- api/core/variables/segments.py | 2 +- api/core/variables/types.py | 2 +- api/core/{ => workflow}/file/__init__.py | 0 api/core/{ => workflow}/file/constants.py | 0 api/core/{ => workflow}/file/enums.py | 0 api/core/{ => workflow}/file/file_manager.py | 76 +++++-------------- api/core/{ => workflow}/file/helpers.py | 43 ++++++----- api/core/{ => workflow}/file/models.py | 22 +++++- api/core/workflow/file/protocols.py | 43 +++++++++++ api/core/workflow/file/runtime.py | 58 ++++++++++++++ api/core/workflow/file/tool_file_parser.py | 9 +++ api/core/workflow/node_events/node.py | 2 +- api/core/workflow/nodes/agent/agent_node.py | 2 +- .../nodes/datasource/datasource_node.py | 4 +- .../workflow/nodes/document_extractor/node.py | 2 +- .../workflow/nodes/http_request/executor.py | 4 +- api/core/workflow/nodes/http_request/node.py | 4 +- .../knowledge_retrieval_node.py | 2 +- api/core/workflow/nodes/list_operator/node.py | 2 +- api/core/workflow/nodes/llm/file_saver.py | 2 +- api/core/workflow/nodes/llm/llm_utils.py | 2 +- api/core/workflow/nodes/llm/node.py | 4 +- api/core/workflow/nodes/loop/loop_node.py | 6 +- .../parameter_extractor_node.py | 2 +- api/core/workflow/nodes/protocols.py | 2 +- .../question_classifier_node.py | 2 +- api/core/workflow/nodes/tool/tool_node.py | 2 +- .../workflow/nodes/trigger_webhook/node.py | 2 +- api/core/workflow/runtime/variable_pool.py | 2 +- api/core/workflow/system_variable.py | 2 +- .../workflow/utils/condition/processor.py | 2 +- api/core/workflow/workflow_entry.py | 2 +- api/core/workflow/workflow_type_encoder.py | 2 +- api/extensions/ext_storage.py | 7 ++ api/extensions/otel/parser/base.py | 2 +- api/factories/file_factory.py | 2 +- api/factories/variable_factory.py | 2 +- api/fields/conversation_fields.py | 2 +- api/fields/member_fields.py | 2 +- api/fields/message_fields.py | 2 +- api/fields/raws.py | 2 +- api/libs/helper.py | 2 +- api/models/model.py | 4 +- api/models/workflow.py | 4 +- api/services/dataset_service.py | 2 +- api/services/file_service.py | 2 +- api/services/trigger/webhook_service.py | 2 +- api/services/variable_truncator.py | 2 +- api/services/workflow/workflow_converter.py | 2 +- .../workflow_draft_variable_service.py | 2 +- api/services/workflow_service.py | 2 +- api/tests/conftest.py | 8 ++ .../factories/test_storage_key_loader.py | 2 +- .../factories/test_storage_key_loader.py | 2 +- .../services/test_agent_service.py | 2 +- .../app/workflow_draft_variables_test.py | 8 +- .../test_datasets_document_download.py | 4 +- .../features/file_upload/test_manager.py | 2 +- .../chat/test_base_app_runner_multimodal.py | 2 +- .../test_workflow_response_converter.py | 2 +- api/tests/unit_tests/core/file/test_models.py | 2 +- .../prompt/test_advanced_prompt_transform.py | 4 +- api/tests/unit_tests/core/test_file.py | 2 +- .../unit_tests/core/variables/test_segment.py | 2 +- .../variables/test_segment_type_validation.py | 4 +- .../workflow/nodes/llm/test_file_saver.py | 2 +- .../core/workflow/nodes/llm/test_node.py | 2 +- .../core/workflow/nodes/llm/test_scenarios.py | 2 +- .../nodes/test_document_extractor_node.py | 4 +- .../core/workflow/nodes/test_if_else.py | 2 +- .../core/workflow/nodes/test_list_operator.py | 2 +- .../workflow/nodes/tool/test_tool_node.py | 2 +- .../nodes/webhook/test_webhook_node.py | 2 +- .../core/workflow/test_system_variable.py | 4 +- .../test_system_variable_read_only_view.py | 2 +- .../core/workflow/test_variable_pool.py | 2 +- .../core/workflow/test_workflow_entry.py | 4 +- .../factories/test_variable_factory.py | 2 +- api/tests/unit_tests/models/test_workflow.py | 4 +- .../services/test_variable_truncator.py | 4 +- 120 files changed, 364 insertions(+), 252 deletions(-) create mode 100644 api/core/app/workflow/file_runtime.py delete mode 100644 api/core/file/tool_file_parser.py rename api/core/{ => workflow}/file/__init__.py (100%) rename api/core/{ => workflow}/file/constants.py (100%) rename api/core/{ => workflow}/file/enums.py (100%) rename api/core/{ => workflow}/file/file_manager.py (64%) rename api/core/{ => workflow}/file/helpers.py (65%) rename api/core/{ => workflow}/file/models.py (90%) create mode 100644 api/core/workflow/file/protocols.py create mode 100644 api/core/workflow/file/runtime.py create mode 100644 api/core/workflow/file/tool_file_parser.py create mode 100644 api/tests/conftest.py diff --git a/api/.importlinter b/api/.importlinter index e30f498ba9..5fe76ce4c8 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -115,18 +115,15 @@ ignore_imports = core.workflow.nodes.datasource.datasource_node -> models.tools core.workflow.nodes.datasource.datasource_node -> services.datasource_provider_service core.workflow.nodes.document_extractor.node -> configs - core.workflow.nodes.document_extractor.node -> core.file.file_manager core.workflow.nodes.document_extractor.node -> core.helper.ssrf_proxy core.workflow.nodes.http_request.entities -> configs core.workflow.nodes.http_request.executor -> configs - core.workflow.nodes.http_request.executor -> core.file.file_manager core.workflow.nodes.http_request.node -> configs core.workflow.nodes.http_request.node -> core.tools.tool_file_manager core.workflow.nodes.iteration.iteration_node -> core.app.workflow.node_factory core.workflow.nodes.knowledge_index.knowledge_index_node -> core.rag.index_processor.index_processor_factory core.workflow.nodes.llm.llm_utils -> configs core.workflow.nodes.llm.llm_utils -> core.app.entities.app_invoke_entities - core.workflow.nodes.llm.llm_utils -> core.file.models core.workflow.nodes.llm.llm_utils -> core.model_manager core.workflow.nodes.llm.llm_utils -> core.model_runtime.model_providers.__base.large_language_model core.workflow.nodes.llm.llm_utils -> models.model @@ -162,36 +159,10 @@ ignore_imports = core.workflow.nodes.llm.llm_utils -> core.entities.provider_entities core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.model_manager core.workflow.nodes.question_classifier.question_classifier_node -> core.model_manager - core.workflow.node_events.node -> core.file - core.workflow.nodes.agent.agent_node -> core.file - core.workflow.nodes.datasource.datasource_node -> core.file - core.workflow.nodes.datasource.datasource_node -> core.file.enums - core.workflow.nodes.document_extractor.node -> core.file - core.workflow.nodes.http_request.executor -> core.file.enums - core.workflow.nodes.http_request.node -> core.file - core.workflow.nodes.http_request.node -> core.file.file_manager - core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> core.file.models - core.workflow.nodes.list_operator.node -> core.file - core.workflow.nodes.llm.file_saver -> core.file core.workflow.nodes.llm.llm_utils -> core.variables.segments - core.workflow.nodes.llm.node -> core.file - core.workflow.nodes.llm.node -> core.file.file_manager - core.workflow.nodes.llm.node -> core.file.models core.workflow.nodes.loop.entities -> core.variables.types - core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.file - core.workflow.nodes.protocols -> core.file - core.workflow.nodes.question_classifier.question_classifier_node -> core.file.models - core.workflow.nodes.tool.tool_node -> core.file core.workflow.nodes.tool.tool_node -> core.tools.utils.message_transformer core.workflow.nodes.tool.tool_node -> models - core.workflow.nodes.trigger_webhook.node -> core.file - core.workflow.runtime.variable_pool -> core.file - core.workflow.runtime.variable_pool -> core.file.file_manager - core.workflow.system_variable -> core.file.models - core.workflow.utils.condition.processor -> core.file - core.workflow.utils.condition.processor -> core.file.file_manager - core.workflow.workflow_entry -> core.file.models - core.workflow.workflow_type_encoder -> core.file.models core.workflow.nodes.agent.agent_node -> models.model core.workflow.nodes.code.code_node -> core.helper.code_executor.code_node_provider core.workflow.nodes.code.code_node -> core.helper.code_executor.javascript.javascript_code_provider diff --git a/api/controllers/common/fields.py b/api/controllers/common/fields.py index c16a23fac8..9b30db8b75 100644 --- a/api/controllers/common/fields.py +++ b/api/controllers/common/fields.py @@ -4,7 +4,7 @@ from typing import Any, TypeAlias from pydantic import BaseModel, ConfigDict, computed_field -from core.file import helpers as file_helpers +from core.workflow.file import helpers as file_helpers from models.model import IconType JSONValue: TypeAlias = str | int | float | bool | None | dict[str, Any] | list[Any] diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 42901ab590..e799e98d3e 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -23,10 +23,10 @@ from controllers.console.wraps import ( is_admin_or_owner_required, setup_required, ) -from core.file import helpers as file_helpers from core.ops.ops_trace_manager import OpsTraceManager from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.workflow.enums import NodeType, WorkflowExecutionStatus +from core.workflow.file import helpers as file_helpers from extensions.ext_database import db from libs.login import current_account_with_tenant, login_required from models import App, DatasetPermissionEnum, Workflow diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index 27e1d01af6..b05d28b686 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -20,7 +20,6 @@ from core.app.app_config.features.file_upload.manager import FileUploadConfigMan from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.apps.workflow.app_generator import SKIP_PREPARE_USER_INPUTS_KEY from core.app.entities.app_invoke_entities import InvokeFrom -from core.file.models import File from core.helper.trace_id_helper import get_external_trace_id from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.impl.exc import PluginInvokeError @@ -31,6 +30,7 @@ from core.trigger.debug.event_selectors import ( select_trigger_debug_events, ) from core.workflow.enums import NodeType +from core.workflow.file.models import File from core.workflow.graph_engine.manager import GraphEngineManager from extensions.ext_database import db from factories import file_factory, variable_factory diff --git a/api/controllers/console/app/workflow_draft_variable.py b/api/controllers/console/app/workflow_draft_variable.py index 3382b65acc..e08758bd3b 100644 --- a/api/controllers/console/app/workflow_draft_variable.py +++ b/api/controllers/console/app/workflow_draft_variable.py @@ -15,11 +15,11 @@ from controllers.console.app.error import ( from controllers.console.app.wraps import get_app_model from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required from controllers.web.error import InvalidArgumentError, NotFoundError -from core.file import helpers as file_helpers from core.variables.segment_group import SegmentGroup from core.variables.segments import ArrayFileSegment, FileSegment, Segment from core.variables.types import SegmentType from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID +from core.workflow.file import helpers as file_helpers from extensions.ext_database import db from factories.file_factory import build_from_mapping, build_from_mappings from factories.variable_factory import build_segment_with_type diff --git a/api/controllers/console/remote_files.py b/api/controllers/console/remote_files.py index b7a2f230e1..f3738319df 100644 --- a/api/controllers/console/remote_files.py +++ b/api/controllers/console/remote_files.py @@ -12,8 +12,8 @@ from controllers.common.errors import ( UnsupportedFileTypeError, ) from controllers.console import console_ns -from core.file import helpers as file_helpers from core.helper import ssrf_proxy +from core.workflow.file import helpers as file_helpers from extensions.ext_database import db from fields.file_fields import FileWithSignedUrl, RemoteFileInfo from libs.login import current_account_with_tenant, login_required diff --git a/api/controllers/files/upload.py b/api/controllers/files/upload.py index 28ec4b3935..b34412ef6d 100644 --- a/api/controllers/files/upload.py +++ b/api/controllers/files/upload.py @@ -7,8 +7,8 @@ from pydantic import BaseModel, Field from werkzeug.exceptions import Forbidden import services -from core.file.helpers import verify_plugin_file_signature from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file.helpers import verify_plugin_file_signature from fields.file_fields import FileResponse from ..common.errors import ( diff --git a/api/controllers/inner_api/plugin/plugin.py b/api/controllers/inner_api/plugin/plugin.py index e4fe8d44bf..4cd1c4745f 100644 --- a/api/controllers/inner_api/plugin/plugin.py +++ b/api/controllers/inner_api/plugin/plugin.py @@ -4,7 +4,6 @@ from controllers.console.wraps import setup_required from controllers.inner_api import inner_api_ns from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data from controllers.inner_api.wraps import plugin_inner_api_only -from core.file.helpers import get_signed_file_url_for_plugin from core.model_runtime.utils.encoders import jsonable_encoder from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse @@ -30,6 +29,7 @@ from core.plugin.entities.request import ( RequestRequestUploadFile, ) from core.tools.entities.tool_entities import ToolProviderType +from core.workflow.file.helpers import get_signed_file_url_for_plugin from libs.helper import length_prefixed_response from models import Account, Tenant from models.model import EndUser diff --git a/api/controllers/web/remote_files.py b/api/controllers/web/remote_files.py index b08b3fe858..1cdae0fe56 100644 --- a/api/controllers/web/remote_files.py +++ b/api/controllers/web/remote_files.py @@ -10,8 +10,8 @@ from controllers.common.errors import ( RemoteFileUploadError, UnsupportedFileTypeError, ) -from core.file import helpers as file_helpers from core.helper import ssrf_proxy +from core.workflow.file import helpers as file_helpers from extensions.ext_database import db from fields.file_fields import FileWithSignedUrl, RemoteFileInfo from services.file_service import FileService diff --git a/api/core/agent/base_agent_runner.py b/api/core/agent/base_agent_runner.py index 3c6d36afe4..a125050082 100644 --- a/api/core/agent/base_agent_runner.py +++ b/api/core/agent/base_agent_runner.py @@ -17,7 +17,6 @@ from core.app.entities.app_invoke_entities import ( ) from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.file import file_manager from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities import ( @@ -40,6 +39,7 @@ from core.tools.entities.tool_entities import ( ) from core.tools.tool_manager import ToolManager from core.tools.utils.dataset_retriever_tool import DatasetRetrieverTool +from core.workflow.file import file_manager from extensions.ext_database import db from factories import file_factory from models.enums import CreatorUserRole diff --git a/api/core/agent/cot_chat_agent_runner.py b/api/core/agent/cot_chat_agent_runner.py index 4d1d94eadc..babb463aba 100644 --- a/api/core/agent/cot_chat_agent_runner.py +++ b/api/core/agent/cot_chat_agent_runner.py @@ -1,7 +1,6 @@ import json from core.agent.cot_agent_runner import CotAgentRunner -from core.file import file_manager from core.model_runtime.entities import ( AssistantPromptMessage, PromptMessage, @@ -11,6 +10,7 @@ from core.model_runtime.entities import ( ) from core.model_runtime.entities.message_entities import ImagePromptMessageContent, PromptMessageContentUnionTypes from core.model_runtime.utils.encoders import jsonable_encoder +from core.workflow.file import file_manager class CotChatAgentRunner(CotAgentRunner): diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index 7c5c9136a7..f9da2f3b43 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -7,7 +7,6 @@ from typing import Any, Union from core.agent.base_agent_runner import BaseAgentRunner from core.app.apps.base_app_queue_manager import PublishFrom from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueMessageEndEvent, QueueMessageFileEvent -from core.file import file_manager from core.model_runtime.entities import ( AssistantPromptMessage, LLMResult, @@ -25,6 +24,7 @@ from core.model_runtime.entities.message_entities import ImagePromptMessageConte from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform from core.tools.entities.tool_entities import ToolInvokeMeta from core.tools.tool_engine import ToolEngine +from core.workflow.file import file_manager from core.workflow.nodes.agent.exc import AgentMaxIterationError from models.model import Message diff --git a/api/core/app/app_config/entities.py b/api/core/app/app_config/entities.py index 13c51529cc..f8538d474c 100644 --- a/api/core/app/app_config/entities.py +++ b/api/core/app/app_config/entities.py @@ -5,9 +5,9 @@ from typing import Any, Literal from jsonschema import Draft7Validator, SchemaError from pydantic import BaseModel, Field, field_validator -from core.file import FileTransferMethod, FileType, FileUploadConfig from core.model_runtime.entities.llm_entities import LLMMode from core.model_runtime.entities.message_entities import PromptMessageRole +from core.workflow.file import FileTransferMethod, FileType, FileUploadConfig from models.model import AppMode diff --git a/api/core/app/app_config/features/file_upload/manager.py b/api/core/app/app_config/features/file_upload/manager.py index 40b6c19214..d69fa85801 100644 --- a/api/core/app/app_config/features/file_upload/manager.py +++ b/api/core/app/app_config/features/file_upload/manager.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from typing import Any from constants import DEFAULT_FILE_NUMBER_LIMITS -from core.file import FileUploadConfig +from core.workflow.file import FileUploadConfig class FileUploadConfigManager: diff --git a/api/core/app/apps/base_app_generator.py b/api/core/app/apps/base_app_generator.py index 07bae66867..48742205f1 100644 --- a/api/core/app/apps/base_app_generator.py +++ b/api/core/app/apps/base_app_generator.py @@ -5,8 +5,8 @@ from sqlalchemy.orm import Session from core.app.app_config.entities import VariableEntityType from core.app.entities.app_invoke_entities import InvokeFrom -from core.file import File, FileUploadConfig from core.workflow.enums import NodeType +from core.workflow.file import File, FileUploadConfig from core.workflow.repositories.draft_variable_repository import ( DraftVariableSaver, DraftVariableSaverFactory, diff --git a/api/core/app/apps/base_app_runner.py b/api/core/app/apps/base_app_runner.py index 617515945b..b98e85dbe9 100644 --- a/api/core/app/apps/base_app_runner.py +++ b/api/core/app/apps/base_app_runner.py @@ -22,7 +22,6 @@ from core.app.entities.queue_entities import ( from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature from core.app.features.hosting_moderation.hosting_moderation import HostingModerationFeature from core.external_data_tool.external_data_fetch import ExternalDataFetch -from core.file.enums import FileTransferMethod, FileType from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage @@ -39,12 +38,13 @@ from core.prompt.advanced_prompt_transform import AdvancedPromptTransform from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig from core.prompt.simple_prompt_transform import ModelMode, SimplePromptTransform from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file.enums import FileTransferMethod, FileType from extensions.ext_database import db from models.enums import CreatorUserRole from models.model import App, AppMode, Message, MessageAnnotation, MessageFile if TYPE_CHECKING: - from core.file.models import File + from core.workflow.file.models import File _logger = logging.getLogger(__name__) diff --git a/api/core/app/apps/chat/app_runner.py b/api/core/app/apps/chat/app_runner.py index 7d1a4c619f..4870a56281 100644 --- a/api/core/app/apps/chat/app_runner.py +++ b/api/core/app/apps/chat/app_runner.py @@ -11,12 +11,12 @@ from core.app.entities.app_invoke_entities import ( ) from core.app.entities.queue_entities import QueueAnnotationReplyEvent from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.file import File from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities.message_entities import ImagePromptMessageContent from core.moderation.base import ModerationError from core.rag.retrieval.dataset_retrieval import DatasetRetrieval +from core.workflow.file import File from extensions.ext_database import db from models.model import App, Conversation, Message diff --git a/api/core/app/apps/common/workflow_response_converter.py b/api/core/app/apps/common/workflow_response_converter.py index c0adb7120b..510abdc1d0 100644 --- a/api/core/app/apps/common/workflow_response_converter.py +++ b/api/core/app/apps/common/workflow_response_converter.py @@ -45,7 +45,6 @@ from core.app.entities.task_entities import ( WorkflowPauseStreamResponse, WorkflowStartStreamResponse, ) -from core.file import FILE_MODEL_IDENTITY, File from core.plugin.impl.datasource import PluginDatasourceManager from core.tools.entities.tool_entities import ToolProviderType from core.tools.tool_manager import ToolManager @@ -60,6 +59,7 @@ from core.workflow.enums import ( WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) +from core.workflow.file import FILE_MODEL_IDENTITY, File from core.workflow.runtime import GraphRuntimeState from core.workflow.system_variable import SystemVariable from core.workflow.workflow_entry import WorkflowEntry diff --git a/api/core/app/apps/completion/app_runner.py b/api/core/app/apps/completion/app_runner.py index a872c2e1f7..30e1a609f8 100644 --- a/api/core/app/apps/completion/app_runner.py +++ b/api/core/app/apps/completion/app_runner.py @@ -10,11 +10,11 @@ from core.app.entities.app_invoke_entities import ( CompletionAppGenerateEntity, ) from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.file import File from core.model_manager import ModelInstance from core.model_runtime.entities.message_entities import ImagePromptMessageContent from core.moderation.base import ModerationError from core.rag.retrieval.dataset_retrieval import DatasetRetrieval +from core.workflow.file import File from extensions.ext_database import db from models.model import App, Message diff --git a/api/core/app/entities/app_invoke_entities.py b/api/core/app/entities/app_invoke_entities.py index 0e68e554c8..65919e89e1 100644 --- a/api/core/app/entities/app_invoke_entities.py +++ b/api/core/app/entities/app_invoke_entities.py @@ -7,8 +7,8 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validat from constants import UUID_NIL from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig from core.entities.provider_configuration import ProviderModelBundle -from core.file import File, FileUploadConfig from core.model_runtime.entities.model_entities import AIModelEntity +from core.workflow.file import File, FileUploadConfig if TYPE_CHECKING: from core.ops.ops_trace_manager import TraceQueueManager diff --git a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py index 833f32fc7d..8792e65512 100644 --- a/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py @@ -45,8 +45,6 @@ from core.app.entities.task_entities import ( from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline from core.app.task_pipeline.message_cycle_manager import MessageCycleManager from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk -from core.file import helpers as file_helpers -from core.file.enums import FileTransferMethod from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import ( @@ -59,6 +57,8 @@ from core.ops.ops_trace_manager import TraceQueueManager, TraceTask from core.prompt.utils.prompt_message_util import PromptMessageUtil from core.prompt.utils.prompt_template_parser import PromptTemplateParser from core.tools.signature import sign_tool_file +from core.workflow.file import helpers as file_helpers +from core.workflow.file.enums import FileTransferMethod from events.message_event import message_was_created from extensions.ext_database import db from libs.datetime_utils import naive_utc_now diff --git a/api/core/app/workflow/file_runtime.py b/api/core/app/workflow/file_runtime.py new file mode 100644 index 0000000000..954638b901 --- /dev/null +++ b/api/core/app/workflow/file_runtime.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Generator + +from configs import dify_config +from core.helper.ssrf_proxy import ssrf_proxy +from core.tools.signature import sign_tool_file +from core.workflow.file.protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol +from core.workflow.file.runtime import set_workflow_file_runtime +from extensions.ext_storage import storage + + +class DifyWorkflowFileRuntime(WorkflowFileRuntimeProtocol): + """Production runtime wiring for ``core.workflow.file``.""" + + @property + def files_url(self) -> str: + return dify_config.FILES_URL + + @property + def internal_files_url(self) -> str | None: + return dify_config.INTERNAL_FILES_URL + + @property + def secret_key(self) -> str: + return dify_config.SECRET_KEY + + @property + def files_access_timeout(self) -> int: + return dify_config.FILES_ACCESS_TIMEOUT + + @property + def multimodal_send_format(self) -> str: + return dify_config.MULTIMODAL_SEND_FORMAT + + def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol: + return ssrf_proxy.get(url, follow_redirects=follow_redirects) + + def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator: + return storage.load(path, stream=stream) + + def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: + return sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external) + + +def bind_dify_workflow_file_runtime() -> None: + set_workflow_file_runtime(DifyWorkflowFileRuntime()) diff --git a/api/core/app/workflow/node_factory.py b/api/core/app/workflow/node_factory.py index 18db750d28..bd58bcb6b0 100644 --- a/api/core/app/workflow/node_factory.py +++ b/api/core/app/workflow/node_factory.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, final from typing_extensions import override from configs import dify_config -from core.file.file_manager import file_manager from core.helper.code_executor.code_executor import CodeExecutor from core.helper.code_executor.code_node_provider import CodeNodeProvider from core.helper.ssrf_proxy import ssrf_proxy @@ -12,6 +11,7 @@ from core.rag.retrieval.dataset_retrieval import DatasetRetrieval from core.tools.tool_file_manager import ToolFileManager from core.workflow.entities.graph_config import NodeConfigDict from core.workflow.enums import NodeType +from core.workflow.file.file_manager import file_manager from core.workflow.graph.graph import NodeFactory from core.workflow.nodes.base.node import Node from core.workflow.nodes.code.code_node import CodeNode diff --git a/api/core/datasource/datasource_file_manager.py b/api/core/datasource/datasource_file_manager.py index 0c50c2f980..f67bfb6ead 100644 --- a/api/core/datasource/datasource_file_manager.py +++ b/api/core/datasource/datasource_file_manager.py @@ -213,6 +213,6 @@ class DatasourceFileManager: # init tool_file_parser -# from core.file.datasource_file_parser import datasource_file_manager +# from core.workflow.file.datasource_file_parser import datasource_file_manager # # datasource_file_manager["manager"] = DatasourceFileManager diff --git a/api/core/datasource/utils/message_transformer.py b/api/core/datasource/utils/message_transformer.py index d0a9eb5e74..ab3302bd6e 100644 --- a/api/core/datasource/utils/message_transformer.py +++ b/api/core/datasource/utils/message_transformer.py @@ -3,8 +3,8 @@ from collections.abc import Generator from mimetypes import guess_extension, guess_type from core.datasource.entities.datasource_entities import DatasourceMessage -from core.file import File, FileTransferMethod, FileType from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file import File, FileTransferMethod, FileType from models.tools import ToolFile logger = logging.getLogger(__name__) diff --git a/api/core/entities/mcp_provider.py b/api/core/entities/mcp_provider.py index 135d2a4945..5902c03e27 100644 --- a/api/core/entities/mcp_provider.py +++ b/api/core/entities/mcp_provider.py @@ -10,12 +10,12 @@ from pydantic import BaseModel from configs import dify_config from core.entities.provider_entities import BasicProviderConfig -from core.file import helpers as file_helpers from core.helper import encrypter from core.helper.provider_cache import NoOpProviderCredentialCache from core.mcp.types import OAuthClientInformation, OAuthClientMetadata, OAuthTokens from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolProviderType +from core.workflow.file import helpers as file_helpers if TYPE_CHECKING: from models.tools import MCPToolProvider diff --git a/api/core/file/tool_file_parser.py b/api/core/file/tool_file_parser.py deleted file mode 100644 index 4c8e7282b8..0000000000 --- a/api/core/file/tool_file_parser.py +++ /dev/null @@ -1,12 +0,0 @@ -from collections.abc import Callable -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from core.tools.tool_file_manager import ToolFileManager - -_tool_file_manager_factory: Callable[[], "ToolFileManager"] | None = None - - -def set_tool_file_manager_factory(factory: Callable[[], "ToolFileManager"]): - global _tool_file_manager_factory - _tool_file_manager_factory = factory diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 3ebbb60f85..2b78a705c9 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -4,7 +4,6 @@ from sqlalchemy import select from sqlalchemy.orm import sessionmaker from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.file import file_manager from core.model_manager import ModelInstance from core.model_runtime.entities import ( AssistantPromptMessage, @@ -16,6 +15,7 @@ from core.model_runtime.entities import ( ) from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes from core.prompt.utils.extract_thread_messages import extract_thread_messages +from core.workflow.file import file_manager from extensions.ext_database import db from factories import file_factory from models.model import AppMode, Conversation, Message, MessageFile diff --git a/api/core/plugin/utils/converter.py b/api/core/plugin/utils/converter.py index 6876285b31..3fe1b84dfa 100644 --- a/api/core/plugin/utils/converter.py +++ b/api/core/plugin/utils/converter.py @@ -1,7 +1,7 @@ from typing import Any -from core.file.models import File from core.tools.entities.tool_entities import ToolSelector +from core.workflow.file.models import File def convert_parameters_to_plugin_format(parameters: dict[str, Any]) -> dict[str, Any]: diff --git a/api/core/prompt/advanced_prompt_transform.py b/api/core/prompt/advanced_prompt_transform.py index d74b2bddf5..fd1b7d838c 100644 --- a/api/core/prompt/advanced_prompt_transform.py +++ b/api/core/prompt/advanced_prompt_transform.py @@ -2,8 +2,6 @@ from collections.abc import Mapping, Sequence from typing import cast from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity -from core.file import file_manager -from core.file.models import File from core.helper.code_executor.jinja2.jinja2_formatter import Jinja2Formatter from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities import ( @@ -18,6 +16,8 @@ from core.model_runtime.entities.message_entities import ImagePromptMessageConte from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig from core.prompt.prompt_transform import PromptTransform from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from core.workflow.file import file_manager +from core.workflow.file.models import File from core.workflow.runtime import VariablePool diff --git a/api/core/prompt/simple_prompt_transform.py b/api/core/prompt/simple_prompt_transform.py index f072092ea7..d6abbaaa69 100644 --- a/api/core/prompt/simple_prompt_transform.py +++ b/api/core/prompt/simple_prompt_transform.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, cast from core.app.app_config.entities import PromptTemplateEntity from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity -from core.file import file_manager from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.message_entities import ( ImagePromptMessageContent, @@ -19,10 +18,11 @@ from core.model_runtime.entities.message_entities import ( from core.prompt.entities.advanced_prompt_entities import MemoryConfig from core.prompt.prompt_transform import PromptTransform from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from core.workflow.file import file_manager from models.model import AppMode if TYPE_CHECKING: - from core.file.models import File + from core.workflow.file.models import File class ModelMode(StrEnum): diff --git a/api/core/rag/index_processor/processor/paragraph_index_processor.py b/api/core/rag/index_processor/processor/paragraph_index_processor.py index 41d7656f8a..3b42560fd6 100644 --- a/api/core/rag/index_processor/processor/paragraph_index_processor.py +++ b/api/core/rag/index_processor/processor/paragraph_index_processor.py @@ -9,7 +9,6 @@ from typing import Any, cast logger = logging.getLogger(__name__) from core.entities.knowledge_entities import PreviewDetail -from core.file import File, FileTransferMethod, FileType, file_manager from core.llm_generator.prompts import DEFAULT_GENERATOR_SUMMARY_PROMPT from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage @@ -35,6 +34,7 @@ from core.rag.index_processor.index_processor_base import BaseIndexProcessor from core.rag.models.document import AttachmentDocument, Document, MultimodalGeneralStructureChunk from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.tools.utils.text_processing_utils import remove_leading_symbols +from core.workflow.file import File, FileTransferMethod, FileType, file_manager from core.workflow.nodes.llm import llm_utils from extensions.ext_database import db from factories.file_factory import build_from_mapping diff --git a/api/core/rag/models/document.py b/api/core/rag/models/document.py index 611fad9a18..48639bf4c8 100644 --- a/api/core/rag/models/document.py +++ b/api/core/rag/models/document.py @@ -4,7 +4,7 @@ from typing import Any from pydantic import BaseModel, Field -from core.file import File +from core.workflow.file import File class ChildDocument(BaseModel): diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index a8133aa556..cfea8d114a 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -23,7 +23,6 @@ from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCa from core.db.session_factory import session_factory from core.entities.agent_entities import PlanningStrategy from core.entities.model_entities import ModelStatus -from core.file import File, FileTransferMethod, FileType from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage @@ -61,6 +60,7 @@ from core.rag.retrieval.template_prompts import ( ) from core.tools.signature import sign_upload_file from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.nodes.knowledge_retrieval import exc from core.workflow.repositories.rag_retrieval_protocol import ( KnowledgeRetrievalRequest, diff --git a/api/core/tools/builtin_tool/providers/audio/tools/asr.py b/api/core/tools/builtin_tool/providers/audio/tools/asr.py index af9b5b31c2..2c1e9fb555 100644 --- a/api/core/tools/builtin_tool/providers/audio/tools/asr.py +++ b/api/core/tools/builtin_tool/providers/audio/tools/asr.py @@ -2,14 +2,14 @@ import io from collections.abc import Generator from typing import Any -from core.file.enums import FileType -from core.file.file_manager import download from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType from core.plugin.entities.parameters import PluginParameterOption from core.tools.builtin_tool.tool import BuiltinTool from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter +from core.workflow.file.enums import FileType +from core.workflow.file.file_manager import download from services.model_provider_service import ModelProviderService diff --git a/api/core/tools/custom_tool/tool.py b/api/core/tools/custom_tool/tool.py index 54c266ffcc..afa2ddffed 100644 --- a/api/core/tools/custom_tool/tool.py +++ b/api/core/tools/custom_tool/tool.py @@ -7,13 +7,13 @@ from urllib.parse import urlencode import httpx -from core.file.file_manager import download from core.helper import ssrf_proxy from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.tool_bundle import ApiToolBundle from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolProviderType from core.tools.errors import ToolInvokeError, ToolParameterValidationError, ToolProviderCredentialValidationError +from core.workflow.file.file_manager import download API_TOOL_DEFAULT_TIMEOUT = ( int(getenv("API_TOOL_DEFAULT_CONNECT_TIMEOUT", "10")), diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py index 3f57a346cd..de476f6461 100644 --- a/api/core/tools/tool_engine.py +++ b/api/core/tools/tool_engine.py @@ -12,8 +12,6 @@ from yarl import URL from core.app.entities.app_invoke_entities import InvokeFrom from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler -from core.file import FileType -from core.file.models import FileTransferMethod from core.ops.ops_trace_manager import TraceQueueManager from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ( @@ -33,6 +31,8 @@ from core.tools.errors import ( ) from core.tools.utils.message_transformer import ToolFileMessageTransformer, safe_json_value from core.tools.workflow_as_tool.tool import WorkflowTool +from core.workflow.file import FileType +from core.workflow.file.models import FileTransferMethod from extensions.ext_database import db from models.enums import CreatorUserRole from models.model import Message, MessageFile diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 6289f1d335..ca0dc27f3d 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -243,7 +243,7 @@ class ToolFileManager: # init tool_file_parser -from core.file.tool_file_parser import set_tool_file_manager_factory +from core.workflow.file.tool_file_parser import set_tool_file_manager_factory def _factory() -> ToolFileManager: diff --git a/api/core/tools/utils/message_transformer.py b/api/core/tools/utils/message_transformer.py index df322eda1c..622cdcf73b 100644 --- a/api/core/tools/utils/message_transformer.py +++ b/api/core/tools/utils/message_transformer.py @@ -8,9 +8,9 @@ from uuid import UUID import numpy as np import pytz -from core.file import File, FileTransferMethod, FileType from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file import File, FileTransferMethod, FileType from libs.login import current_user from models import Account diff --git a/api/core/tools/workflow_as_tool/tool.py b/api/core/tools/workflow_as_tool/tool.py index 01fa5de31e..b2606009a6 100644 --- a/api/core/tools/workflow_as_tool/tool.py +++ b/api/core/tools/workflow_as_tool/tool.py @@ -8,7 +8,6 @@ from typing import Any, cast from sqlalchemy import select from core.db.session_factory import session_factory -from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod from core.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadata from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime @@ -19,6 +18,7 @@ from core.tools.entities.tool_entities import ( ToolProviderType, ) from core.tools.errors import ToolInvokeError +from core.workflow.file import FILE_MODEL_IDENTITY, File, FileTransferMethod from factories.file_factory import build_from_mapping from models import Account, Tenant from models.model import App, EndUser diff --git a/api/core/variables/segments.py b/api/core/variables/segments.py index 8330f1fe19..64bba7dbe2 100644 --- a/api/core/variables/segments.py +++ b/api/core/variables/segments.py @@ -5,7 +5,7 @@ from typing import Annotated, Any, TypeAlias from pydantic import BaseModel, ConfigDict, Discriminator, Tag, field_validator -from core.file import File +from core.workflow.file import File from .types import SegmentType diff --git a/api/core/variables/types.py b/api/core/variables/types.py index 13b926c978..596905c26d 100644 --- a/api/core/variables/types.py +++ b/api/core/variables/types.py @@ -4,7 +4,7 @@ from collections.abc import Mapping from enum import StrEnum from typing import TYPE_CHECKING, Any -from core.file.models import File +from core.workflow.file.models import File if TYPE_CHECKING: pass diff --git a/api/core/file/__init__.py b/api/core/workflow/file/__init__.py similarity index 100% rename from api/core/file/__init__.py rename to api/core/workflow/file/__init__.py diff --git a/api/core/file/constants.py b/api/core/workflow/file/constants.py similarity index 100% rename from api/core/file/constants.py rename to api/core/workflow/file/constants.py diff --git a/api/core/file/enums.py b/api/core/workflow/file/enums.py similarity index 100% rename from api/core/file/enums.py rename to api/core/workflow/file/enums.py diff --git a/api/core/file/file_manager.py b/api/core/workflow/file/file_manager.py similarity index 64% rename from api/core/file/file_manager.py rename to api/core/workflow/file/file_manager.py index 9945d7c1ab..a7719400d9 100644 --- a/api/core/file/file_manager.py +++ b/api/core/workflow/file/file_manager.py @@ -1,8 +1,8 @@ +from __future__ import annotations + import base64 from collections.abc import Mapping -from configs import dify_config -from core.helper import ssrf_proxy from core.model_runtime.entities import ( AudioPromptMessageContent, DocumentPromptMessageContent, @@ -11,12 +11,11 @@ from core.model_runtime.entities import ( VideoPromptMessageContent, ) from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes -from core.tools.signature import sign_tool_file -from extensions.ext_storage import storage from . import helpers from .enums import FileAttribute from .models import File, FileTransferMethod, FileType +from .runtime import get_workflow_file_runtime def get_attr(*, file: File, attr: FileAttribute): @@ -45,26 +44,7 @@ def to_prompt_message_content( *, image_detail_config: ImagePromptMessageContent.DETAIL | None = None, ) -> PromptMessageContentUnionTypes: - """ - Convert a file to prompt message content. - - This function converts files to their appropriate prompt message content types. - For supported file types (IMAGE, AUDIO, VIDEO, DOCUMENT), it creates the - corresponding message content with proper encoding/URL. - - For unsupported file types, instead of raising an error, it returns a - TextPromptMessageContent with a descriptive message about the file. - - Args: - f: The file to convert - image_detail_config: Optional detail configuration for image files - - Returns: - PromptMessageContentUnionTypes: The appropriate message content type - - Raises: - ValueError: If file extension or mime_type is missing - """ + """Convert a file to prompt message content.""" if f.extension is None: raise ValueError("Missing file extension") if f.mime_type is None: @@ -77,15 +57,13 @@ def to_prompt_message_content( FileType.DOCUMENT: DocumentPromptMessageContent, } - # Check if file type is supported if f.type not in prompt_class_map: - # For unsupported file types, return a text description return TextPromptMessageContent(data=f"[Unsupported file type: {f.filename} ({f.type.value})]") - # Process supported file types + send_format = get_workflow_file_runtime().multimodal_send_format params = { - "base64_data": _get_encoded_string(f) if dify_config.MULTIMODAL_SEND_FORMAT == "base64" else "", - "url": _to_url(f) if dify_config.MULTIMODAL_SEND_FORMAT == "url" else "", + "base64_data": _get_encoded_string(f) if send_format == "base64" else "", + "url": _to_url(f) if send_format == "url" else "", "format": f.extension.removeprefix("."), "mime_type": f.mime_type, "filename": f.filename or "", @@ -96,7 +74,7 @@ def to_prompt_message_content( return prompt_class_map[f.type].model_validate(params) -def download(f: File, /): +def download(f: File, /) -> bytes: if f.transfer_method in ( FileTransferMethod.TOOL_FILE, FileTransferMethod.LOCAL_FILE, @@ -106,39 +84,26 @@ def download(f: File, /): elif f.transfer_method == FileTransferMethod.REMOTE_URL: if f.remote_url is None: raise ValueError("Missing file remote_url") - response = ssrf_proxy.get(f.remote_url, follow_redirects=True) + response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True) response.raise_for_status() return response.content raise ValueError(f"unsupported transfer method: {f.transfer_method}") -def _download_file_content(path: str, /): - """ - Download and return the contents of a file as bytes. - - This function loads the file from storage and ensures it's in bytes format. - - Args: - path (str): The path to the file in storage. - - Returns: - bytes: The contents of the file as a bytes object. - - Raises: - ValueError: If the loaded file is not a bytes object. - """ - data = storage.load(path, stream=False) +def _download_file_content(path: str, /) -> bytes: + """Download and return a file from storage as bytes.""" + data = get_workflow_file_runtime().storage_load(path, stream=False) if not isinstance(data, bytes): raise ValueError(f"file {path} is not a bytes object") return data -def _get_encoded_string(f: File, /): +def _get_encoded_string(f: File, /) -> str: match f.transfer_method: case FileTransferMethod.REMOTE_URL: if f.remote_url is None: raise ValueError("Missing file remote_url") - response = ssrf_proxy.get(f.remote_url, follow_redirects=True) + response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True) response.raise_for_status() data = response.content case FileTransferMethod.LOCAL_FILE: @@ -148,8 +113,7 @@ def _get_encoded_string(f: File, /): case FileTransferMethod.DATASOURCE_FILE: data = _download_file_content(f.storage_key) - encoded_string = base64.b64encode(data).decode("utf-8") - return encoded_string + return base64.b64encode(data).decode("utf-8") def _to_url(f: File, /): @@ -162,21 +126,15 @@ def _to_url(f: File, /): raise ValueError("Missing file related_id") return f.remote_url or helpers.get_signed_file_url(upload_file_id=f.related_id) elif f.transfer_method == FileTransferMethod.TOOL_FILE: - # add sign url if f.related_id is None or f.extension is None: raise ValueError("Missing file related_id or extension") - return sign_tool_file(tool_file_id=f.related_id, extension=f.extension) + return helpers.get_signed_tool_file_url(tool_file_id=f.related_id, extension=f.extension) else: raise ValueError(f"Unsupported transfer method: {f.transfer_method}") class FileManager: - """ - Adapter exposing file manager helpers behind FileManagerProtocol. - - This is intentionally a thin wrapper over the existing module-level functions so callers can inject it - where a protocol-typed file manager is expected. - """ + """Adapter exposing file manager helpers behind FileManagerProtocol.""" def download(self, f: File, /) -> bytes: return download(f) diff --git a/api/core/file/helpers.py b/api/core/workflow/file/helpers.py similarity index 65% rename from api/core/file/helpers.py rename to api/core/workflow/file/helpers.py index 2ac483673a..310cb1310b 100644 --- a/api/core/file/helpers.py +++ b/api/core/workflow/file/helpers.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import base64 import hashlib import hmac @@ -5,20 +7,21 @@ import os import time import urllib.parse -from configs import dify_config +from .runtime import get_workflow_file_runtime -def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external: bool = True) -> str: - base_url = dify_config.FILES_URL if for_external else (dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL) +def get_signed_file_url(upload_file_id: str, as_attachment: bool = False, for_external: bool = True) -> str: + runtime = get_workflow_file_runtime() + base_url = runtime.files_url if for_external else (runtime.internal_files_url or runtime.files_url) url = f"{base_url}/files/{upload_file_id}/file-preview" timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - key = dify_config.SECRET_KEY.encode() + key = runtime.secret_key.encode() msg = f"file-preview|{upload_file_id}|{timestamp}|{nonce}" sign = hmac.new(key, msg.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() - query = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign} + query: dict[str, str] = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign} if as_attachment: query["as_attachment"] = "true" query_string = urllib.parse.urlencode(query) @@ -27,57 +30,63 @@ def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external: def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str, user_id: str) -> str: - # Plugin access should use internal URL for Docker network communication - base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL + runtime = get_workflow_file_runtime() + # Plugin access should use internal URL for Docker network communication. + base_url = runtime.internal_files_url or runtime.files_url url = f"{base_url}/files/upload/for-plugin" timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - key = dify_config.SECRET_KEY.encode() + key = runtime.secret_key.encode() msg = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}" sign = hmac.new(key, msg.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}" +def get_signed_tool_file_url(tool_file_id: str, extension: str, for_external: bool = True) -> str: + runtime = get_workflow_file_runtime() + return runtime.sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external) + + def verify_plugin_file_signature( *, filename: str, mimetype: str, tenant_id: str, user_id: str, timestamp: str, nonce: str, sign: str ) -> bool: + runtime = get_workflow_file_runtime() data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}" - secret_key = dify_config.SECRET_KEY.encode() + secret_key = runtime.secret_key.encode() recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() - # verify signature if sign != recalculated_encoded_sign: return False current_time = int(time.time()) - return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT + return current_time - int(timestamp) <= runtime.files_access_timeout def verify_image_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool: + runtime = get_workflow_file_runtime() data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = dify_config.SECRET_KEY.encode() + secret_key = runtime.secret_key.encode() recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() - # verify signature if sign != recalculated_encoded_sign: return False current_time = int(time.time()) - return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT + return current_time - int(timestamp) <= runtime.files_access_timeout def verify_file_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool: + runtime = get_workflow_file_runtime() data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = dify_config.SECRET_KEY.encode() + secret_key = runtime.secret_key.encode() recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() - # verify signature if sign != recalculated_encoded_sign: return False current_time = int(time.time()) - return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT + return current_time - int(timestamp) <= runtime.files_access_timeout diff --git a/api/core/file/models.py b/api/core/workflow/file/models.py similarity index 90% rename from api/core/file/models.py rename to api/core/workflow/file/models.py index 6324523b22..cd7d3edde8 100644 --- a/api/core/file/models.py +++ b/api/core/workflow/file/models.py @@ -1,16 +1,26 @@ +from __future__ import annotations + from collections.abc import Mapping, Sequence from typing import Any from pydantic import BaseModel, Field, model_validator from core.model_runtime.entities.message_entities import ImagePromptMessageContent -from core.tools.signature import sign_tool_file from . import helpers from .constants import FILE_MODEL_IDENTITY from .enums import FileTransferMethod, FileType +def sign_tool_file(*, tool_file_id: str, extension: str, for_external: bool = True) -> str: + """Compatibility shim for tests and legacy callers patching ``models.sign_tool_file``.""" + return helpers.get_signed_tool_file_url( + tool_file_id=tool_file_id, + extension=extension, + for_external=for_external, + ) + + class ImageConfig(BaseModel): """ NOTE: This part of validation is deprecated, but still used in app features "Image Upload". @@ -122,7 +132,11 @@ class File(BaseModel): elif self.transfer_method in [FileTransferMethod.TOOL_FILE, FileTransferMethod.DATASOURCE_FILE]: assert self.related_id is not None assert self.extension is not None - return sign_tool_file(tool_file_id=self.related_id, extension=self.extension, for_external=for_external) + return sign_tool_file( + tool_file_id=self.related_id, + extension=self.extension, + for_external=for_external, + ) return None def to_plugin_parameter(self) -> dict[str, Any]: @@ -137,7 +151,7 @@ class File(BaseModel): } @model_validator(mode="after") - def validate_after(self): + def validate_after(self) -> File: match self.transfer_method: case FileTransferMethod.REMOTE_URL: if not self.remote_url: @@ -160,5 +174,5 @@ class File(BaseModel): return self._storage_key @storage_key.setter - def storage_key(self, value: str): + def storage_key(self, value: str) -> None: self._storage_key = value diff --git a/api/core/workflow/file/protocols.py b/api/core/workflow/file/protocols.py new file mode 100644 index 0000000000..8d923148e0 --- /dev/null +++ b/api/core/workflow/file/protocols.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import Protocol + + +class HttpResponseProtocol(Protocol): + """Subset of response behavior needed by workflow file helpers.""" + + @property + def content(self) -> bytes: ... + + def raise_for_status(self) -> object: ... + + +class WorkflowFileRuntimeProtocol(Protocol): + """Runtime dependencies required by ``core.workflow.file``. + + Implementations are expected to be provided by integration layers (for example, + ``core.app.workflow.file_runtime``) so the workflow package avoids importing + application infrastructure modules directly. + """ + + @property + def files_url(self) -> str: ... + + @property + def internal_files_url(self) -> str | None: ... + + @property + def secret_key(self) -> str: ... + + @property + def files_access_timeout(self) -> int: ... + + @property + def multimodal_send_format(self) -> str: ... + + def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol: ... + + def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator: ... + + def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: ... diff --git a/api/core/workflow/file/runtime.py b/api/core/workflow/file/runtime.py new file mode 100644 index 0000000000..94253e0255 --- /dev/null +++ b/api/core/workflow/file/runtime.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from collections.abc import Generator +from typing import NoReturn + +from .protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol + + +class WorkflowFileRuntimeNotConfiguredError(RuntimeError): + """Raised when workflow file runtime dependencies were not configured.""" + + +class _UnconfiguredWorkflowFileRuntime(WorkflowFileRuntimeProtocol): + def _raise(self) -> NoReturn: + raise WorkflowFileRuntimeNotConfiguredError( + "workflow file runtime is not configured, call set_workflow_file_runtime(...) first" + ) + + @property + def files_url(self) -> str: + self._raise() + + @property + def internal_files_url(self) -> str | None: + self._raise() + + @property + def secret_key(self) -> str: + self._raise() + + @property + def files_access_timeout(self) -> int: + self._raise() + + @property + def multimodal_send_format(self) -> str: + self._raise() + + def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol: + self._raise() + + def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator: + self._raise() + + def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: + self._raise() + + +_runtime: WorkflowFileRuntimeProtocol = _UnconfiguredWorkflowFileRuntime() + + +def set_workflow_file_runtime(runtime: WorkflowFileRuntimeProtocol) -> None: + global _runtime + _runtime = runtime + + +def get_workflow_file_runtime() -> WorkflowFileRuntimeProtocol: + return _runtime diff --git a/api/core/workflow/file/tool_file_parser.py b/api/core/workflow/file/tool_file_parser.py new file mode 100644 index 0000000000..2d7a3d43df --- /dev/null +++ b/api/core/workflow/file/tool_file_parser.py @@ -0,0 +1,9 @@ +from collections.abc import Callable +from typing import Any + +_tool_file_manager_factory: Callable[[], Any] | None = None + + +def set_tool_file_manager_factory(factory: Callable[[], Any]): + global _tool_file_manager_factory + _tool_file_manager_factory = factory diff --git a/api/core/workflow/node_events/node.py b/api/core/workflow/node_events/node.py index 9c76b7d7c2..2468bd0ac3 100644 --- a/api/core/workflow/node_events/node.py +++ b/api/core/workflow/node_events/node.py @@ -3,10 +3,10 @@ from datetime import datetime from pydantic import Field -from core.file import File from core.model_runtime.entities.llm_entities import LLMUsage from core.rag.entities.citation_metadata import RetrievalSourceMetadata from core.workflow.entities.pause_reason import PauseReason +from core.workflow.file import File from core.workflow.node_events import NodeRunResult from .base import NodeEventBase diff --git a/api/core/workflow/nodes/agent/agent_node.py b/api/core/workflow/nodes/agent/agent_node.py index e195aebe6d..5c39a67102 100644 --- a/api/core/workflow/nodes/agent/agent_node.py +++ b/api/core/workflow/nodes/agent/agent_node.py @@ -11,7 +11,6 @@ from sqlalchemy.orm import Session from core.agent.entities import AgentToolEntity from core.agent.plugin_entities import AgentStrategyParameter -from core.file import File, FileTransferMethod from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadata @@ -33,6 +32,7 @@ from core.workflow.enums import ( WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) +from core.workflow.file import File, FileTransferMethod from core.workflow.node_events import ( AgentLogEvent, NodeEventBase, diff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.py index a732a70417..80869ac7f7 100644 --- a/api/core/workflow/nodes/datasource/datasource_node.py +++ b/api/core/workflow/nodes/datasource/datasource_node.py @@ -14,13 +14,13 @@ from core.datasource.entities.datasource_entities import ( from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin from core.datasource.utils.message_transformer import DatasourceFileMessageTransformer -from core.file import File -from core.file.enums import FileTransferMethod, FileType from core.plugin.impl.exc import PluginDaemonClientSideError from core.variables.segments import ArrayAnySegment from core.variables.variables import ArrayAnyVariable from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey +from core.workflow.file import File +from core.workflow.file.enums import FileTransferMethod, FileType from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser diff --git a/api/core/workflow/nodes/document_extractor/node.py b/api/core/workflow/nodes/document_extractor/node.py index 14ebd1f9ae..0a14b81633 100644 --- a/api/core/workflow/nodes/document_extractor/node.py +++ b/api/core/workflow/nodes/document_extractor/node.py @@ -21,11 +21,11 @@ from docx.table import Table from docx.text.paragraph import Paragraph from configs import dify_config -from core.file import File, FileTransferMethod, file_manager from core.helper import ssrf_proxy from core.variables import ArrayFileSegment from core.variables.segments import ArrayStringSegment, FileSegment from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod, file_manager from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.node import Node diff --git a/api/core/workflow/nodes/http_request/executor.py b/api/core/workflow/nodes/http_request/executor.py index 7de8216562..1e6e14482b 100644 --- a/api/core/workflow/nodes/http_request/executor.py +++ b/api/core/workflow/nodes/http_request/executor.py @@ -11,10 +11,10 @@ import httpx from json_repair import repair_json from configs import dify_config -from core.file.enums import FileTransferMethod -from core.file.file_manager import file_manager as default_file_manager from core.helper.ssrf_proxy import ssrf_proxy from core.variables.segments import ArrayFileSegment, FileSegment +from core.workflow.file.enums import FileTransferMethod +from core.workflow.file.file_manager import file_manager as default_file_manager from core.workflow.runtime import VariablePool from ..protocols import FileManagerProtocol, HttpClientProtocol diff --git a/api/core/workflow/nodes/http_request/node.py b/api/core/workflow/nodes/http_request/node.py index 480482375f..c9aca1b992 100644 --- a/api/core/workflow/nodes/http_request/node.py +++ b/api/core/workflow/nodes/http_request/node.py @@ -4,12 +4,12 @@ from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Any from configs import dify_config -from core.file import File, FileTransferMethod -from core.file.file_manager import file_manager as default_file_manager from core.helper.ssrf_proxy import ssrf_proxy from core.tools.tool_file_manager import ToolFileManager from core.variables.segments import ArrayFileSegment from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod +from core.workflow.file.file_manager import file_manager as default_file_manager from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base import variable_template_parser from core.workflow.nodes.base.entities import VariableSelector diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 65c2792355..b25c3a3d29 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -30,7 +30,7 @@ from .exc import ( ) if TYPE_CHECKING: - from core.file.models import File + from core.workflow.file.models import File from core.workflow.runtime import GraphRuntimeState logger = logging.getLogger(__name__) diff --git a/api/core/workflow/nodes/list_operator/node.py b/api/core/workflow/nodes/list_operator/node.py index 235f5b9c52..3978a79550 100644 --- a/api/core/workflow/nodes/list_operator/node.py +++ b/api/core/workflow/nodes/list_operator/node.py @@ -1,10 +1,10 @@ from collections.abc import Callable, Sequence from typing import Any, TypeAlias, TypeVar -from core.file import File from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment from core.variables.segments import ArrayAnySegment, ArrayBooleanSegment, ArraySegment from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.file import File from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.node import Node diff --git a/api/core/workflow/nodes/llm/file_saver.py b/api/core/workflow/nodes/llm/file_saver.py index 3f32fa894a..3c06ab7d81 100644 --- a/api/core/workflow/nodes/llm/file_saver.py +++ b/api/core/workflow/nodes/llm/file_saver.py @@ -4,10 +4,10 @@ import typing as tp from sqlalchemy import Engine from constants.mimetypes import DEFAULT_EXTENSION, DEFAULT_MIME_TYPE -from core.file import File, FileTransferMethod, FileType from core.helper import ssrf_proxy from core.tools.signature import sign_tool_file from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file import File, FileTransferMethod, FileType from extensions.ext_database import db as global_db diff --git a/api/core/workflow/nodes/llm/llm_utils.py b/api/core/workflow/nodes/llm/llm_utils.py index 01e25cbf5c..78fad37659 100644 --- a/api/core/workflow/nodes/llm/llm_utils.py +++ b/api/core/workflow/nodes/llm/llm_utils.py @@ -7,7 +7,6 @@ from sqlalchemy.orm import Session from configs import dify_config from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity from core.entities.provider_entities import ProviderQuotaType, QuotaUnit -from core.file.models import File from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.llm_entities import LLMUsage @@ -16,6 +15,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large from core.prompt.entities.advanced_prompt_entities import MemoryConfig from core.variables.segments import ArrayAnySegment, ArrayFileSegment, FileSegment, NoneSegment, StringSegment from core.workflow.enums import SystemVariableKey +from core.workflow.file.models import File from core.workflow.nodes.llm.entities import ModelConfig from core.workflow.runtime import VariablePool from extensions.ext_database import db diff --git a/api/core/workflow/nodes/llm/node.py b/api/core/workflow/nodes/llm/node.py index beccf79344..49ae5d16c7 100644 --- a/api/core/workflow/nodes/llm/node.py +++ b/api/core/workflow/nodes/llm/node.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Literal from sqlalchemy import select from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity -from core.file import File, FileTransferMethod, FileType, file_manager from core.helper.code_executor import CodeExecutor, CodeLanguage from core.llm_generator.output_parser.errors import OutputParserError from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output @@ -65,6 +64,7 @@ from core.workflow.enums import ( WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) +from core.workflow.file import File, FileTransferMethod, FileType, file_manager from core.workflow.node_events import ( ModelInvokeCompletedEvent, NodeEventBase, @@ -101,7 +101,7 @@ from .exc import ( from .file_saver import FileSaverImpl, LLMFileSaver if TYPE_CHECKING: - from core.file.models import File + from core.workflow.file.models import File from core.workflow.runtime import GraphRuntimeState logger = logging.getLogger(__name__) diff --git a/api/core/workflow/nodes/loop/loop_node.py b/api/core/workflow/nodes/loop/loop_node.py index 84a9c29414..241a186a94 100644 --- a/api/core/workflow/nodes/loop/loop_node.py +++ b/api/core/workflow/nodes/loop/loop_node.py @@ -71,9 +71,9 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]): if self.node_data.loop_variables: value_processor: dict[Literal["constant", "variable"], Callable[[LoopVariableData], Segment | None]] = { "constant": lambda var: self._get_segment_for_constant(var.var_type, var.value), - "variable": lambda var: self.graph_runtime_state.variable_pool.get(var.value) - if isinstance(var.value, list) - else None, + "variable": lambda var: ( + self.graph_runtime_state.variable_pool.get(var.value) if isinstance(var.value, list) else None + ), } for loop_variable in self.node_data.loop_variables: if loop_variable.value_type not in value_processor: diff --git a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py index 08e0542d61..2f11a91b7e 100644 --- a/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py +++ b/api/core/workflow/nodes/parameter_extractor/parameter_extractor_node.py @@ -6,7 +6,6 @@ from collections.abc import Mapping, Sequence from typing import Any, cast from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity -from core.file import File from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities import ImagePromptMessageContent @@ -28,6 +27,7 @@ from core.prompt.simple_prompt_transform import ModelMode from core.prompt.utils.prompt_message_util import PromptMessageUtil from core.variables.types import ArrayValidation, SegmentType from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus +from core.workflow.file import File from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base import variable_template_parser from core.workflow.nodes.base.node import Node diff --git a/api/core/workflow/nodes/protocols.py b/api/core/workflow/nodes/protocols.py index 2ad39e0ab5..a1f3e20835 100644 --- a/api/core/workflow/nodes/protocols.py +++ b/api/core/workflow/nodes/protocols.py @@ -2,7 +2,7 @@ from typing import Any, Protocol import httpx -from core.file import File +from core.workflow.file import File class HttpClientProtocol(Protocol): diff --git a/api/core/workflow/nodes/question_classifier/question_classifier_node.py b/api/core/workflow/nodes/question_classifier/question_classifier_node.py index 4a3e8e56f8..6491e8e531 100644 --- a/api/core/workflow/nodes/question_classifier/question_classifier_node.py +++ b/api/core/workflow/nodes/question_classifier/question_classifier_node.py @@ -39,7 +39,7 @@ from .template_prompts import ( ) if TYPE_CHECKING: - from core.file.models import File + from core.workflow.file.models import File from core.workflow.runtime import GraphRuntimeState diff --git a/api/core/workflow/nodes/tool/tool_node.py b/api/core/workflow/nodes/tool/tool_node.py index 60d76db9b6..a7bf7d6642 100644 --- a/api/core/workflow/nodes/tool/tool_node.py +++ b/api/core/workflow/nodes/tool/tool_node.py @@ -5,7 +5,6 @@ from sqlalchemy import select from sqlalchemy.orm import Session from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler -from core.file import File, FileTransferMethod from core.model_runtime.entities.llm_entities import LLMUsage from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter @@ -20,6 +19,7 @@ from core.workflow.enums import ( WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus, ) +from core.workflow.file import File, FileTransferMethod from core.workflow.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent from core.workflow.nodes.base.node import Node from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser diff --git a/api/core/workflow/nodes/trigger_webhook/node.py b/api/core/workflow/nodes/trigger_webhook/node.py index ec8c4b8ee3..060afd6ae6 100644 --- a/api/core/workflow/nodes/trigger_webhook/node.py +++ b/api/core/workflow/nodes/trigger_webhook/node.py @@ -2,12 +2,12 @@ import logging from collections.abc import Mapping from typing import Any -from core.file import FileTransferMethod from core.variables.types import SegmentType from core.variables.variables import FileVariable from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus from core.workflow.enums import NodeExecutionType, NodeType +from core.workflow.file import FileTransferMethod from core.workflow.node_events import NodeRunResult from core.workflow.nodes.base.node import Node from factories import file_factory diff --git a/api/core/workflow/runtime/variable_pool.py b/api/core/workflow/runtime/variable_pool.py index c4b077fa69..0ba9d8b3a8 100644 --- a/api/core/workflow/runtime/variable_pool.py +++ b/api/core/workflow/runtime/variable_pool.py @@ -8,7 +8,6 @@ from typing import Annotated, Any, Union, cast from pydantic import BaseModel, Field -from core.file import File, FileAttribute, file_manager from core.variables import Segment, SegmentGroup, VariableBase from core.variables.consts import SELECTORS_LENGTH from core.variables.segments import FileSegment, ObjectSegment @@ -19,6 +18,7 @@ from core.workflow.constants import ( RAG_PIPELINE_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID, ) +from core.workflow.file import File, FileAttribute, file_manager from core.workflow.system_variable import SystemVariable from factories import variable_factory diff --git a/api/core/workflow/system_variable.py b/api/core/workflow/system_variable.py index 6946e3e6ab..4144f79b8a 100644 --- a/api/core/workflow/system_variable.py +++ b/api/core/workflow/system_variable.py @@ -7,8 +7,8 @@ from uuid import uuid4 from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator -from core.file.models import File from core.workflow.enums import SystemVariableKey +from core.workflow.file.models import File class SystemVariable(BaseModel): diff --git a/api/core/workflow/utils/condition/processor.py b/api/core/workflow/utils/condition/processor.py index c6070b83b8..c3f25a4d62 100644 --- a/api/core/workflow/utils/condition/processor.py +++ b/api/core/workflow/utils/condition/processor.py @@ -2,9 +2,9 @@ import json from collections.abc import Mapping, Sequence from typing import Literal, NamedTuple -from core.file import FileAttribute, file_manager from core.variables import ArrayFileSegment from core.variables.segments import ArrayBooleanSegment, BooleanSegment +from core.workflow.file import FileAttribute, file_manager from core.workflow.runtime import VariablePool from .entities import Condition, SubCondition, SupportedComparisonOperator diff --git a/api/core/workflow/workflow_entry.py b/api/core/workflow/workflow_entry.py index 4b1845cda2..29ffb8027f 100644 --- a/api/core/workflow/workflow_entry.py +++ b/api/core/workflow/workflow_entry.py @@ -9,10 +9,10 @@ from core.app.apps.exc import GenerateTaskStoppedError from core.app.entities.app_invoke_entities import InvokeFrom from core.app.workflow.layers.observability import ObservabilityLayer from core.app.workflow.node_factory import DifyNodeFactory -from core.file.models import File from core.workflow.constants import ENVIRONMENT_VARIABLE_NODE_ID from core.workflow.entities import GraphInitParams from core.workflow.errors import WorkflowNodeRunFailedError +from core.workflow.file.models import File from core.workflow.graph import Graph from core.workflow.graph_engine import GraphEngine, GraphEngineConfig from core.workflow.graph_engine.command_channels import InMemoryChannel diff --git a/api/core/workflow/workflow_type_encoder.py b/api/core/workflow/workflow_type_encoder.py index f1f549e1f8..93c6a31960 100644 --- a/api/core/workflow/workflow_type_encoder.py +++ b/api/core/workflow/workflow_type_encoder.py @@ -4,8 +4,8 @@ from typing import Any, overload from pydantic import BaseModel -from core.file.models import File from core.variables import Segment +from core.workflow.file.models import File class WorkflowRuntimeTypeConverter: diff --git a/api/extensions/ext_storage.py b/api/extensions/ext_storage.py index 6df0879694..db5a6e4812 100644 --- a/api/extensions/ext_storage.py +++ b/api/extensions/ext_storage.py @@ -94,6 +94,10 @@ class Storage: @overload def load(self, filename: str, /, *, stream: Literal[True]) -> Generator: ... + # Keep a bool fallback overload for callers that forward a runtime bool flag. + @overload + def load(self, filename: str, /, *, stream: bool = False) -> Union[bytes, Generator]: ... + def load(self, filename: str, /, *, stream: bool = False) -> Union[bytes, Generator]: if stream: return self.load_stream(filename) @@ -124,3 +128,6 @@ storage = Storage() def init_app(app: DifyApp): storage.init_app(app) + from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime + + bind_dify_workflow_file_runtime() diff --git a/api/extensions/otel/parser/base.py b/api/extensions/otel/parser/base.py index f4db26e840..c6589dd99f 100644 --- a/api/extensions/otel/parser/base.py +++ b/api/extensions/otel/parser/base.py @@ -9,9 +9,9 @@ from opentelemetry.trace import Span from opentelemetry.trace.status import Status, StatusCode from pydantic import BaseModel -from core.file.models import File from core.variables import Segment from core.workflow.enums import NodeType +from core.workflow.file.models import File from core.workflow.graph_events import GraphNodeEventBase from core.workflow.nodes.base.node import Node from extensions.otel.semconv.gen_ai import ChainAttributes, GenAIAttributes diff --git a/api/factories/file_factory.py b/api/factories/file_factory.py index 0be836c8f1..47396831fa 100644 --- a/api/factories/file_factory.py +++ b/api/factories/file_factory.py @@ -13,8 +13,8 @@ from sqlalchemy.orm import Session from werkzeug.http import parse_options_header from constants import AUDIO_EXTENSIONS, DOCUMENT_EXTENSIONS, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS -from core.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig, helpers from core.helper import ssrf_proxy +from core.workflow.file import File, FileBelongsTo, FileTransferMethod, FileType, FileUploadConfig, helpers from extensions.ext_database import db from models import MessageFile, ToolFile, UploadFile diff --git a/api/factories/variable_factory.py b/api/factories/variable_factory.py index 3f030ae127..a7cfb6a65e 100644 --- a/api/factories/variable_factory.py +++ b/api/factories/variable_factory.py @@ -3,7 +3,6 @@ from typing import Any, cast from uuid import uuid4 from configs import dify_config -from core.file import File from core.variables.exc import VariableError from core.variables.segments import ( ArrayAnySegment, @@ -44,6 +43,7 @@ from core.workflow.constants import ( CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, ) +from core.workflow.file import File class UnsupportedSegmentTypeError(Exception): diff --git a/api/fields/conversation_fields.py b/api/fields/conversation_fields.py index cda46f2339..faa3606f0e 100644 --- a/api/fields/conversation_fields.py +++ b/api/fields/conversation_fields.py @@ -5,7 +5,7 @@ from typing import Any, TypeAlias from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from core.file import File +from core.workflow.file import File JSONValue: TypeAlias = Any diff --git a/api/fields/member_fields.py b/api/fields/member_fields.py index 11d9a1a2fc..29b9e40242 100644 --- a/api/fields/member_fields.py +++ b/api/fields/member_fields.py @@ -5,7 +5,7 @@ from datetime import datetime from flask_restx import fields from pydantic import BaseModel, ConfigDict, computed_field, field_validator -from core.file import helpers as file_helpers +from core.workflow.file import helpers as file_helpers simple_account_fields = { "id": fields.String, diff --git a/api/fields/message_fields.py b/api/fields/message_fields.py index 77b26a7423..55bd0a5fbd 100644 --- a/api/fields/message_fields.py +++ b/api/fields/message_fields.py @@ -7,7 +7,7 @@ from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field, field_validator from core.entities.execution_extra_content import ExecutionExtraContentDomainModel -from core.file import File +from core.workflow.file import File from fields.conversation_fields import AgentThought, JSONValue, MessageFile JSONValueType: TypeAlias = JSONValue diff --git a/api/fields/raws.py b/api/fields/raws.py index 9bc6a12c78..33b47ba2c3 100644 --- a/api/fields/raws.py +++ b/api/fields/raws.py @@ -1,6 +1,6 @@ from flask_restx import fields -from core.file import File +from core.workflow.file import File class FilesContainedField(fields.Raw): diff --git a/api/libs/helper.py b/api/libs/helper.py index fb577b9c99..206bb8fd81 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -21,8 +21,8 @@ from pydantic.functional_validators import AfterValidator from configs import dify_config from core.app.features.rate_limiting.rate_limit import RateLimitGenerator -from core.file import helpers as file_helpers from core.model_runtime.utils.encoders import jsonable_encoder +from core.workflow.file import helpers as file_helpers from extensions.ext_redis import redis_client if TYPE_CHECKING: diff --git a/api/models/model.py b/api/models/model.py index e2a9bb70cf..4a95faf7f7 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -18,10 +18,10 @@ from sqlalchemy.orm import Mapped, Session, mapped_column from configs import dify_config from constants import DEFAULT_FILE_NUMBER_LIMITS -from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod -from core.file import helpers as file_helpers from core.tools.signature import sign_tool_file from core.workflow.enums import WorkflowExecutionStatus +from core.workflow.file import FILE_MODEL_IDENTITY, File, FileTransferMethod +from core.workflow.file import helpers as file_helpers from libs.helper import generate_string # type: ignore[import-not-found] from libs.uuid_utils import uuidv7 diff --git a/api/models/workflow.py b/api/models/workflow.py index 5e9e099ccd..c88a48632a 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -22,8 +22,6 @@ from sqlalchemy import ( from sqlalchemy.orm import Mapped, declared_attr, mapped_column from typing_extensions import deprecated -from core.file.constants import maybe_file_object -from core.file.models import File from core.variables import utils as variable_utils from core.variables.variables import FloatVariable, IntegerVariable, StringVariable from core.workflow.constants import ( @@ -33,6 +31,8 @@ from core.workflow.constants import ( from core.workflow.entities.graph_config import NodeConfigDict, NodeConfigDictAdapter from core.workflow.entities.pause_reason import HumanInputRequired, PauseReason, PauseReasonType, SchedulingPause from core.workflow.enums import NodeType, WorkflowExecutionStatus +from core.workflow.file.constants import maybe_file_object +from core.workflow.file.models import File from extensions.ext_storage import Storage from factories.variable_factory import TypeMismatchError, build_segment_with_type from libs.datetime_utils import naive_utc_now diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index b208e394b0..785e02a19a 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -18,7 +18,6 @@ from werkzeug.exceptions import Forbidden, NotFound from configs import dify_config from core.db.session_factory import session_factory from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError -from core.file import helpers as file_helpers from core.helper.name_generator import generate_incremental_name from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelFeature, ModelType @@ -26,6 +25,7 @@ from core.model_runtime.model_providers.__base.text_embedding_model import TextE from core.rag.index_processor.constant.built_in_field import BuiltInField from core.rag.index_processor.constant.index_type import IndexStructureType from core.rag.retrieval.retrieval_methods import RetrievalMethod +from core.workflow.file import helpers as file_helpers from enums.cloud_plan import CloudPlan from events.dataset_event import dataset_was_deleted from events.document_event import document_was_deleted diff --git a/api/services/file_service.py b/api/services/file_service.py index a0a99f3f82..da99a66bb9 100644 --- a/api/services/file_service.py +++ b/api/services/file_service.py @@ -19,8 +19,8 @@ from constants import ( IMAGE_EXTENSIONS, VIDEO_EXTENSIONS, ) -from core.file import helpers as file_helpers from core.rag.extractor.extract_processor import ExtractProcessor +from core.workflow.file import helpers as file_helpers from extensions.ext_database import db from extensions.ext_storage import storage from libs.datetime_utils import naive_utc_now diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 4159f5f8f4..edbc7e0cc8 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -15,10 +15,10 @@ from werkzeug.exceptions import RequestEntityTooLarge from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom -from core.file.models import FileTransferMethod from core.tools.tool_file_manager import ToolFileManager from core.variables.types import SegmentType from core.workflow.enums import NodeType +from core.workflow.file.models import FileTransferMethod from enums.quota_type import QuotaType from extensions.ext_database import db from extensions.ext_redis import redis_client diff --git a/api/services/variable_truncator.py b/api/services/variable_truncator.py index f973361341..056ea4d78a 100644 --- a/api/services/variable_truncator.py +++ b/api/services/variable_truncator.py @@ -6,7 +6,6 @@ from collections.abc import Mapping from typing import Any, Generic, TypeAlias, TypeVar, overload from configs import dify_config -from core.file.models import File from core.variables.segments import ( ArrayFileSegment, ArraySegment, @@ -20,6 +19,7 @@ from core.variables.segments import ( StringSegment, ) from core.variables.utils import dumps_with_segments +from core.workflow.file.models import File from core.workflow.nodes.variable_assigner.common.helpers import UpdatedVariable _MAX_DEPTH = 100 diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 067feb994f..809151b91a 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -13,12 +13,12 @@ from core.app.app_config.entities import ( from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager from core.app.apps.chat.app_config_manager import ChatAppConfigManager from core.app.apps.completion.app_config_manager import CompletionAppConfigManager -from core.file.models import FileUploadConfig from core.helper import encrypter from core.model_runtime.entities.llm_entities import LLMMode from core.model_runtime.utils.encoders import jsonable_encoder from core.prompt.simple_prompt_transform import SimplePromptTransform from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from core.workflow.file.models import FileUploadConfig from core.workflow.nodes import NodeType from events.app_event import app_was_created from extensions.ext_database import db diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index 70b0190231..991925ae6b 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -14,7 +14,6 @@ from sqlalchemy.sql.expression import and_, or_ from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom -from core.file.models import File from core.variables import Segment, StringSegment, VariableBase from core.variables.consts import SELECTORS_LENGTH from core.variables.segments import ( @@ -25,6 +24,7 @@ from core.variables.types import SegmentType from core.variables.utils import dumps_with_segments from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID from core.workflow.enums import SystemVariableKey +from core.workflow.file.models import File from core.workflow.nodes import NodeType from core.workflow.nodes.variable_assigner.common.helpers import get_updated_variables from core.workflow.variable_loader import VariableLoader diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 4e1e515de5..cff334a44a 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -13,7 +13,6 @@ from core.app.app_config.entities import VariableEntityType from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.app.entities.app_invoke_entities import InvokeFrom -from core.file import File from core.repositories import DifyCoreRepositoryFactory from core.repositories.human_input_repository import HumanInputFormRepositoryImpl from core.variables import VariableBase @@ -22,6 +21,7 @@ from core.workflow.entities import GraphInitParams, WorkflowNodeExecution from core.workflow.entities.pause_reason import HumanInputRequired from core.workflow.enums import ErrorStrategy, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus from core.workflow.errors import WorkflowNodeRunFailedError +from core.workflow.file import File from core.workflow.graph_events import GraphNodeEventBase, NodeRunFailedEvent, NodeRunSucceededEvent from core.workflow.node_events import NodeRunResult from core.workflow.nodes import NodeType diff --git a/api/tests/conftest.py b/api/tests/conftest.py new file mode 100644 index 0000000000..e526685433 --- /dev/null +++ b/api/tests/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime + + +@pytest.fixture(autouse=True) +def _bind_workflow_file_runtime() -> None: + bind_dify_workflow_file_runtime() diff --git a/api/tests/integration_tests/factories/test_storage_key_loader.py b/api/tests/integration_tests/factories/test_storage_key_loader.py index bc64fda9c2..16a66bc3f1 100644 --- a/api/tests/integration_tests/factories/test_storage_key_loader.py +++ b/api/tests/integration_tests/factories/test_storage_key_loader.py @@ -6,7 +6,7 @@ from uuid import uuid4 import pytest from sqlalchemy.orm import Session -from core.file import File, FileTransferMethod, FileType +from core.workflow.file import File, FileTransferMethod, FileType from extensions.ext_database import db from factories.file_factory import StorageKeyLoader from models import ToolFile, UploadFile diff --git a/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py b/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py index 21a792de06..3568a8b070 100644 --- a/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py +++ b/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py @@ -6,7 +6,7 @@ from uuid import uuid4 import pytest from sqlalchemy.orm import Session -from core.file import File, FileTransferMethod, FileType +from core.workflow.file import File, FileTransferMethod, FileType from extensions.ext_database import db from factories.file_factory import StorageKeyLoader from models import ToolFile, UploadFile diff --git a/api/tests/test_containers_integration_tests/services/test_agent_service.py b/api/tests/test_containers_integration_tests/services/test_agent_service.py index 6eedbd6cfa..fb6304a59e 100644 --- a/api/tests/test_containers_integration_tests/services/test_agent_service.py +++ b/api/tests/test_containers_integration_tests/services/test_agent_service.py @@ -841,7 +841,7 @@ class TestAgentService: app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) conversation, message = self._create_test_conversation_and_message(db_session_with_containers, app, account) - from core.file import FileTransferMethod, FileType + from core.workflow.file import FileTransferMethod, FileType from extensions.ext_database import db from models.enums import CreatorUserRole diff --git a/api/tests/unit_tests/controllers/console/app/workflow_draft_variables_test.py b/api/tests/unit_tests/controllers/console/app/workflow_draft_variables_test.py index c8de059109..ec35366d02 100644 --- a/api/tests/unit_tests/controllers/console/app/workflow_draft_variables_test.py +++ b/api/tests/unit_tests/controllers/console/app/workflow_draft_variables_test.py @@ -310,8 +310,8 @@ def test_workflow_node_variables_fields(): def test_workflow_file_variable_with_signed_url(): """Test that File type variables include signed URLs in API responses.""" - from core.file.enums import FileTransferMethod, FileType - from core.file.models import File + from core.workflow.file.enums import FileTransferMethod, FileType + from core.workflow.file.models import File # Create a File object with LOCAL_FILE transfer method (which generates signed URLs) test_file = File( @@ -368,8 +368,8 @@ def test_workflow_file_variable_with_signed_url(): def test_workflow_file_variable_remote_url(): """Test that File type variables with REMOTE_URL transfer method return the remote URL.""" - from core.file.enums import FileTransferMethod, FileType - from core.file.models import File + from core.workflow.file.enums import FileTransferMethod, FileType + from core.workflow.file.models import File # Create a File object with REMOTE_URL transfer method test_file = File( diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py index d5d7ee95c5..23aee22d63 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py @@ -49,8 +49,8 @@ def datasets_document_module(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(wraps, "account_initialization_required", _noop) # Bypass billing-related decorators used by other endpoints in this module. - monkeypatch.setattr(wraps, "cloud_edition_billing_resource_check", lambda *_args, **_kwargs: (lambda f: f)) - monkeypatch.setattr(wraps, "cloud_edition_billing_rate_limit_check", lambda *_args, **_kwargs: (lambda f: f)) + monkeypatch.setattr(wraps, "cloud_edition_billing_resource_check", lambda *_args, **_kwargs: lambda f: f) + monkeypatch.setattr(wraps, "cloud_edition_billing_rate_limit_check", lambda *_args, **_kwargs: lambda f: f) # Avoid Flask-RESTX route registration side effects during import. def _noop_route(*_args, **_kwargs): # type: ignore[override] diff --git a/api/tests/unit_tests/core/app/app_config/features/file_upload/test_manager.py b/api/tests/unit_tests/core/app/app_config/features/file_upload/test_manager.py index 2acf8815a5..9dddb18595 100644 --- a/api/tests/unit_tests/core/app/app_config/features/file_upload/test_manager.py +++ b/api/tests/unit_tests/core/app/app_config/features/file_upload/test_manager.py @@ -1,6 +1,6 @@ from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.file.models import FileTransferMethod, FileUploadConfig, ImageConfig from core.model_runtime.entities.message_entities import ImagePromptMessageContent +from core.workflow.file.models import FileTransferMethod, FileUploadConfig, ImageConfig def test_convert_with_vision(): diff --git a/api/tests/unit_tests/core/app/apps/chat/test_base_app_runner_multimodal.py b/api/tests/unit_tests/core/app/apps/chat/test_base_app_runner_multimodal.py index 421a5246eb..0bbfd452e1 100644 --- a/api/tests/unit_tests/core/app/apps/chat/test_base_app_runner_multimodal.py +++ b/api/tests/unit_tests/core/app/apps/chat/test_base_app_runner_multimodal.py @@ -9,8 +9,8 @@ from core.app.apps.base_app_queue_manager import PublishFrom from core.app.apps.base_app_runner import AppRunner from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import QueueMessageFileEvent -from core.file.enums import FileTransferMethod, FileType from core.model_runtime.entities.message_entities import ImagePromptMessageContent +from core.workflow.file.enums import FileTransferMethod, FileType from models.enums import CreatorUserRole diff --git a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter.py b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter.py index 8423f1ab02..f252324a85 100644 --- a/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter.py +++ b/api/tests/unit_tests/core/app/apps/common/test_workflow_response_converter.py @@ -1,8 +1,8 @@ from collections.abc import Mapping, Sequence from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter -from core.file import FILE_MODEL_IDENTITY, File, FileTransferMethod, FileType from core.variables.segments import ArrayFileSegment, FileSegment +from core.workflow.file import FILE_MODEL_IDENTITY, File, FileTransferMethod, FileType class TestWorkflowResponseConverterFetchFilesFromVariableValue: diff --git a/api/tests/unit_tests/core/file/test_models.py b/api/tests/unit_tests/core/file/test_models.py index f55063ee1a..4d4ccc2672 100644 --- a/api/tests/unit_tests/core/file/test_models.py +++ b/api/tests/unit_tests/core/file/test_models.py @@ -1,4 +1,4 @@ -from core.file import File, FileTransferMethod, FileType +from core.workflow.file import File, FileTransferMethod, FileType def test_file(): diff --git a/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py b/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py index 8abed0a3f9..f07e55d534 100644 --- a/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py +++ b/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py @@ -4,7 +4,6 @@ import pytest from configs import dify_config from core.app.app_config.entities import ModelConfigEntity -from core.file import File, FileTransferMethod, FileType from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.message_entities import ( AssistantPromptMessage, @@ -15,6 +14,7 @@ from core.model_runtime.entities.message_entities import ( from core.prompt.advanced_prompt_transform import AdvancedPromptTransform from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate, MemoryConfig from core.prompt.utils.prompt_template_parser import PromptTemplateParser +from core.workflow.file import File, FileTransferMethod, FileType from models.model import Conversation @@ -142,7 +142,7 @@ def test__get_chat_model_prompt_messages_with_files_no_memory(get_chat_model_arg prompt_transform = AdvancedPromptTransform() prompt_transform._calculate_rest_token = MagicMock(return_value=2000) - with patch("core.file.file_manager.to_prompt_message_content") as mock_get_encoded_string: + with patch("core.workflow.file.file_manager.to_prompt_message_content") as mock_get_encoded_string: mock_get_encoded_string.return_value = ImagePromptMessageContent( url=str(files[0].remote_url), format="jpg", mime_type="image/jpg" ) diff --git a/api/tests/unit_tests/core/test_file.py b/api/tests/unit_tests/core/test_file.py index e02d882780..b9c5fbd7d8 100644 --- a/api/tests/unit_tests/core/test_file.py +++ b/api/tests/unit_tests/core/test_file.py @@ -1,6 +1,6 @@ import json -from core.file import File, FileTransferMethod, FileType, FileUploadConfig +from core.workflow.file import File, FileTransferMethod, FileType, FileUploadConfig from models.workflow import Workflow diff --git a/api/tests/unit_tests/core/variables/test_segment.py b/api/tests/unit_tests/core/variables/test_segment.py index aa16c8af1c..bb9e381834 100644 --- a/api/tests/unit_tests/core/variables/test_segment.py +++ b/api/tests/unit_tests/core/variables/test_segment.py @@ -2,7 +2,6 @@ import dataclasses from pydantic import BaseModel -from core.file import File, FileTransferMethod, FileType from core.helper import encrypter from core.variables.segments import ( ArrayAnySegment, @@ -36,6 +35,7 @@ from core.variables.variables import ( StringVariable, Variable, ) +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.runtime import VariablePool from core.workflow.system_variable import SystemVariable diff --git a/api/tests/unit_tests/core/variables/test_segment_type_validation.py b/api/tests/unit_tests/core/variables/test_segment_type_validation.py index 3a0054cd46..0ec0fc536e 100644 --- a/api/tests/unit_tests/core/variables/test_segment_type_validation.py +++ b/api/tests/unit_tests/core/variables/test_segment_type_validation.py @@ -10,8 +10,6 @@ from typing import Any import pytest -from core.file.enums import FileTransferMethod, FileType -from core.file.models import File from core.variables.segment_group import SegmentGroup from core.variables.segments import ( ArrayFileSegment, @@ -23,6 +21,8 @@ from core.variables.segments import ( StringSegment, ) from core.variables.types import ArrayValidation, SegmentType +from core.workflow.file.enums import FileTransferMethod, FileType +from core.workflow.file.models import File def create_test_file( diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py index 1e224d56a5..0677f1bb52 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_file_saver.py @@ -6,10 +6,10 @@ import httpx import pytest from sqlalchemy import Engine -from core.file import FileTransferMethod, FileType, models from core.helper import ssrf_proxy from core.tools import signature from core.tools.tool_file_manager import ToolFileManager +from core.workflow.file import FileTransferMethod, FileType, models from core.workflow.nodes.llm.file_saver import ( FileSaverImpl, _extract_content_type_and_extension, diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py index 3d1b8b2f27..b0661f7d29 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_node.py @@ -8,7 +8,6 @@ import pytest from core.app.entities.app_invoke_entities import InvokeFrom, ModelConfigWithCredentialsEntity from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle from core.entities.provider_entities import CustomConfiguration, SystemConfiguration -from core.file import File, FileTransferMethod, FileType from core.model_runtime.entities.common_entities import I18nObject from core.model_runtime.entities.message_entities import ( ImagePromptMessageContent, @@ -21,6 +20,7 @@ from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory from core.variables import ArrayAnySegment, ArrayFileSegment, NoneSegment from core.workflow.entities import GraphInitParams +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.nodes.llm import llm_utils from core.workflow.nodes.llm.entities import ( ContextConfig, diff --git a/api/tests/unit_tests/core/workflow/nodes/llm/test_scenarios.py b/api/tests/unit_tests/core/workflow/nodes/llm/test_scenarios.py index 21bb857353..ac0c1df9c5 100644 --- a/api/tests/unit_tests/core/workflow/nodes/llm/test_scenarios.py +++ b/api/tests/unit_tests/core/workflow/nodes/llm/test_scenarios.py @@ -2,9 +2,9 @@ from collections.abc import Mapping, Sequence from pydantic import BaseModel, Field -from core.file import File from core.model_runtime.entities.message_entities import PromptMessage from core.model_runtime.entities.model_entities import ModelFeature +from core.workflow.file import File from core.workflow.nodes.llm.entities import LLMNodeChatModelMessage diff --git a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py index 088c60a337..669f36c100 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_document_extractor_node.py @@ -6,12 +6,12 @@ import pytest from docx.oxml.text.paragraph import CT_P from core.app.entities.app_invoke_entities import InvokeFrom -from core.file import File, FileTransferMethod from core.variables import ArrayFileSegment from core.variables.segments import ArrayStringSegment from core.variables.variables import StringVariable from core.workflow.entities import GraphInitParams from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod from core.workflow.node_events import NodeRunResult from core.workflow.nodes.document_extractor import DocumentExtractorNode, DocumentExtractorNodeData from core.workflow.nodes.document_extractor.node import ( @@ -146,7 +146,7 @@ def test_run_extract_text( mock_ssrf_proxy_get.return_value.content = file_content mock_ssrf_proxy_get.return_value.raise_for_status = Mock() - monkeypatch.setattr("core.file.file_manager.download", mock_download) + monkeypatch.setattr("core.workflow.file.file_manager.download", mock_download) monkeypatch.setattr("core.helper.ssrf_proxy.get", mock_ssrf_proxy_get) if mime_type == "application/pdf": diff --git a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py index d700888c2f..930bdbda4a 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_if_else.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_if_else.py @@ -6,10 +6,10 @@ import pytest from core.app.entities.app_invoke_entities import InvokeFrom from core.app.workflow.node_factory import DifyNodeFactory -from core.file import File, FileTransferMethod, FileType from core.variables import ArrayFileSegment from core.workflow.entities import GraphInitParams from core.workflow.enums import WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.graph import Graph from core.workflow.nodes.if_else.entities import IfElseNodeData from core.workflow.nodes.if_else.if_else_node import IfElseNode diff --git a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py index ff3eec0608..66ddc0d3c7 100644 --- a/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py +++ b/api/tests/unit_tests/core/workflow/nodes/test_list_operator.py @@ -3,9 +3,9 @@ from unittest.mock import MagicMock import pytest from core.app.entities.app_invoke_entities import InvokeFrom -from core.file import File, FileTransferMethod, FileType from core.variables import ArrayFileSegment from core.workflow.enums import WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.nodes.list_operator.entities import ( ExtractConfig, FilterBy, diff --git a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py index 06927cddcf..526ff72c8c 100644 --- a/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node.py @@ -8,12 +8,12 @@ from unittest.mock import MagicMock, patch import pytest -from core.file import File, FileTransferMethod, FileType from core.model_runtime.entities.llm_entities import LLMUsage from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.utils.message_transformer import ToolFileMessageTransformer from core.variables.segments import ArrayFileSegment from core.workflow.entities import GraphInitParams +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.node_events import StreamChunkEvent, StreamCompletedEvent from core.workflow.runtime import GraphRuntimeState, VariablePool from core.workflow.system_variable import SystemVariable diff --git a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py index 3b5aedebca..8ceaad5cc9 100644 --- a/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/webhook/test_webhook_node.py @@ -3,10 +3,10 @@ from unittest.mock import patch import pytest from core.app.entities.app_invoke_entities import InvokeFrom -from core.file import File, FileTransferMethod, FileType from core.variables import FileVariable, StringVariable from core.workflow.entities.graph_init_params import GraphInitParams from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.nodes.trigger_webhook.entities import ( ContentType, Method, diff --git a/api/tests/unit_tests/core/workflow/test_system_variable.py b/api/tests/unit_tests/core/workflow/test_system_variable.py index f76e81ae55..93e7c9f68d 100644 --- a/api/tests/unit_tests/core/workflow/test_system_variable.py +++ b/api/tests/unit_tests/core/workflow/test_system_variable.py @@ -4,8 +4,8 @@ from typing import Any import pytest from pydantic import ValidationError -from core.file.enums import FileTransferMethod, FileType -from core.file.models import File +from core.workflow.file.enums import FileTransferMethod, FileType +from core.workflow.file.models import File from core.workflow.system_variable import SystemVariable # Test data constants for SystemVariable serialization tests diff --git a/api/tests/unit_tests/core/workflow/test_system_variable_read_only_view.py b/api/tests/unit_tests/core/workflow/test_system_variable_read_only_view.py index 57bc96fe71..743fecaed0 100644 --- a/api/tests/unit_tests/core/workflow/test_system_variable_read_only_view.py +++ b/api/tests/unit_tests/core/workflow/test_system_variable_read_only_view.py @@ -2,7 +2,7 @@ from typing import cast import pytest -from core.file.models import File, FileTransferMethod, FileType +from core.workflow.file.models import File, FileTransferMethod, FileType from core.workflow.system_variable import SystemVariable, SystemVariableReadOnlyView diff --git a/api/tests/unit_tests/core/workflow/test_variable_pool.py b/api/tests/unit_tests/core/workflow/test_variable_pool.py index b8869dbf1d..fb9a893d43 100644 --- a/api/tests/unit_tests/core/workflow/test_variable_pool.py +++ b/api/tests/unit_tests/core/workflow/test_variable_pool.py @@ -3,7 +3,6 @@ from collections import defaultdict import pytest -from core.file import File, FileTransferMethod, FileType from core.variables import FileSegment, StringSegment from core.variables.segments import ( ArrayAnySegment, @@ -27,6 +26,7 @@ from core.variables.variables import ( Variable, ) from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID +from core.workflow.file import File, FileTransferMethod, FileType from core.workflow.runtime import VariablePool from core.workflow.system_variable import SystemVariable from factories.variable_factory import build_segment, segment_to_variable diff --git a/api/tests/unit_tests/core/workflow/test_workflow_entry.py b/api/tests/unit_tests/core/workflow/test_workflow_entry.py index 27ffa455d6..793b0d4eba 100644 --- a/api/tests/unit_tests/core/workflow/test_workflow_entry.py +++ b/api/tests/unit_tests/core/workflow/test_workflow_entry.py @@ -3,14 +3,14 @@ from types import SimpleNamespace import pytest from configs import dify_config -from core.file.enums import FileType -from core.file.models import File, FileTransferMethod from core.helper.code_executor.code_executor import CodeLanguage from core.variables.variables import StringVariable from core.workflow.constants import ( CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, ) +from core.workflow.file.enums import FileType +from core.workflow.file.models import File, FileTransferMethod from core.workflow.nodes.code.code_node import CodeNode from core.workflow.nodes.code.limits import CodeNodeLimits from core.workflow.runtime import VariablePool diff --git a/api/tests/unit_tests/factories/test_variable_factory.py b/api/tests/unit_tests/factories/test_variable_factory.py index f12e5993dc..53ae18a61d 100644 --- a/api/tests/unit_tests/factories/test_variable_factory.py +++ b/api/tests/unit_tests/factories/test_variable_factory.py @@ -7,7 +7,6 @@ import pytest from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st -from core.file import File, FileTransferMethod, FileType from core.variables import ( ArrayNumberVariable, ArrayObjectVariable, @@ -34,6 +33,7 @@ from core.variables.segments import ( StringSegment, ) from core.variables.types import SegmentType +from core.workflow.file import File, FileTransferMethod, FileType from factories import variable_factory from factories.variable_factory import TypeMismatchError, build_segment, build_segment_with_type diff --git a/api/tests/unit_tests/models/test_workflow.py b/api/tests/unit_tests/models/test_workflow.py index 4c61320c29..29f71767d0 100644 --- a/api/tests/unit_tests/models/test_workflow.py +++ b/api/tests/unit_tests/models/test_workflow.py @@ -4,10 +4,10 @@ from unittest import mock from uuid import uuid4 from constants import HIDDEN_VALUE -from core.file.enums import FileTransferMethod, FileType -from core.file.models import File from core.variables import FloatVariable, IntegerVariable, SecretVariable, StringVariable from core.variables.segments import IntegerSegment, Segment +from core.workflow.file.enums import FileTransferMethod, FileType +from core.workflow.file.models import File from factories.variable_factory import build_segment from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel, is_system_variable_editable diff --git a/api/tests/unit_tests/services/test_variable_truncator.py b/api/tests/unit_tests/services/test_variable_truncator.py index ec819ae57a..4534e68b4e 100644 --- a/api/tests/unit_tests/services/test_variable_truncator.py +++ b/api/tests/unit_tests/services/test_variable_truncator.py @@ -17,8 +17,6 @@ from uuid import uuid4 import pytest -from core.file.enums import FileTransferMethod, FileType -from core.file.models import File from core.variables.segments import ( ArrayFileSegment, ArrayNumberSegment, @@ -30,6 +28,8 @@ from core.variables.segments import ( ObjectSegment, StringSegment, ) +from core.workflow.file.enums import FileTransferMethod, FileType +from core.workflow.file.models import File from services.variable_truncator import ( DummyVariableTruncator, MaxDepthExceededError,