diff --git a/api/core/plugin/impl/model_runtime.py b/api/core/plugin/impl/model_runtime.py index 507d5fa4f5..0050eea307 100644 --- a/api/core/plugin/impl/model_runtime.py +++ b/api/core/plugin/impl/model_runtime.py @@ -481,9 +481,12 @@ class PluginModelRuntime(ModelRuntime): ) -> str: cache_key = f"{self.tenant_id}:{provider}:{model_type.value}:{model}" sorted_credentials = sorted(credentials.items()) if credentials else [] - return cache_key + ":".join( + if not sorted_credentials: + return cache_key + hashed_credentials = ":".join( [hashlib.md5(f"{key}:{value}".encode()).hexdigest() for key, value in sorted_credentials] ) + return f"{cache_key}:{hashed_credentials}" def _split_provider(self, provider: str) -> tuple[str, str]: provider_id = ModelProviderID(provider) diff --git a/api/dify_graph/file/models.py b/api/dify_graph/file/models.py index f8b1701cdb..0437b481d9 100644 --- a/api/dify_graph/file/models.py +++ b/api/dify_graph/file/models.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import json from collections.abc import Mapping, Sequence from typing import Any @@ -11,6 +13,8 @@ from . import helpers from .constants import FILE_MODEL_IDENTITY from .enums import FileTransferMethod, FileType +_FILE_REFERENCE_PREFIX = "dify-file-ref:" + def sign_tool_file(*, tool_file_id: str, extension: str, for_external: bool = True) -> str: """Compatibility shim for tests and legacy callers patching ``models.sign_tool_file``.""" @@ -43,6 +47,31 @@ class FileUploadConfig(BaseModel): number_limits: int = 0 +def _parse_reference(reference: str | None) -> tuple[str | None, str | None]: + """Best-effort parser for legacy aliases backed by the opaque file reference.""" + if not reference: + return None, None + + if not reference.startswith(_FILE_REFERENCE_PREFIX): + return reference, None + + encoded_payload = reference.removeprefix(_FILE_REFERENCE_PREFIX) + try: + payload = json.loads(base64.urlsafe_b64decode(encoded_payload.encode())) + except (ValueError, json.JSONDecodeError): + return reference, None + + record_id = payload.get("record_id") + if not isinstance(record_id, str) or not record_id: + return reference, None + + storage_key = payload.get("storage_key") + if not isinstance(storage_key, str): + storage_key = None + + return record_id, storage_key + + class File(BaseModel): """Graph-owned file reference. @@ -67,11 +96,13 @@ class File(BaseModel): extension: str | None = Field(default=None, description="File extension, should contain dot") mime_type: str | None = None size: int = -1 + _storage_key: str def __init__( self, *, id: str | None = None, + tenant_id: str | None = None, type: FileType, transfer_method: FileTransferMethod, remote_url: str | None = None, @@ -89,10 +120,11 @@ class File(BaseModel): upload_file_id: str | None = None, datasource_file_id: str | None = None, ): - legacy_record_id = tool_file_id or upload_file_id or datasource_file_id or related_id + legacy_record_id = related_id or tool_file_id or upload_file_id or datasource_file_id normalized_reference = reference if normalized_reference is None and legacy_record_id is not None: normalized_reference = str(legacy_record_id) + _, parsed_storage_key = _parse_reference(normalized_reference) super().__init__( id=id, @@ -107,12 +139,15 @@ class File(BaseModel): dify_model_identity=dify_model_identity, url=url, ) + # Accept legacy constructor fields without promoting them back into the graph model. + _ = tenant_id + self._storage_key = storage_key or parsed_storage_key or "" def to_dict(self) -> Mapping[str, str | int | None]: data = self.model_dump(mode="json") return { **data, - "related_id": self.reference, + "related_id": self.related_id, "url": self.generate_url(), } @@ -161,7 +196,8 @@ class File(BaseModel): @property def related_id(self) -> str | None: - return self.reference + record_id, _ = _parse_reference(self.reference) + return record_id @related_id.setter def related_id(self, value: str | None) -> None: @@ -169,4 +205,9 @@ class File(BaseModel): @property def storage_key(self) -> str: - return "" + _, storage_key = _parse_reference(self.reference) + return storage_key or self._storage_key + + @storage_key.setter + def storage_key(self, value: str) -> None: + self._storage_key = value diff --git a/api/dify_graph/runtime/variable_pool.py b/api/dify_graph/runtime/variable_pool.py index 29ba0496be..b58359c005 100644 --- a/api/dify_graph/runtime/variable_pool.py +++ b/api/dify_graph/runtime/variable_pool.py @@ -207,13 +207,7 @@ class VariablePool(BaseModel): return result def flatten(self, *, unprefixed_node_id: str | None = None) -> Mapping[str, object]: - """Return a selector-style snapshot of the entire variable pool. - - Variables belonging to ``unprefixed_node_id`` keep their original names so callers - can expose the current node's values without duplicating its namespace. All other - entries are emitted as ``"."`` to preserve their source prefix in a - single flat mapping. - """ + """Return a selector-style snapshot of the entire variable pool.""" result: dict[str, object] = {} for node_id, variables in self.variable_dictionary.items(): diff --git a/api/factories/file_factory.py b/api/factories/file_factory.py index 16f6a0d3b7..49b5886b07 100644 --- a/api/factories/file_factory.py +++ b/api/factories/file_factory.py @@ -570,6 +570,7 @@ class StorageKeyLoader: record_id=str(upload_file_row.id), storage_key=upload_file_row.key, ) + file.storage_key = upload_file_row.key elif file.transfer_method == FileTransferMethod.TOOL_FILE: tool_file_row = tool_files.get(model_id) if tool_file_row is None: @@ -578,3 +579,4 @@ class StorageKeyLoader: record_id=str(tool_file_row.id), storage_key=tool_file_row.file_key, ) + file.storage_key = tool_file_row.file_key diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index 15bb14b59b..599b6db659 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -898,22 +898,7 @@ class DraftVariableSaver: for name, value in output.items(): value_seg = _build_segment_for_serialized_values(value) node_id, name = self._normalize_variable_for_start_node(name) - if node_id == self._node_id: - # Variables without a reserved prefix belong to the Start node itself. - draft_vars.append( - WorkflowDraftVariable.new_node_variable( - app_id=self._app_id, - user_id=self._user.id, - node_id=self._node_id, - name=name, - node_execution_id=self._node_execution_id, - value=value_seg, - visible=True, - editable=True, - ) - ) - has_non_sys_variables = True - elif node_id == SYSTEM_VARIABLE_NODE_ID: + if node_id == SYSTEM_VARIABLE_NODE_ID: if name == SystemVariableKey.FILES: # Here we know the type of variable must be `array[file]`, we # just build files from the value. @@ -947,6 +932,7 @@ class DraftVariableSaver: value=value_seg, ) ) + has_non_sys_variables = True else: draft_vars.append( WorkflowDraftVariable.new_node_variable( @@ -960,6 +946,7 @@ class DraftVariableSaver: editable=self._should_variable_be_editable(node_id, name), ) ) + has_non_sys_variables = True if not has_non_sys_variables: draft_vars.append(self._create_dummy_output_variable()) return draft_vars diff --git a/api/tests/integration_tests/factories/test_storage_key_loader.py b/api/tests/integration_tests/factories/test_storage_key_loader.py index 66f23bdc77..ce47a1dbc6 100644 --- a/api/tests/integration_tests/factories/test_storage_key_loader.py +++ b/api/tests/integration_tests/factories/test_storage_key_loader.py @@ -118,6 +118,7 @@ class TestStorageKeyLoader(unittest.TestCase): return File( id=str(uuid4()), # Generate new UUID for File.id + tenant_id=tenant_id, type=FileType.DOCUMENT, transfer_method=transfer_method, related_id=file_related_id, @@ -191,19 +192,16 @@ class TestStorageKeyLoader(unittest.TestCase): # Should not raise any exceptions self.loader.load_storage_keys([]) - def test_load_storage_keys_tenant_mismatch(self): - """Test tenant_id validation.""" - # Create file with different tenant_id + def test_load_storage_keys_ignores_legacy_file_tenant_id(self): + """Legacy file tenant_id should not override the loader tenant scope.""" upload_file = self._create_upload_file() file = self._create_file( related_id=upload_file.id, transfer_method=FileTransferMethod.LOCAL_FILE, tenant_id=str(uuid4()) ) - # Should raise ValueError for tenant mismatch - with pytest.raises(ValueError) as context: - self.loader.load_storage_keys([file]) + self.loader.load_storage_keys([file]) - assert "invalid file, expected tenant_id" in str(context.value) + assert file._storage_key == upload_file.key def test_load_storage_keys_missing_file_id(self): """Test with None file.related_id.""" diff --git a/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py b/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py index 842ca7f730..090f234555 100644 --- a/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py +++ b/api/tests/test_containers_integration_tests/factories/test_storage_key_loader.py @@ -119,6 +119,7 @@ class TestStorageKeyLoader(unittest.TestCase): return File( id=str(uuid4()), # Generate new UUID for File.id + tenant_id=tenant_id, type=FileType.DOCUMENT, transfer_method=transfer_method, related_id=file_related_id, @@ -192,19 +193,16 @@ class TestStorageKeyLoader(unittest.TestCase): # Should not raise any exceptions self.loader.load_storage_keys([]) - def test_load_storage_keys_tenant_mismatch(self): - """Test tenant_id validation.""" - # Create file with different tenant_id + def test_load_storage_keys_ignores_legacy_file_tenant_id(self): + """Legacy file tenant_id should not override the loader tenant scope.""" upload_file = self._create_upload_file() file = self._create_file( related_id=upload_file.id, transfer_method=FileTransferMethod.LOCAL_FILE, tenant_id=str(uuid4()) ) - # Should raise ValueError for tenant mismatch - with pytest.raises(ValueError) as context: - self.loader.load_storage_keys([file]) + self.loader.load_storage_keys([file]) - assert "invalid file, expected tenant_id" in str(context.value) + assert file._storage_key == upload_file.key def test_load_storage_keys_missing_file_id(self): """Test with None file.related_id.""" diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow.py b/api/tests/unit_tests/controllers/console/app/test_workflow.py index 02190b0f47..0e22db9f9b 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow.py @@ -30,6 +30,7 @@ def test_parse_file_with_config(monkeypatch: pytest.MonkeyPatch) -> None: config = object() file_list = [ File( + tenant_id="t1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="http://u", diff --git a/api/tests/unit_tests/core/file/test_models.py b/api/tests/unit_tests/core/file/test_models.py index 29f6ac6469..a06907ce4b 100644 --- a/api/tests/unit_tests/core/file/test_models.py +++ b/api/tests/unit_tests/core/file/test_models.py @@ -1,11 +1,10 @@ -import pytest - from dify_graph.file import File, FileTransferMethod, FileType def test_file(): file = File( id="test-file", + tenant_id="test-tenant-id", type=FileType.IMAGE, transfer_method=FileTransferMethod.TOOL_FILE, related_id="test-related-id", @@ -19,13 +18,14 @@ def test_file(): assert file.type == FileType.IMAGE assert file.transfer_method == FileTransferMethod.TOOL_FILE assert file.related_id == "test-related-id" + assert file.storage_key == "test-storage-key" assert file.filename == "image.png" assert file.extension == ".png" assert file.mime_type == "image/png" assert file.size == 67 -def test_file_model_validate_rejects_removed_tenant_id(): +def test_file_model_validate_accepts_legacy_tenant_id(): data = { "id": "test-file", "tenant_id": "test-tenant-id", @@ -44,5 +44,8 @@ def test_file_model_validate_rejects_removed_tenant_id(): "datasource_file_id": "datasource-file-789", } - with pytest.raises(TypeError, match="tenant_id"): - File.model_validate(data) + file = File.model_validate(data) + + assert file.related_id == "test-related-id" + assert file.storage_key == "test-storage-key" + assert "tenant_id" not in file.model_dump() diff --git a/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py b/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py index fd1ba33890..c7e94aa4cf 100644 --- a/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py +++ b/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py @@ -466,6 +466,7 @@ class TestChunkMerger: class TestConverter: def test_convert_parameters_to_plugin_format_with_single_file_and_selector(self): file_param = File( + tenant_id="tenant-1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/file.png", @@ -498,12 +499,14 @@ class TestConverter: def test_convert_parameters_to_plugin_format_with_lists_and_passthrough_values(self): file_one = File( + tenant_id="tenant-1", type=FileType.DOCUMENT, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/a.txt", storage_key="", ) file_two = File( + tenant_id="tenant-1", type=FileType.DOCUMENT, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/b.txt", diff --git a/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py b/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py index 3be62a444e..3d08525aba 100644 --- a/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py +++ b/api/tests/unit_tests/core/prompt/test_advanced_prompt_transform.py @@ -135,6 +135,7 @@ def test__get_chat_model_prompt_messages_with_files_no_memory(get_chat_model_arg files = [ File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image1.jpg", @@ -245,6 +246,7 @@ def test_completion_prompt_jinja2_with_files(): file = File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image.jpg", @@ -378,6 +380,7 @@ def test_chat_prompt_memory_with_files_and_query(): prompt_template = [ChatModelMessage(text="sys", role=PromptMessageRole.SYSTEM)] file = File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image.jpg", @@ -411,6 +414,7 @@ def test_chat_prompt_files_without_query_updates_last_user_or_appends_new(): model_config_mock = MagicMock(spec=ModelConfigEntity) file = File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image.jpg", @@ -460,6 +464,7 @@ def test_chat_prompt_files_with_query_branch(): model_config_mock = MagicMock(spec=ModelConfigEntity) file = File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image.jpg", diff --git a/api/tests/unit_tests/core/test_file.py b/api/tests/unit_tests/core/test_file.py index a5bbe374b1..251d6fd25e 100644 --- a/api/tests/unit_tests/core/test_file.py +++ b/api/tests/unit_tests/core/test_file.py @@ -7,6 +7,7 @@ from models.workflow import Workflow def test_file_to_dict(): file = File( id="file1", + tenant_id="tenant1", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/image1.jpg", diff --git a/api/tests/unit_tests/core/variables/test_segment_type_validation.py b/api/tests/unit_tests/core/variables/test_segment_type_validation.py index 19f7673f6b..41ce483447 100644 --- a/api/tests/unit_tests/core/variables/test_segment_type_validation.py +++ b/api/tests/unit_tests/core/variables/test_segment_type_validation.py @@ -35,6 +35,7 @@ def create_test_file( ) -> File: """Factory function to create File objects for testing.""" return File( + tenant_id="test-tenant", type=file_type, transfer_method=transfer_method, filename=filename, diff --git a/api/tests/unit_tests/factories/test_variable_factory.py b/api/tests/unit_tests/factories/test_variable_factory.py index f63f523389..ce6b9232ce 100644 --- a/api/tests/unit_tests/factories/test_variable_factory.py +++ b/api/tests/unit_tests/factories/test_variable_factory.py @@ -227,6 +227,7 @@ def test_build_segment_array_file_single_file(): """Test building ArrayFileSegment from list with single file.""" file = File( id="test_file_id", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file.png", @@ -246,6 +247,7 @@ def test_build_segment_array_file_multiple_files(): """Test building ArrayFileSegment from list with multiple files.""" file1 = File( id="test_file_id_1", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file1.png", @@ -256,6 +258,7 @@ def test_build_segment_array_file_multiple_files(): ) file2 = File( id="test_file_id_2", + tenant_id="test_tenant_id", type=FileType.DOCUMENT, transfer_method=FileTransferMethod.LOCAL_FILE, related_id="test_relation_id", @@ -302,6 +305,7 @@ def test_build_segment_array_any_mixed_with_files(): """Test building ArrayAnySegment from list with files and other types.""" file = File( id="test_file_id", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file.png", @@ -330,6 +334,7 @@ def test_build_segment_array_file_properties(): """Test ArrayFileSegment properties and methods.""" file1 = File( id="test_file_id_1", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file1.png", @@ -340,6 +345,7 @@ def test_build_segment_array_file_properties(): ) file2 = File( id="test_file_id_2", + tenant_id="test_tenant_id", type=FileType.DOCUMENT, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file2.txt", @@ -388,6 +394,7 @@ def test_build_segment_file_array_with_different_file_types(): """Test ArrayFileSegment with different file types.""" image_file = File( id="image_id", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/image.png", @@ -399,6 +406,7 @@ def test_build_segment_file_array_with_different_file_types(): video_file = File( id="video_id", + tenant_id="test_tenant_id", type=FileType.VIDEO, transfer_method=FileTransferMethod.LOCAL_FILE, related_id="video_relation_id", @@ -410,6 +418,7 @@ def test_build_segment_file_array_with_different_file_types(): audio_file = File( id="audio_id", + tenant_id="test_tenant_id", type=FileType.AUDIO, transfer_method=FileTransferMethod.LOCAL_FILE, related_id="audio_relation_id", @@ -447,6 +456,7 @@ def _generate_file(draw) -> File: url = "https://test.example.com/test-file" file = File( id="test_file_id", + tenant_id="test_tenant_id", type=file_type, transfer_method=transfer_method, remote_url=url, @@ -461,6 +471,7 @@ def _generate_file(draw) -> File: file = File( id="test_file_id", + tenant_id="test_tenant_id", type=file_type, transfer_method=transfer_method, related_id=str(relation_id), @@ -508,6 +519,7 @@ def test_build_segment_type_for_scalar(): file = File( id="test_file_id", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file.png", @@ -564,6 +576,7 @@ class TestBuildSegmentWithType: """Test building a file segment with correct type.""" test_file = File( id="test_file_id", + tenant_id="test_tenant_id", type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://test.example.com/test-file.png", diff --git a/api/tests/unit_tests/fields/test_file_fields.py b/api/tests/unit_tests/fields/test_file_fields.py index 5581b55277..9214b2c9f3 100644 --- a/api/tests/unit_tests/fields/test_file_fields.py +++ b/api/tests/unit_tests/fields/test_file_fields.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import pytest +from core.workflow.file_reference import build_file_reference from dify_graph.file import File, FileTransferMethod, FileType from fields import conversation_fields, message_fields from fields.file_fields import FileResponse, FileWithSignedUrl, RemoteFileInfo, UploadConfig @@ -91,12 +92,13 @@ def test_remote_file_info_and_upload_config() -> None: ) def test_file_formatters_preserve_legacy_file_keys(monkeypatch: pytest.MonkeyPatch, formatter) -> None: monkeypatch.setattr(File, "generate_url", lambda self, for_external=True: "https://preview.example/file") + reference = build_file_reference(record_id="upload-1", storage_key="files/source.pdf") file = File( type=FileType.DOCUMENT, transfer_method=FileTransferMethod.LOCAL_FILE, remote_url="https://storage.example/source.pdf", - reference="dify-file-ref:opaque-upload-1", + reference=reference, filename="source.pdf", extension=".pdf", mime_type="application/pdf", @@ -105,7 +107,7 @@ def test_file_formatters_preserve_legacy_file_keys(monkeypatch: pytest.MonkeyPat serialized = formatter(file) - assert serialized["reference"] == "dify-file-ref:opaque-upload-1" - assert serialized["related_id"] == "dify-file-ref:opaque-upload-1" + assert serialized["reference"] == reference + assert serialized["related_id"] == "upload-1" assert serialized["remote_url"] == "https://storage.example/source.pdf" assert serialized["url"] == "https://preview.example/file" diff --git a/api/tests/unit_tests/services/test_variable_truncator.py b/api/tests/unit_tests/services/test_variable_truncator.py index 2564d0e0dd..c703ab64d0 100644 --- a/api/tests/unit_tests/services/test_variable_truncator.py +++ b/api/tests/unit_tests/services/test_variable_truncator.py @@ -43,6 +43,7 @@ from services.variable_truncator import ( def file() -> File: return File( id=str(uuid4()), # Generate new UUID for File.id + tenant_id=str(uuid.uuid4()), type=FileType.DOCUMENT, transfer_method=FileTransferMethod.LOCAL_FILE, related_id=str(uuid.uuid4()), diff --git a/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py b/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py index eb53edcffa..65c2ab5c2d 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_draft_variable_service.py @@ -264,7 +264,7 @@ class TestDraftVariableSaver: mock_batch_upsert.assert_called_once() draft_vars = mock_batch_upsert.call_args[0][1] - assert len(draft_vars) == 4 + assert len(draft_vars) == 3 env_var = next(v for v in draft_vars if v.node_id == ENVIRONMENT_VARIABLE_NODE_ID) assert env_var.name == "API_KEY"