jprdonnelly/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2025-12-19 17:27:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement file extension blacklist for upload security (#27540)

Browse Source
This commit is contained in:
Novice
2025-11-04 15:45:22 +08:00
committed by GitHub
parent f9c67621ca
commit ef1db35f80
19 changed files with 277 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/.env.example
View File

@@ -371,6 +371,12 @@ UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
# Comma-separated list of file extensions blocked from upload for security reasons.
# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll).
# Empty by default to allow all file types.
# Recommended: exe,bat,cmd,com,scr,vbs,ps1,msi,dll
UPLOAD_FILE_EXTENSION_BLACKLIST=
# Model configuration
MULTIMODAL_SEND_FORMAT=base64
PROMPT_GENERATION_MAX_TOKENS=512

25
api/configs/feature/__init__.py
View File

@@ -331,6 +331,31 @@ class FileUploadConfig(BaseSettings):
default=10,
)
inner_UPLOAD_FILE_EXTENSION_BLACKLIST: str = Field(
description=(
"Comma-separated list of file extensions that are blocked from upload. "
"Extensions should be lowercase without dots (e.g., 'exe,bat,sh,dll'). "
"Empty by default to allow all file types."
),
validation_alias=AliasChoices("UPLOAD_FILE_EXTENSION_BLACKLIST"),
default="",
)
@computed_field # type: ignore[misc]
@property
def UPLOAD_FILE_EXTENSION_BLACKLIST(self) -> set[str]:
"""
Parse and return the blacklist as a set of lowercase extensions.
Returns an empty set if no blacklist is configured.
"""
if not self.inner_UPLOAD_FILE_EXTENSION_BLACKLIST:
return set()
return {
ext.strip().lower().strip(".")
for ext in self.inner_UPLOAD_FILE_EXTENSION_BLACKLIST.split(",")
if ext.strip()
}
class HttpConfig(BaseSettings):
"""

6
api/controllers/common/errors.py
View File

@@ -25,6 +25,12 @@ class UnsupportedFileTypeError(BaseHTTPException):
code = 415
class BlockedFileExtensionError(BaseHTTPException):
error_code = "file_extension_blocked"
description = "The file extension is blocked for security reasons."
code = 400
class TooManyFilesError(BaseHTTPException):
error_code = "too_many_files"
description = "Only one file is allowed."

3
api/controllers/console/files.py
View File

@@ -8,6 +8,7 @@ import services
from configs import dify_config
from constants import DOCUMENT_EXTENSIONS
from controllers.common.errors import (
BlockedFileExtensionError,
FilenameNotExistsError,
FileTooLargeError,
NoFileUploadedError,
@@ -83,6 +84,8 @@ class FileApi(Resource):
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:
raise UnsupportedFileTypeError()
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
raise BlockedFileExtensionError(blocked_extension_error.description)
return upload_file, 201

4
api/services/errors/file.py
View File

@@ -11,3 +11,7 @@ class FileTooLargeError(BaseServiceError):
class UnsupportedFileTypeError(BaseServiceError):
pass
class BlockedFileExtensionError(BaseServiceError):
description = "File extension '{extension}' is not allowed for security reasons"

6
api/services/file_service.py
View File

@@ -23,7 +23,7 @@ from models import Account
from models.enums import CreatorUserRole
from models.model import EndUser, UploadFile
from .errors.file import FileTooLargeError, UnsupportedFileTypeError
from .errors.file import BlockedFileExtensionError, FileTooLargeError, UnsupportedFileTypeError
PREVIEW_WORDS_LIMIT = 3000
@@ -59,6 +59,10 @@ class FileService:
if len(filename) > 200:
filename = filename.split(".")[0][:200] + "." + extension
# check if extension is in blacklist
if extension and extension in dify_config.UPLOAD_FILE_EXTENSION_BLACKLIST:
raise BlockedFileExtensionError(f"File extension '.{extension}' is not allowed for security reasons")
if source == "datasets" and extension not in DOCUMENT_EXTENSIONS:
raise UnsupportedFileTypeError()

149
api/tests/test_containers_integration_tests/services/test_file_service.py
View File

@@ -11,7 +11,7 @@ from configs import dify_config
from models import Account, Tenant
from models.enums import CreatorUserRole
from models.model import EndUser, UploadFile
from services.errors.file import FileTooLargeError, UnsupportedFileTypeError
from services.errors.file import BlockedFileExtensionError, FileTooLargeError, UnsupportedFileTypeError
from services.file_service import FileService
@@ -943,3 +943,150 @@ class TestFileService:
# Should have the signed URL when source_url is empty
assert upload_file2.source_url == "https://example.com/signed-url"
# Test file extension blacklist
def test_upload_file_blocked_extension(
self, db_session_with_containers, engine, mock_external_service_dependencies
):
"""
Test file upload with blocked extension.
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock blacklist configuration by patching the inner field
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", "exe,bat,sh"):
filename = "malware.exe"
content = b"test content"
mimetype = "application/x-msdownload"
with pytest.raises(BlockedFileExtensionError):
FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
def test_upload_file_blocked_extension_case_insensitive(
self, db_session_with_containers, engine, mock_external_service_dependencies
):
"""
Test file upload with blocked extension (case insensitive).
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock blacklist configuration by patching the inner field
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", "exe,bat"):
# Test with uppercase extension
filename = "malware.EXE"
content = b"test content"
mimetype = "application/x-msdownload"
with pytest.raises(BlockedFileExtensionError):
FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
def test_upload_file_not_in_blacklist(self, db_session_with_containers, engine, mock_external_service_dependencies):
"""
Test file upload with extension not in blacklist.
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock blacklist configuration by patching the inner field
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", "exe,bat,sh"):
filename = "document.pdf"
content = b"test content"
mimetype = "application/pdf"
upload_file = FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
assert upload_file is not None
assert upload_file.name == filename
assert upload_file.extension == "pdf"
def test_upload_file_empty_blacklist(self, db_session_with_containers, engine, mock_external_service_dependencies):
"""
Test file upload with empty blacklist (default behavior).
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock empty blacklist configuration by patching the inner field
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", ""):
# Should allow all file types when blacklist is empty
filename = "script.sh"
content = b"#!/bin/bash\necho test"
mimetype = "application/x-sh"
upload_file = FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
assert upload_file is not None
assert upload_file.extension == "sh"
def test_upload_file_multiple_blocked_extensions(
self, db_session_with_containers, engine, mock_external_service_dependencies
):
"""
Test file upload with multiple blocked extensions.
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock blacklist with multiple extensions by patching the inner field
blacklist_str = "exe,bat,cmd,com,scr,vbs,ps1,msi,dll"
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", blacklist_str):
for ext in blacklist_str.split(","):
filename = f"malware.{ext}"
content = b"test content"
mimetype = "application/octet-stream"
with pytest.raises(BlockedFileExtensionError):
FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
def test_upload_file_no_extension_with_blacklist(
self, db_session_with_containers, engine, mock_external_service_dependencies
):
"""
Test file upload with no extension when blacklist is configured.
"""
fake = Faker()
account = self._create_test_account(db_session_with_containers, mock_external_service_dependencies)
# Mock blacklist configuration by patching the inner field
with patch.object(dify_config, "inner_UPLOAD_FILE_EXTENSION_BLACKLIST", "exe,bat"):
# Files with no extension should not be blocked
filename = "README"
content = b"test content"
mimetype = "text/plain"
upload_file = FileService(engine).upload_file(
filename=filename,
content=content,
mimetype=mimetype,
user=account,
)
assert upload_file is not None
assert upload_file.extension == ""

6
docker/.env.example
View File

@@ -762,6 +762,12 @@ UPLOAD_FILE_SIZE_LIMIT=15
# The maximum number of files that can be uploaded at a time, default 5.
UPLOAD_FILE_BATCH_LIMIT=5
# Comma-separated list of file extensions blocked from upload for security reasons.
# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll).
# Empty by default to allow all file types.
# Recommended: exe,bat,cmd,com,scr,vbs,ps1,msi,dll
UPLOAD_FILE_EXTENSION_BLACKLIST=
# ETL type, support: `dify`, `Unstructured`
# `dify` Dify's proprietary file extraction scheme
# `Unstructured` Unstructured.io file extraction scheme

1
docker/docker-compose.yaml
View File

@@ -353,6 +353,7 @@ x-shared-env: &shared-api-worker-env
CLICKZETTA_VECTOR_DISTANCE_FUNCTION: ${CLICKZETTA_VECTOR_DISTANCE_FUNCTION:-cosine_distance}
UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15}
UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5}
UPLOAD_FILE_EXTENSION_BLACKLIST: ${UPLOAD_FILE_EXTENSION_BLACKLIST:-}
ETL_TYPE: ${ETL_TYPE:-dify}
UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-}
UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY:-}

11
web/app/components/base/file-uploader/hooks.ts
View File

@@ -11,6 +11,7 @@ import type { FileEntity } from './types'
import { useFileStore } from './store'
import {
fileUpload,
getFileUploadErrorMessage,
getSupportFileType,
isAllowedFileExtension,
} from './utils'
@@ -172,8 +173,9 @@ export const useFile = (fileConfig: FileUpload) => {
onSuccessCallback: (res) => {
handleUpdateFile({ ...uploadingFile, uploadedId: res.id, progress: 100 })
},
onErrorCallback: () => {
notify({ type: 'error', message: t('common.fileUploader.uploadFromComputerUploadError') })
onErrorCallback: (error?: any) => {
const errorMessage = getFileUploadErrorMessage(error, t('common.fileUploader.uploadFromComputerUploadError'), t)
notify({ type: 'error', message: errorMessage })
handleUpdateFile({ ...uploadingFile, progress: -1 })
},
}, !!params.token)
@@ -279,8 +281,9 @@ export const useFile = (fileConfig: FileUpload) => {
onSuccessCallback: (res) => {
handleUpdateFile({ ...uploadingFile, uploadedId: res.id, progress: 100 })
},
onErrorCallback: () => {
notify({ type: 'error', message: t('common.fileUploader.uploadFromComputerUploadError') })
onErrorCallback: (error?: any) => {
const errorMessage = getFileUploadErrorMessage(error, t('common.fileUploader.uploadFromComputerUploadError'), t)
notify({ type: 'error', message: errorMessage })
handleUpdateFile({ ...uploadingFile, progress: -1 })
},
}, !!params.token)

25
web/app/components/base/file-uploader/utils.ts
View File

@@ -7,11 +7,30 @@ import { SupportUploadFileTypes } from '@/app/components/workflow/types'
import type { FileResponse } from '@/types/workflow'
import { TransferMethod } from '@/types/app'
/**
* Get appropriate error message for file upload errors
* @param error - The error object from upload failure
* @param defaultMessage - Default error message to use if no specific error is matched
* @param t - Translation function
* @returns Localized error message
*/
export const getFileUploadErrorMessage = (error: any, defaultMessage: string, t: (key: string) => string): string => {
const errorCode = error?.response?.code
if (errorCode === 'forbidden')
return error?.response?.message
if (errorCode === 'file_extension_blocked')
return t('common.fileUploader.fileExtensionBlocked')
return defaultMessage
}
type FileUploadParams = {
file: File
onProgressCallback: (progress: number) => void
onSuccessCallback: (res: { id: string }) => void
onErrorCallback: () => void
onErrorCallback: (error?: any) => void
}
type FileUpload = (v: FileUploadParams, isPublic?: boolean, url?: string) => void
export const fileUpload: FileUpload = ({
@@ -37,8 +56,8 @@ export const fileUpload: FileUpload = ({
.then((res: { id: string }) => {
onSuccessCallback(res)
})
.catch(() => {
onErrorCallback()
.catch((error) => {
onErrorCallback(error)
})
}

12
web/app/components/base/image-uploader/hooks.ts
View File

@@ -2,7 +2,7 @@ import { useCallback, useMemo, useRef, useState } from 'react'
import type { ClipboardEvent } from 'react'
import { useParams } from 'next/navigation'
import { useTranslation } from 'react-i18next'
import { imageUpload } from './utils'
import { getImageUploadErrorMessage, imageUpload } from './utils'
import { useToastContext } from '@/app/components/base/toast'
import { ALLOW_FILE_EXTENSIONS, TransferMethod } from '@/types/app'
import type { ImageFile, VisionSettings } from '@/types/app'
@@ -81,8 +81,9 @@ export const useImageFiles = () => {
filesRef.current = newFiles
setFiles(newFiles)
},
onErrorCallback: () => {
notify({ type: 'error', message: t('common.imageUploader.uploadFromComputerUploadError') })
onErrorCallback: (error?: any) => {
const errorMessage = getImageUploadErrorMessage(error, t('common.imageUploader.uploadFromComputerUploadError'), t)
notify({ type: 'error', message: errorMessage })
const newFiles = [...files.slice(0, index), { ...currentImageFile, progress: -1 }, ...files.slice(index + 1)]
filesRef.current = newFiles
setFiles(newFiles)
@@ -158,8 +159,9 @@ export const useLocalFileUploader = ({ limit, disabled = false, onUpload }: useL
onSuccessCallback: (res) => {
onUpload({ ...imageFile, fileId: res.id, progress: 100 })
},
onErrorCallback: () => {
notify({ type: 'error', message: t('common.imageUploader.uploadFromComputerUploadError') })
onErrorCallback: (error?: any) => {
const errorMessage = getImageUploadErrorMessage(error, t('common.imageUploader.uploadFromComputerUploadError'), t)
notify({ type: 'error', message: errorMessage })
onUpload({ ...imageFile, progress: -1 })
},
}, !!params.token)

25
web/app/components/base/image-uploader/utils.ts
View File

@@ -1,10 +1,29 @@
import { upload } from '@/service/base'
/**
* Get appropriate error message for image upload errors
* @param error - The error object from upload failure
* @param defaultMessage - Default error message to use if no specific error is matched
* @param t - Translation function
* @returns Localized error message
*/
export const getImageUploadErrorMessage = (error: any, defaultMessage: string, t: (key: string) => string): string => {
const errorCode = error?.response?.code
if (errorCode === 'forbidden')
return error?.response?.message
if (errorCode === 'file_extension_blocked')
return t('common.fileUploader.fileExtensionBlocked')
return defaultMessage
}
type ImageUploadParams = {
file: File
onProgressCallback: (progress: number) => void
onSuccessCallback: (res: { id: string }) => void
onErrorCallback: () => void
onErrorCallback: (error?: any) => void
}
type ImageUpload = (v: ImageUploadParams, isPublic?: boolean, url?: string) => void
export const imageUpload: ImageUpload = ({
@@ -30,7 +49,7 @@ export const imageUpload: ImageUpload = ({
.then((res: { id: string }) => {
onSuccessCallback(res)
})
.catch(() => {
onErrorCallback()
.catch((error) => {
onErrorCallback(error)
})
}

7
web/app/components/custom/custom-web-app-brand/index.tsx
View File

@@ -16,7 +16,7 @@ import Button from '@/app/components/base/button'
import Divider from '@/app/components/base/divider'
import { useProviderContext } from '@/context/provider-context'
import { Plan } from '@/app/components/billing/type'
import { imageUpload } from '@/app/components/base/image-uploader/utils'
import { getImageUploadErrorMessage, imageUpload } from '@/app/components/base/image-uploader/utils'
import { useToastContext } from '@/app/components/base/toast'
import { BubbleTextMod } from '@/app/components/base/icons/src/vender/solid/communication'
import {
@@ -67,8 +67,9 @@ const CustomWebAppBrand = () => {
setUploadProgress(100)
setFileId(res.id)
},
onErrorCallback: () => {
notify({ type: 'error', message: t('common.imageUploader.uploadFromComputerUploadError') })
onErrorCallback: (error?: any) => {
const errorMessage = getImageUploadErrorMessage(error, t('common.imageUploader.uploadFromComputerUploadError'), t)
notify({ type: 'error', message: errorMessage })
setUploadProgress(-1)
},
}, false, '/workspaces/custom-config/webapp-logo/upload')

4
web/app/components/datasets/create/file-uploader/index.tsx
View File

@@ -18,6 +18,7 @@ import { LanguagesSupported } from '@/i18n-config/language'
import { IS_CE_EDITION } from '@/config'
import { Theme } from '@/types/app'
import useTheme from '@/hooks/use-theme'
import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils'
type IFileUploaderProps = {
fileList: FileItem[]
@@ -134,7 +135,8 @@ const FileUploader = ({
return Promise.resolve({ ...completeFile })
})
.catch((e) => {
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
const errorMessage = getFileUploadErrorMessage(e, t('datasetCreation.stepOne.uploader.failed'), t)
notify({ type: 'error', message: errorMessage })
onFileUpdate(fileItem, -2, fileListRef.current)
return Promise.resolve({ ...fileItem })
})

4
web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx
View File

@@ -8,6 +8,7 @@ import cn from '@/utils/classnames'
import type { CustomFile as File, FileItem } from '@/models/datasets'
import { ToastContext } from '@/app/components/base/toast'
import { upload } from '@/service/base'
import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils'
import I18n from '@/context/i18n'
import { LanguagesSupported } from '@/i18n-config/language'
import { IS_CE_EDITION } from '@/config'
@@ -154,7 +155,8 @@ const LocalFile = ({
return Promise.resolve({ ...completeFile })
})
.catch((e) => {
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
const errorMessage = getFileUploadErrorMessage(e, t('datasetCreation.stepOne.uploader.failed'), t)
notify({ type: 'error', message: errorMessage })
updateFile(fileItem, -2, fileListRef.current)
return Promise.resolve({ ...fileItem })
})

4
web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
View File

@@ -12,6 +12,7 @@ import { ToastContext } from '@/app/components/base/toast'
import Button from '@/app/components/base/button'
import type { FileItem } from '@/models/datasets'
import { upload } from '@/service/base'
import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils'
import useSWR from 'swr'
import { fetchFileUploadConfig } from '@/service/common'
import SimplePieChart from '@/app/components/base/simple-pie-chart'
@@ -74,7 +75,8 @@ const CSVUploader: FC<Props> = ({
return Promise.resolve({ ...completeFile })
})
.catch((e) => {
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
const errorMessage = getFileUploadErrorMessage(e, t('datasetCreation.stepOne.uploader.failed'), t)
notify({ type: 'error', message: errorMessage })
const errorFile = {
...fileItem,
progress: -2,

1
web/i18n/en-US/common.ts
View File

@@ -734,6 +734,7 @@ const translation = {
uploadFromComputerLimit: 'Upload {{type}} cannot exceed {{size}}',
pasteFileLinkInvalid: 'Invalid file link',
fileExtensionNotSupport: 'File extension not supported',
fileExtensionBlocked: 'This file type is blocked for security reasons',
},
tag: {
placeholder: 'All Tags',

1
web/i18n/zh-Hans/common.ts
View File

@@ -728,6 +728,7 @@ const translation = {
uploadFromComputerLimit: '上传 {{type}} 不能超过 {{size}}',
pasteFileLinkInvalid: '文件链接无效',
fileExtensionNotSupport: '文件类型不支持',
fileExtensionBlocked: '出于安全考虑，该文件类型已被禁止上传',
},
tag: {
placeholder: '全部标签',