diff --git a/api/core/rag/entities/__init__.py b/api/core/rag/entities/__init__.py index ce48496110..63c6708704 100644 --- a/api/core/rag/entities/__init__.py +++ b/api/core/rag/entities/__init__.py @@ -1,12 +1,20 @@ from core.rag.entities.citation_metadata import RetrievalSourceMetadata from core.rag.entities.context_entities import DocumentContext +from core.rag.entities.event import DatasourceCompletedEvent, DatasourceErrorEvent, DatasourceProcessingEvent +from core.rag.entities.index_entities import EconomySetting, EmbeddingSetting, IndexMethod from core.rag.entities.metadata_entities import Condition, MetadataFilteringCondition, SupportedComparisonOperator from core.rag.entities.processing_entities import ParentMode, PreProcessingRule, Rule, Segmentation -from core.rag.entities.retrieval_settings import KeywordSetting, VectorSetting +from core.rag.entities.retrieval_settings import KeywordSetting, VectorSetting, WeightedScoreConfig __all__ = [ "Condition", + "DatasourceCompletedEvent", + "DatasourceErrorEvent", + "DatasourceProcessingEvent", "DocumentContext", + "EconomySetting", + "EmbeddingSetting", + "IndexMethod", "KeywordSetting", "MetadataFilteringCondition", "ParentMode", @@ -16,4 +24,5 @@ __all__ = [ "Segmentation", "SupportedComparisonOperator", "VectorSetting", + "WeightedScoreConfig", ] diff --git a/api/core/rag/entities/index_entities.py b/api/core/rag/entities/index_entities.py new file mode 100644 index 0000000000..f86a04fa9f --- /dev/null +++ b/api/core/rag/entities/index_entities.py @@ -0,0 +1,30 @@ +from typing import Literal + +from pydantic import BaseModel + + +class EmbeddingSetting(BaseModel): + """ + Embedding Setting. + """ + + embedding_provider_name: str + embedding_model_name: str + + +class EconomySetting(BaseModel): + """ + Economy Setting. + """ + + keyword_number: int + + +class IndexMethod(BaseModel): + """ + Knowledge Index Setting. + """ + + indexing_technique: Literal["high_quality", "economy"] + embedding_setting: EmbeddingSetting + economy_setting: EconomySetting diff --git a/api/core/rag/entities/retrieval_settings.py b/api/core/rag/entities/retrieval_settings.py index f52e0f0142..a0c6512c9c 100644 --- a/api/core/rag/entities/retrieval_settings.py +++ b/api/core/rag/entities/retrieval_settings.py @@ -17,3 +17,12 @@ class KeywordSetting(BaseModel): """ keyword_weight: float + + +class WeightedScoreConfig(BaseModel): + """ + Weighted score Config. + """ + + vector_setting: VectorSetting + keyword_setting: KeywordSetting diff --git a/api/core/workflow/nodes/knowledge_index/entities.py b/api/core/workflow/nodes/knowledge_index/entities.py index f4789a42a5..6ff162973c 100644 --- a/api/core/workflow/nodes/knowledge_index/entities.py +++ b/api/core/workflow/nodes/knowledge_index/entities.py @@ -1,10 +1,10 @@ -from typing import Literal, Union +from typing import Union from graphon.entities.base_node_data import BaseNodeData from graphon.enums import NodeType from pydantic import BaseModel -from core.rag.entities import KeywordSetting, VectorSetting +from core.rag.entities import WeightedScoreConfig from core.rag.index_processor.index_processor_base import SummaryIndexSettingDict from core.rag.retrieval.retrieval_methods import RetrievalMethod from core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPE @@ -19,32 +19,6 @@ class RerankingModelConfig(BaseModel): reranking_model_name: str -class WeightedScoreConfig(BaseModel): - """ - Weighted score Config. - """ - - vector_setting: VectorSetting - keyword_setting: KeywordSetting - - -class EmbeddingSetting(BaseModel): - """ - Embedding Setting. - """ - - embedding_provider_name: str - embedding_model_name: str - - -class EconomySetting(BaseModel): - """ - Economy Setting. - """ - - keyword_number: int - - class RetrievalSetting(BaseModel): """ Retrieval Setting. @@ -60,16 +34,6 @@ class RetrievalSetting(BaseModel): weights: WeightedScoreConfig | None = None -class IndexMethod(BaseModel): - """ - Knowledge Index Setting. - """ - - indexing_technique: Literal["high_quality", "economy"] - embedding_setting: EmbeddingSetting - economy_setting: EconomySetting - - class FileInfo(BaseModel): """ File Info. diff --git a/api/core/workflow/nodes/knowledge_retrieval/entities.py b/api/core/workflow/nodes/knowledge_retrieval/entities.py index 00a36e8d5f..f4bc3fb9d3 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/entities.py +++ b/api/core/workflow/nodes/knowledge_retrieval/entities.py @@ -5,7 +5,7 @@ from graphon.enums import BuiltinNodeTypes, NodeType from graphon.nodes.llm.entities import ModelConfig, VisionConfig from pydantic import BaseModel, Field -from core.rag.entities import Condition, KeywordSetting, MetadataFilteringCondition, VectorSetting +from core.rag.entities import Condition, MetadataFilteringCondition, WeightedScoreConfig __all__ = ["Condition"] @@ -19,15 +19,6 @@ class RerankingModelConfig(BaseModel): model: str -class WeightedScoreConfig(BaseModel): - """ - Weighted score Config. - """ - - vector_setting: VectorSetting - keyword_setting: KeywordSetting - - class MultipleRetrievalConfig(BaseModel): """ Multiple Retrieval Config. diff --git a/api/services/entities/knowledge_entities/rag_pipeline_entities.py b/api/services/entities/knowledge_entities/rag_pipeline_entities.py index 07fbe963d6..a360fd2854 100644 --- a/api/services/entities/knowledge_entities/rag_pipeline_entities.py +++ b/api/services/entities/knowledge_entities/rag_pipeline_entities.py @@ -46,23 +46,6 @@ class WeightedScoreConfig(BaseModel): keyword_setting: KeywordSetting | None -class EmbeddingSetting(BaseModel): - """ - Embedding Setting. - """ - - embedding_provider_name: str - embedding_model_name: str - - -class EconomySetting(BaseModel): - """ - Economy Setting. - """ - - keyword_number: int - - class RetrievalSetting(BaseModel): """ Retrieval Setting. @@ -78,16 +61,6 @@ class RetrievalSetting(BaseModel): weights: WeightedScoreConfig | None = None -class IndexMethod(BaseModel): - """ - Knowledge Index Setting. - """ - - indexing_technique: Literal["high_quality", "economy"] - embedding_setting: EmbeddingSetting - economy_setting: EconomySetting - - class KnowledgeConfiguration(BaseModel): """ Knowledge Base Configuration. diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.py index 5b3668aebb..f03d0b33ad 100644 --- a/api/services/rag_pipeline/rag_pipeline.py +++ b/api/services/rag_pipeline/rag_pipeline.py @@ -38,11 +38,7 @@ from core.datasource.online_document.online_document_plugin import OnlineDocumen from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin from core.helper import marketplace -from core.rag.entities.event import ( - DatasourceCompletedEvent, - DatasourceErrorEvent, - DatasourceProcessingEvent, -) +from core.rag.entities import DatasourceCompletedEvent, DatasourceErrorEvent, DatasourceProcessingEvent from core.repositories.factory import DifyCoreRepositoryFactory, OrderConfig from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository from core.workflow.node_factory import LATEST_VERSION, get_node_type_classes_mapping