refactor: update installed app component to handle missing params and improve type safety (#27331)

2025-12-25 01:00:42 -05:00 · 2025-10-27 14:38:58 +08:00
parent f06025a342
commit 43bcf40f80
49 changed files with 531 additions and 302 deletions
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx
@@ -121,6 +121,8 @@ const LocalFile = ({
    return isValidType && isValidSize
  }, [fileUploadConfig, notify, t, ACCEPTS])

+  type UploadResult = Awaited<ReturnType<typeof upload>>
+
  const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
    const formData = new FormData()
    formData.append('file', fileItem.file)
@@ -136,10 +138,14 @@ const LocalFile = ({
      data: formData,
      onprogress: onProgress,
    }, false, undefined, '?source=datasets')
-      .then((res: File) => {
-        const completeFile = {
+      .then((res: UploadResult) => {
+        const updatedFile = Object.assign({}, fileItem.file, {
+          id: res.id,
+          ...(res as Partial<File>),
+        }) as File
+        const completeFile: FileItem = {
          fileID: fileItem.fileID,
-          file: res,
+          file: updatedFile,
          progress: -1,
        }
        const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
--- a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
+++ b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx
@@ -38,6 +38,8 @@ const CSVUploader: FC<Props> = ({
    file_size_limit: 15,
  }, [fileUploadConfigResponse])

+  type UploadResult = Awaited<ReturnType<typeof upload>>
+
  const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
    fileItem.progress = 0

@@ -58,10 +60,14 @@ const CSVUploader: FC<Props> = ({
      data: formData,
      onprogress: onProgress,
    }, false, undefined, '?source=datasets')
-      .then((res: File) => {
-        const completeFile = {
+      .then((res: UploadResult) => {
+        const updatedFile = Object.assign({}, fileItem.file, {
+          id: res.id,
+          ...(res as Partial<File>),
+        }) as File
+        const completeFile: FileItem = {
          fileID: fileItem.fileID,
-          file: res,
+          file: updatedFile,
          progress: 100,
        }
        updateFile(completeFile)
--- a/web/app/components/datasets/documents/detail/index.tsx
+++ b/web/app/components/datasets/documents/detail/index.tsx
@@ -17,7 +17,7 @@ import Divider from '@/app/components/base/divider'
 import Loading from '@/app/components/base/loading'
 import Toast from '@/app/components/base/toast'
 import { ChunkingMode } from '@/models/datasets'
-import type { FileItem } from '@/models/datasets'
+import type { DataSourceInfo, FileItem, LegacyDataSourceInfo } from '@/models/datasets'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import FloatRightContainer from '@/app/components/base/float-right-container'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
@@ -109,6 +109,18 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {

  const embedding = ['queuing', 'indexing', 'paused'].includes((documentDetail?.display_status || '').toLowerCase())

+  const isLegacyDataSourceInfo = (info?: DataSourceInfo): info is LegacyDataSourceInfo => {
+    return !!info && 'upload_file' in info
+  }
+
+  const documentUploadFile = useMemo(() => {
+    if (!documentDetail?.data_source_info)
+      return undefined
+    if (isLegacyDataSourceInfo(documentDetail.data_source_info))
+      return documentDetail.data_source_info.upload_file
+    return undefined
+  }, [documentDetail?.data_source_info])
+
  const invalidChunkList = useInvalid(useSegmentListKey)
  const invalidChildChunkList = useInvalid(useChildSegmentListKey)
  const invalidDocumentList = useInvalidDocumentList(datasetId)
@@ -153,7 +165,7 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {
          </div>
          <DocumentTitle
            datasetId={datasetId}
-            extension={documentDetail?.data_source_info?.upload_file?.extension}
+            extension={documentUploadFile?.extension}
            name={documentDetail?.name}
            wrapperCls='mr-2'
            parent_mode={parentMode}
--- a/web/app/components/datasets/documents/detail/metadata/index.tsx
+++ b/web/app/components/datasets/documents/detail/metadata/index.tsx
@@ -131,9 +131,15 @@ type IMetadataProps = {
  onUpdate: () => void
 }

+type MetadataState = {
+  documentType?: DocType | ''
+  metadata: Record<string, string>
+}
+
 const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {
  const { doc_metadata = {} } = docDetail || {}
-  const doc_type = docDetail?.doc_type || ''
+  const rawDocType = docDetail?.doc_type ?? ''
+  const doc_type = rawDocType === 'others' ? '' : rawDocType

  const { t } = useTranslation()
  const metadataMap = useMetadataMap()
@@ -143,18 +149,16 @@ const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {
  const businessDocCategoryMap = useBusinessDocCategories()
  const [editStatus, setEditStatus] = useState(!doc_type) // if no documentType, in editing status by default
  // the initial values are according to the documentType
-  const [metadataParams, setMetadataParams] = useState<{
-    documentType?: DocType | ''
-    metadata: { [key: string]: string }
-  }>(
+  const [metadataParams, setMetadataParams] = useState<MetadataState>(
    doc_type
      ? {
-        documentType: doc_type,
-        metadata: doc_metadata || {},
+        documentType: doc_type as DocType,
+        metadata: (doc_metadata || {}) as Record<string, string>,
      }
-      : { metadata: {} })
+      : { metadata: {} },
+  )
  const [showDocTypes, setShowDocTypes] = useState(!doc_type) // whether show doc types
-  const [tempDocType, setTempDocType] = useState<DocType | undefined | ''>('') // for remember icon click
+  const [tempDocType, setTempDocType] = useState<DocType | ''>('') // for remember icon click
  const [saveLoading, setSaveLoading] = useState(false)

  const { notify } = useContext(ToastContext)
@@ -165,13 +169,13 @@ const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {
    if (docDetail?.doc_type) {
      setEditStatus(false)
      setShowDocTypes(false)
-      setTempDocType(docDetail?.doc_type)
+      setTempDocType(doc_type as DocType | '')
      setMetadataParams({
-        documentType: docDetail?.doc_type,
-        metadata: docDetail?.doc_metadata || {},
+        documentType: doc_type as DocType | '',
+        metadata: (docDetail?.doc_metadata || {}) as Record<string, string>,
      })
    }
-  }, [docDetail?.doc_type])
+  }, [docDetail?.doc_type, docDetail?.doc_metadata, doc_type])

  // confirm doc type
  const confirmDocType = () => {
@@ -179,7 +183,7 @@ const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {
      return
    setMetadataParams({
      documentType: tempDocType,
-      metadata: tempDocType === metadataParams.documentType ? metadataParams.metadata : {}, // change doc type, clear metadata
+      metadata: tempDocType === metadataParams.documentType ? metadataParams.metadata : {} as Record<string, string>, // change doc type, clear metadata
    })
    setEditStatus(true)
    setShowDocTypes(false)
@@ -187,7 +191,7 @@ const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {

  // cancel doc type
  const cancelDocType = () => {
-    setTempDocType(metadataParams.documentType)
+    setTempDocType(metadataParams.documentType ?? '')
    setEditStatus(true)
    setShowDocTypes(false)
  }
@@ -209,7 +213,7 @@ const Metadata: FC<IMetadataProps> = ({ docDetail, loading, onUpdate }) => {
            <span className={s.title}>{t('datasetDocuments.metadata.docTypeChangeTitle')}</span>
            <span className={s.changeTip}>{t('datasetDocuments.metadata.docTypeSelectWarning')}</span>
          </>}
-          <Radio.Group value={tempDocType ?? documentType} onChange={setTempDocType} className={s.radioGroup}>
+          <Radio.Group value={tempDocType ?? documentType ?? ''} onChange={setTempDocType} className={s.radioGroup}>
            {CUSTOMIZABLE_DOC_TYPES.map((type, index) => {
              const currValue = tempDocType ?? documentType
              return <Radio key={index} value={type} className={`${s.radio} ${currValue === type ? 'shadow-none' : ''}`}>
--- a/web/app/components/datasets/documents/detail/settings/document-settings.tsx
+++ b/web/app/components/datasets/documents/detail/settings/document-settings.tsx
@@ -4,7 +4,17 @@ import { useBoolean } from 'ahooks'
 import { useContext } from 'use-context-selector'
 import { useRouter } from 'next/navigation'
 import DatasetDetailContext from '@/context/dataset-detail'
-import type { CrawlOptions, CustomFile, DataSourceType } from '@/models/datasets'
+import type {
+  CrawlOptions,
+  CustomFile,
+  DataSourceInfo,
+  DataSourceType,
+  LegacyDataSourceInfo,
+  LocalFileInfo,
+  OnlineDocumentInfo,
+  WebsiteCrawlInfo,
+} from '@/models/datasets'
+import type { DataSourceProvider } from '@/models/common'
 import Loading from '@/app/components/base/loading'
 import StepTwo from '@/app/components/datasets/create/step-two'
 import AccountSetting from '@/app/components/header/account-setting'
@@ -42,15 +52,78 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
    params: { metadata: 'without' },
  })

+  const dataSourceInfo = documentDetail?.data_source_info
+
+  const isLegacyDataSourceInfo = (info: DataSourceInfo | undefined): info is LegacyDataSourceInfo => {
+    return !!info && 'upload_file' in info
+  }
+  const isWebsiteCrawlInfo = (info: DataSourceInfo | undefined): info is WebsiteCrawlInfo => {
+    return !!info && 'source_url' in info && 'title' in info
+  }
+  const isOnlineDocumentInfo = (info: DataSourceInfo | undefined): info is OnlineDocumentInfo => {
+    return !!info && 'page' in info
+  }
+  const isLocalFileInfo = (info: DataSourceInfo | undefined): info is LocalFileInfo => {
+    return !!info && 'related_id' in info && 'transfer_method' in info
+  }
+  const legacyInfo = isLegacyDataSourceInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const websiteInfo = isWebsiteCrawlInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const onlineDocumentInfo = isOnlineDocumentInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const localFileInfo = isLocalFileInfo(dataSourceInfo) ? dataSourceInfo : undefined
+
  const currentPage = useMemo(() => {
-    return {
-      workspace_id: documentDetail?.data_source_info.notion_workspace_id,
-      page_id: documentDetail?.data_source_info.notion_page_id,
-      page_name: documentDetail?.name,
-      page_icon: documentDetail?.data_source_info.notion_page_icon,
-      type: documentDetail?.data_source_type,
+    if (legacyInfo) {
+      return {
+        workspace_id: legacyInfo.notion_workspace_id ?? '',
+        page_id: legacyInfo.notion_page_id ?? '',
+        page_name: documentDetail?.name,
+        page_icon: legacyInfo.notion_page_icon,
+        type: documentDetail?.data_source_type,
+      }
    }
-  }, [documentDetail])
+    if (onlineDocumentInfo) {
+      return {
+        workspace_id: onlineDocumentInfo.workspace_id,
+        page_id: onlineDocumentInfo.page.page_id,
+        page_name: onlineDocumentInfo.page.page_name,
+        page_icon: onlineDocumentInfo.page.page_icon,
+        type: onlineDocumentInfo.page.type,
+      }
+    }
+    return undefined
+  }, [documentDetail?.data_source_type, documentDetail?.name, legacyInfo, onlineDocumentInfo])
+
+  const files = useMemo<CustomFile[]>(() => {
+    if (legacyInfo?.upload_file)
+      return [legacyInfo.upload_file as CustomFile]
+    if (localFileInfo) {
+      const { related_id, name, extension } = localFileInfo
+      return [{
+        id: related_id,
+        name,
+        extension,
+      } as unknown as CustomFile]
+    }
+    return []
+  }, [legacyInfo?.upload_file, localFileInfo])
+
+  const websitePages = useMemo(() => {
+    if (!websiteInfo)
+      return []
+    return [{
+      title: websiteInfo.title,
+      source_url: websiteInfo.source_url,
+      content: websiteInfo.content,
+      description: websiteInfo.description,
+    }]
+  }, [websiteInfo])
+
+  const crawlOptions = (dataSourceInfo && typeof dataSourceInfo === 'object' && 'includes' in dataSourceInfo && 'excludes' in dataSourceInfo)
+    ? dataSourceInfo as unknown as CrawlOptions
+    : undefined
+
+  const websiteCrawlProvider = (websiteInfo?.provider ?? legacyInfo?.provider) as DataSourceProvider | undefined
+  const websiteCrawlJobId = websiteInfo?.job_id ?? legacyInfo?.job_id

  if (error)
    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
@@ -65,22 +138,16 @@ const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
            onSetting={showSetAPIKey}
            datasetId={datasetId}
            dataSourceType={documentDetail.data_source_type as DataSourceType}
-            notionPages={[currentPage as unknown as NotionPage]}
-            websitePages={[
-              {
-                title: documentDetail.name,
-                source_url: documentDetail.data_source_info?.url,
-                content: '',
-                description: '',
-              },
-            ]}
-            websiteCrawlProvider={documentDetail.data_source_info?.provider}
-            websiteCrawlJobId={documentDetail.data_source_info?.job_id}
-            crawlOptions={documentDetail.data_source_info as unknown as CrawlOptions}
+            notionPages={currentPage ? [currentPage as unknown as NotionPage] : []}
+            notionCredentialId={legacyInfo?.credential_id || onlineDocumentInfo?.credential_id || ''}
+            websitePages={websitePages}
+            websiteCrawlProvider={websiteCrawlProvider}
+            websiteCrawlJobId={websiteCrawlJobId || ''}
+            crawlOptions={crawlOptions}
            indexingType={indexingTechnique}
            isSetting
            documentDetail={documentDetail}
-            files={[documentDetail.data_source_info.upload_file as CustomFile]}
+            files={files}
            onSave={saveHandler}
            onCancel={cancelHandler}
          />