feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: twwu <twwu@dify.ai> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: jyong <718720800@qq.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com> Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com> Co-authored-by: quicksand <quicksandzn@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Hanqing Zhao <sherry9277@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry <xh001x@hotmail.com>
2025-12-25 10:01:31 -05:00 · 2025-09-18 12:49:10 +08:00
parent 7dadb33003
commit 85cda47c70
1772 changed files with 102407 additions and 31710 deletions
--- a/web/app/components/datasets/create/website/base/header.tsx
+++ b/web/app/components/datasets/create/website/base/header.tsx
@@ -0,0 +1,61 @@
+import React from 'react'
+import Divider from '@/app/components/base/divider'
+import Button from '@/app/components/base/button'
+import cn from '@/utils/classnames'
+import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
+
+type HeaderProps = {
+  isInPipeline?: boolean
+  onClickConfiguration?: () => void
+  title: string
+  buttonText?: string
+  docTitle: string
+  docLink: string
+}
+
+const Header = ({
+  isInPipeline = false,
+  onClickConfiguration,
+  title,
+  buttonText,
+  docTitle,
+  docLink,
+}: HeaderProps) => {
+  return (
+    <div className='flex items-center gap-x-2'>
+      <div className='flex shrink-0 grow items-center gap-x-1'>
+        <div className={cn(
+          'text-text-secondary',
+          isInPipeline ? 'system-sm-semibold' : 'system-md-semibold',
+        )}>
+          {title}
+        </div>
+        <Divider type='vertical' className='mx-1 h-3.5' />
+        <Button
+          variant='secondary'
+          size='small'
+          className={cn(isInPipeline ? 'size-6 px-1' : 'gap-x-0.5 px-1.5')}
+          onClick={onClickConfiguration}
+        >
+          <RiEqualizer2Line className='size-4' />
+          {!isInPipeline && (
+            <span className='system-xs-medium'>
+              {buttonText}
+            </span>
+          )}
+        </Button>
+      </div>
+      <a
+        className='system-xs-medium flex items-center gap-x-1 overflow-hidden text-text-accent'
+        href={docLink}
+        target='_blank'
+        rel='noopener noreferrer'
+      >
+        <RiBookOpenLine className='size-3.5 shrink-0' />
+        <span className='grow truncate' title={docTitle}>{docTitle}</span>
+      </a>
+    </div>
+  )
+}
+
+export default React.memo(Header)
--- a/web/app/components/datasets/create/website/base/input.tsx
+++ b/web/app/components/datasets/create/website/base/input.tsx
@@ -21,7 +21,7 @@ const Input: FC<Props> = ({
    const value = e.target.value
    if (isNumber) {
      let numberValue = Number.parseInt(value, 10) // integer only
-      if (isNaN(numberValue)) {
+      if (Number.isNaN(numberValue)) {
        onChange('')
        return
      }
--- a/web/app/components/datasets/create/website/base/mock-crawl-result.ts
+++ b/web/app/components/datasets/create/website/base/mock-crawl-result.ts
@@ -3,19 +3,19 @@ import type { CrawlResultItem } from '@/models/datasets'
 const result: CrawlResultItem[] = [
  {
    title: 'Start the frontend Docker container separately',
-    markdown: 'Markdown 1',
+    content: 'Markdown 1',
    description: 'Description 1',
    source_url: 'https://example.com/1',
  },
  {
    title: 'Advanced Tool Integration',
-    markdown: 'Markdown 2',
+    content: 'Markdown 2',
    description: 'Description 2',
    source_url: 'https://example.com/2',
  },
  {
    title: 'Local Source Code Start | English | Dify',
-    markdown: 'Markdown 3',
+    content: 'Markdown 3',
    description: 'Description 3',
    source_url: 'https://example.com/3',
  },
--- a/web/app/components/datasets/create/website/firecrawl/header.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/header.tsx
@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureFirecrawl`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://docs.firecrawl.dev/introduction'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.firecrawlDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
-import { useModalContext } from '@/context/modal-context'
+import { useModalContextSelector } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
+import Header from '../base/header'

 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -48,7 +48,7 @@ const FireCrawl: FC<Props> = ({
    if (step !== Step.init)
      setControlFoldOptions(Date.now())
  }, [step])
-  const { setShowAccountSettingModal } = useModalContext()
+  const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
  const handleSetting = useCallback(() => {
    setShowAccountSettingModal({
      payload: 'data-source',
@@ -112,6 +112,10 @@ const FireCrawl: FC<Props> = ({
          },
        }
      }
+      res.data = res.data.map((item: any) => ({
+        ...item,
+        content: item.markdown,
+      }))
      // update the progress
      setCrawlResult({
        ...res,
@@ -131,7 +135,7 @@ const FireCrawl: FC<Props> = ({
        },
      }
    }
-  }, [crawlOptions.limit])
+  }, [crawlOptions.limit, onCheckedCrawlResultChange])

  const handleRun = useCallback(async (url: string) => {
    const { isValid, errorMsg } = checkValid(url)
@@ -161,6 +165,10 @@ const FireCrawl: FC<Props> = ({
        setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
      }
      else {
+        data.data = data.data.map((item: any) => ({
+          ...item,
+          content: item.markdown,
+        }))
        setCrawlResult(data)
        onCheckedCrawlResultChange(data.data || []) // default select the crawl result
        setCrawlErrorMessage('')
@@ -173,11 +181,17 @@ const FireCrawl: FC<Props> = ({
    finally {
      setStep(Step.finished)
    }
-  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])

  return (
    <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.firecrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
+        docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
+        docLink={'https://docs.firecrawl.dev/introduction'}
+      />
      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
        <UrlInput onRun={handleRun} isRunning={isRunning} />
        <OptionsWrap
--- a/web/app/components/datasets/create/website/index.module.css
+++ b/web/app/components/datasets/create/website/index.module.css
@@ -6,7 +6,7 @@
 }

 .watercrawlLogo {
-  @apply w-5 h-5 bg-center bg-no-repeat inline-block;
+  @apply w-4 h-4 bg-center bg-no-repeat inline-block;
  /*background-color: #F5FAFF;*/
  background-image: url(../assets/watercrawl.svg);
  background-size: 16px;
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback, useEffect, useState } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import s from './index.module.css'
 import NoData from './no-data'
@@ -10,9 +10,9 @@ import JinaReader from './jina-reader'
 import cn from '@/utils/classnames'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
-import { fetchDataSources } from '@/service/datasets'
-import { type DataSourceItem, DataSourceProvider } from '@/models/common'
+import { DataSourceProvider } from '@/models/common'
 import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'

 type Props = {
  onPreview: (payload: CrawlResultItem) => void
@@ -22,6 +22,7 @@ type Props = {
  onJobIdChange: (jobId: string) => void
  crawlOptions: CrawlOptions
  onCrawlOptionsChange: (payload: CrawlOptions) => void
+  authedDataSourceList: DataSourceAuth[]
 }

 const Website: FC<Props> = ({
@@ -32,58 +33,34 @@ const Website: FC<Props> = ({
  onJobIdChange,
  crawlOptions,
  onCrawlOptionsChange,
+  authedDataSourceList,
 }) => {
  const { t } = useTranslation()
  const { setShowAccountSettingModal } = useModalContext()
-  const [isLoaded, setIsLoaded] = useState(false)
  const [selectedProvider, setSelectedProvider] = useState<DataSourceProvider>(DataSourceProvider.jinaReader)
-  const [sources, setSources] = useState<DataSourceItem[]>([])

-  useEffect(() => {
-    onCrawlProviderChange(selectedProvider)
-  }, [selectedProvider, onCrawlProviderChange])
+  const availableProviders = useMemo(() => authedDataSourceList.filter((item) => {
+    return [
+      DataSourceProvider.jinaReader,
+      DataSourceProvider.fireCrawl,
+      DataSourceProvider.waterCrawl].includes(item.provider as DataSourceProvider) && item.credentials_list.length > 0
+  }), [authedDataSourceList])

-  const checkSetApiKey = useCallback(async () => {
-    const res = await fetchDataSources() as any
-    setSources(res.sources)
-
-    // If users have configured one of the providers, select it.
-    const availableProviders = res.sources.filter((item: DataSourceItem) =>
-      [
-        DataSourceProvider.jinaReader,
-        DataSourceProvider.fireCrawl,
-        DataSourceProvider.waterCrawl,
-      ].includes(item.provider),
-    )
-
-    if (availableProviders.length > 0)
-      setSelectedProvider(availableProviders[0].provider)
-  }, [])
-
-  useEffect(() => {
-    checkSetApiKey().then(() => {
-      setIsLoaded(true)
-    })
-  }, [])
  const handleOnConfig = useCallback(() => {
    setShowAccountSettingModal({
      payload: 'data-source',
-      onCancelCallback: checkSetApiKey,
    })
-  }, [checkSetApiKey, setShowAccountSettingModal])
+  }, [setShowAccountSettingModal])

-  if (!isLoaded)
-    return null
-
-  const source = sources.find(source => source.provider === selectedProvider)
+  const source = availableProviders.find(source => source.provider === selectedProvider)

  return (
    <div>
-      <div className="mb-4">
-        <div className="system-md-medium mb-2 text-text-secondary">
+      <div className='mb-4'>
+        <div className='system-md-medium mb-2 text-text-secondary'>
          {t('datasetCreation.stepOne.website.chooseProvider')}
        </div>
-        <div className="flex space-x-2">
+        <div className='flex space-x-2'>
          {ENABLE_WEBSITE_JINAREADER && <button
            className={cn('flex items-center justify-center rounded-lg px-4 py-2',
              selectedProvider === DataSourceProvider.jinaReader
@@ -91,9 +68,12 @@ const Website: FC<Props> = ({
                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
            )}
-            onClick={() => setSelectedProvider(DataSourceProvider.jinaReader)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.jinaReader)
+              onCrawlProviderChange(DataSourceProvider.jinaReader)
+            }}
          >
-            <span className={cn(s.jinaLogo, 'mr-2')}/>
+            <span className={cn(s.jinaLogo, 'mr-2')} />
            <span>Jina Reader</span>
          </button>}
          {ENABLE_WEBSITE_FIRECRAWL && <button
@@ -103,7 +83,10 @@ const Website: FC<Props> = ({
                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
            )}
-            onClick={() => setSelectedProvider(DataSourceProvider.fireCrawl)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.fireCrawl)
+              onCrawlProviderChange(DataSourceProvider.fireCrawl)
+            }}
          >
            🔥 Firecrawl
          </button>}
@@ -114,9 +97,12 @@ const Website: FC<Props> = ({
                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
            )}
-            onClick={() => setSelectedProvider(DataSourceProvider.waterCrawl)}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.waterCrawl)
+              onCrawlProviderChange(DataSourceProvider.waterCrawl)
+            }}
          >
-            <span className={cn(s.watercrawlLogo, 'mr-2')}/>
+            <span className={cn(s.watercrawlLogo, 'mr-2')} />
            <span>WaterCrawl</span>
          </button>}
        </div>
@@ -152,7 +138,7 @@ const Website: FC<Props> = ({
        />
      )}
      {!source && (
-        <NoData onConfig={handleOnConfig} provider={selectedProvider}/>
+        <NoData onConfig={handleOnConfig} provider={selectedProvider} />
      )}
    </div>
  )
--- a/web/app/components/datasets/create/website/jina-reader/base/checkbox-with-label.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/checkbox-with-label.tsx
@@ -1,40 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import cn from '@/utils/classnames'
-import Checkbox from '@/app/components/base/checkbox'
-import Tooltip from '@/app/components/base/tooltip'
-
-type Props = {
-  className?: string
-  isChecked: boolean
-  onChange: (isChecked: boolean) => void
-  label: string
-  labelClassName?: string
-  tooltip?: string
-}
-
-const CheckboxWithLabel: FC<Props> = ({
-  className = '',
-  isChecked,
-  onChange,
-  label,
-  labelClassName,
-  tooltip,
-}) => {
-  return (
-    <label className={cn(className, 'flex h-7 items-center space-x-2')}>
-      <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
-      <div className={cn('text-sm font-normal text-gray-800', labelClassName)}>{label}</div>
-      {tooltip && (
-        <Tooltip
-          popupContent={
-            <div className='w-[200px]'>{tooltip}</div>
-          }
-          triggerClassName='ml-0.5 w-4 h-4'
-        />
-      )}
-    </label>
-  )
-}
-export default React.memo(CheckboxWithLabel)
--- a/web/app/components/datasets/create/website/jina-reader/base/error-message.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/error-message.tsx
@@ -1,30 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import cn from '@/utils/classnames'
-import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
-
-type Props = {
-  className?: string
-  title: string
-  errorMsg?: string
-}
-
-const ErrorMessage: FC<Props> = ({
-  className,
-  title,
-  errorMsg,
-}) => {
-  return (
-    <div className={cn(className, 'border-t border-gray-200 bg-[#FFFAEB] px-4 py-2')}>
-      <div className='flex h-5 items-center'>
-        <AlertTriangle className='mr-2 h-4 w-4 text-[#F79009]' />
-        <div className='text-sm font-medium text-[#DC6803]'>{title}</div>
-      </div>
-      {errorMsg && (
-        <div className='mt-1 pl-6 text-xs font-normal leading-[18px] text-gray-700'>{errorMsg}</div>
-      )}
-    </div>
-  )
-}
-export default React.memo(ErrorMessage)
--- a/web/app/components/datasets/create/website/jina-reader/base/field.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/field.tsx
@@ -1,54 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import Input from './input'
-import cn from '@/utils/classnames'
-import Tooltip from '@/app/components/base/tooltip'
-
-type Props = {
-  className?: string
-  label: string
-  labelClassName?: string
-  value: string | number
-  onChange: (value: string | number) => void
-  isRequired?: boolean
-  placeholder?: string
-  isNumber?: boolean
-  tooltip?: string
-}
-
-const Field: FC<Props> = ({
-  className,
-  label,
-  labelClassName,
-  value,
-  onChange,
-  isRequired = false,
-  placeholder = '',
-  isNumber = false,
-  tooltip,
-}) => {
-  return (
-    <div className={cn(className)}>
-      <div className='flex py-[7px]'>
-        <div className={cn(labelClassName, 'flex h-[18px] items-center text-[13px] font-medium text-gray-900')}>{label} </div>
-        {isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
-        {tooltip && (
-          <Tooltip
-            popupContent={
-              <div className='w-[200px]'>{tooltip}</div>
-            }
-            triggerClassName='ml-0.5 w-4 h-4'
-          />
-        )}
-      </div>
-      <Input
-        value={value}
-        onChange={onChange}
-        placeholder={placeholder}
-        isNumber={isNumber}
-      />
-    </div>
-  )
-}
-export default React.memo(Field)
--- a/web/app/components/datasets/create/website/jina-reader/base/input.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/input.tsx
@@ -1,58 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React, { useCallback } from 'react'
-
-type Props = {
-  value: string | number
-  onChange: (value: string | number) => void
-  placeholder?: string
-  isNumber?: boolean
-}
-
-const MIN_VALUE = 0
-
-const Input: FC<Props> = ({
-  value,
-  onChange,
-  placeholder = '',
-  isNumber = false,
-}) => {
-  const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
-    const value = e.target.value
-    if (isNumber) {
-      let numberValue = Number.parseInt(value, 10) // integer only
-      if (isNaN(numberValue)) {
-        onChange('')
-        return
-      }
-      if (numberValue < MIN_VALUE)
-        numberValue = MIN_VALUE
-
-      onChange(numberValue)
-      return
-    }
-    onChange(value)
-  }, [isNumber, onChange])
-
-  const otherOption = (() => {
-    if (isNumber) {
-      return {
-        min: MIN_VALUE,
-      }
-    }
-    return {
-
-    }
-  })()
-  return (
-    <input
-      type={isNumber ? 'number' : 'text'}
-      {...otherOption}
-      value={value}
-      onChange={handleChange}
-      className='flex h-9 w-full rounded-lg bg-gray-100 px-2 py-1 text-xs leading-normal caret-primary-600 placeholder:text-gray-400 hover:bg-gray-100 focus:bg-gray-50 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none'
-      placeholder={placeholder}
-    />
-  )
-}
-export default React.memo(Input)
--- a/web/app/components/datasets/create/website/jina-reader/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/options-wrap.tsx
@@ -1,54 +0,0 @@
-'use client'
-import { useBoolean } from 'ahooks'
-import type { FC } from 'react'
-import React, { useEffect } from 'react'
-import { useTranslation } from 'react-i18next'
-import cn from '@/utils/classnames'
-import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
-import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  className?: string
-  children: React.ReactNode
-  controlFoldOptions?: number
-}
-
-const OptionsWrap: FC<Props> = ({
-  className = '',
-  children,
-  controlFoldOptions,
-}) => {
-  const { t } = useTranslation()
-
-  const [fold, {
-    toggle: foldToggle,
-    setTrue: foldHide,
-  }] = useBoolean(false)
-
-  useEffect(() => {
-    if (controlFoldOptions)
-      foldHide()
-  }, [controlFoldOptions])
-  return (
-    <div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
-      <div
-        className='flex h-[26px] cursor-pointer select-none items-center justify-between py-1'
-        onClick={foldToggle}
-      >
-        <div className='flex items-center text-gray-700'>
-          <Settings04 className='mr-1 h-4 w-4' />
-          <div className='text-[13px] font-semibold uppercase text-gray-800'>{t(`${I18N_PREFIX}.options`)}</div>
-        </div>
-        <ChevronRight className={cn(!fold && 'rotate-90', 'h-4 w-4 text-gray-500')} />
-      </div>
-      {!fold && (
-        <div className='mb-4'>
-          {children}
-        </div>
-      )}
-
-    </div>
-  )
-}
-export default React.memo(OptionsWrap)
--- a/web/app/components/datasets/create/website/jina-reader/base/url-input.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/base/url-input.tsx
@@ -2,7 +2,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import Input from './input'
+import Input from '../../base/input'
 import Button from '@/app/components/base/button'
 import { useDocLink } from '@/context/i18n'

--- a/web/app/components/datasets/create/website/jina-reader/header.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/header.tsx
@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-sm font-semibold text-text-secondary'>{t(`${I18N_PREFIX}.jinaReaderTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureJinaReader`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://jina.ai/reader'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.jinaReaderDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)
--- a/web/app/components/datasets/create/website/jina-reader/index.tsx
+++ b/web/app/components/datasets/create/website/jina-reader/index.tsx
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
 import { useModalContext } from '@/context/modal-context'
 import Toast from '@/app/components/base/toast'
 import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
 import { sleep } from '@/utils'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import Header from '../base/header'

 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -150,14 +150,15 @@ const JinaReader: FC<Props> = ({
      }) as any

      if (res.data) {
+        const { title, content, description, url } = res.data
        const data = {
          current: 1,
          total: 1,
          data: [{
-            title: res.data.title,
-            markdown: res.data.content,
-            description: res.data.description,
-            source_url: res.data.url,
+            title,
+            content,
+            description,
+            source_url: url,
          }],
          time_consuming: (Date.now() - startTime) / 1000,
        }
@@ -190,7 +191,13 @@ const JinaReader: FC<Props> = ({

  return (
    <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.jinaReaderTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureJinaReader`)}
+        docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`)}
+        docLink={'https://jina.ai/reader'}
+      />
      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
        <UrlInput onRun={handleRun} isRunning={isRunning} />
        <OptionsWrap
--- a/web/app/components/datasets/create/website/no-data.tsx
+++ b/web/app/components/datasets/create/website/no-data.tsx
@@ -5,8 +5,8 @@ import { useTranslation } from 'react-i18next'
 import s from './index.module.css'
 import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
 import Button from '@/app/components/base/button'
-import { DataSourceProvider } from '@/models/common'
 import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import { DataSourceProvider } from '@/models/common'

 const I18N_PREFIX = 'datasetCreation.stepOne.website'

@@ -21,7 +21,11 @@ const NoData: FC<Props> = ({
 }) => {
  const { t } = useTranslation()

-  const providerConfig = {
+  const providerConfig: Record<DataSourceProvider, {
+    emoji: React.ReactNode
+    title: string
+    description: string
+  } | null> = {
    [DataSourceProvider.jinaReader]: ENABLE_WEBSITE_JINAREADER ? {
      emoji: <span className={s.jinaLogo} />,
      title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`),
@@ -39,7 +43,7 @@ const NoData: FC<Props> = ({
    } : null,
  }

-  const currentProvider = providerConfig[provider] || providerConfig[DataSourceProvider.jinaReader]
+  const currentProvider = providerConfig[provider] || providerConfig.jinareader

  if (!currentProvider) return null

--- a/web/app/components/datasets/create/website/preview.tsx
+++ b/web/app/components/datasets/create/website/preview.tsx
@@ -32,7 +32,7 @@ const WebsitePreview = ({
        <div className='system-xs-medium truncate text-text-tertiary' title={payload.source_url}>{payload.source_url}</div>
      </div>
      <div className={cn(s.previewContent, 'body-md-regular')}>
-        <div className={cn(s.fileContent)}>{payload.markdown}</div>
+        <div className={cn(s.fileContent)}>{payload.content}</div>
      </div>
    </div>
  )
--- a/web/app/components/datasets/create/website/watercrawl/header.tsx
+++ b/web/app/components/datasets/create/website/watercrawl/header.tsx
@@ -1,43 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
-import Button from '@/app/components/base/button'
-
-const I18N_PREFIX = 'datasetCreation.stepOne.website'
-
-type Props = {
-  onSetting: () => void
-}
-
-const Header: FC<Props> = ({
-  onSetting,
-}) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='flex h-6 items-center justify-between'>
-      <div className='flex items-center'>
-        <div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.watercrawlTitle`)}</div>
-        <div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
-        <Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
-          <RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
-          <span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
-            {t(`${I18N_PREFIX}.configureWatercrawl`)}
-          </span>
-        </Button>
-      </div>
-      <a
-        href='https://docs.watercrawl.dev/'
-        target='_blank'
-        rel='noopener noreferrer'
-        className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
-      >
-        <RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
-        <span>{t(`${I18N_PREFIX}.watercrawlDoc`)}</span>
-      </a>
-    </div>
-  )
-}
-export default React.memo(Header)
--- a/web/app/components/datasets/create/website/watercrawl/index.tsx
+++ b/web/app/components/datasets/create/website/watercrawl/index.tsx
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
 import CrawledResult from '../base/crawled-result'
 import Crawling from '../base/crawling'
 import ErrorMessage from '../base/error-message'
-import Header from './header'
 import Options from './options'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
+import Header from '../base/header'

 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -177,7 +177,13 @@ const WaterCrawl: FC<Props> = ({

  return (
    <div>
-      <Header onSetting={handleSetting} />
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.watercrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureWatercrawl`)}
+        docTitle={t(`${I18N_PREFIX}.watercrawlDoc`)}
+        docLink={'https://docs.watercrawl.dev/'}
+      />
      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
        <UrlInput onRun={handleRun} isRunning={isRunning} />
        <OptionsWrap