feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/app/components/header/account-setting/data-source-page-new/hooks/index.ts

@@ -0,0 +1,2 @@
+export * from './use-marketplace-all-plugins'
+export * from './use-data-source-auth-update'

web/app/components/header/account-setting/data-source-page-new/hooks/use-data-source-auth-update.ts

@@ -0,0 +1,30 @@
+import { useCallback } from 'react'
+import { useInvalidDataSourceAuth, useInvalidDataSourceListAuth } from '@/service/use-datasource'
+import { useInvalidDefaultDataSourceListAuth } from '@/service/use-datasource'
+import { useInvalidDataSourceList } from '@/service/use-pipeline'
+
+export const useDataSourceAuthUpdate = ({
+  pluginId,
+  provider,
+}: {
+  pluginId: string
+  provider: string
+}) => {
+  const invalidateDataSourceListAuth = useInvalidDataSourceListAuth()
+  const invalidDefaultDataSourceListAuth = useInvalidDefaultDataSourceListAuth()
+  const invalidateDataSourceList = useInvalidDataSourceList()
+  const invalidateDataSourceAuth = useInvalidDataSourceAuth({
+    pluginId,
+    provider,
+  })
+  const handleAuthUpdate = useCallback(() => {
+    invalidateDataSourceListAuth()
+    invalidDefaultDataSourceListAuth()
+    invalidateDataSourceList()
+    invalidateDataSourceAuth()
+  }, [invalidateDataSourceListAuth, invalidateDataSourceList, invalidateDataSourceAuth, invalidDefaultDataSourceListAuth])
+
+  return {
+    handleAuthUpdate,
+  }
+}

web/app/components/header/account-setting/data-source-page-new/hooks/use-marketplace-all-plugins.ts

@@ -0,0 +1,80 @@
+import {
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from 'react'
+import {
+  useMarketplacePlugins,
+} from '@/app/components/plugins/marketplace/hooks'
+import type { Plugin } from '@/app/components/plugins/types'
+import { PluginType } from '@/app/components/plugins/types'
+import { getMarketplacePluginsByCollectionId } from '@/app/components/plugins/marketplace/utils'
+
+export const useMarketplaceAllPlugins = (providers: any[], searchText: string) => {
+  const exclude = useMemo(() => {
+    return providers.map(provider => provider.plugin_id)
+  }, [providers])
+  const [collectionPlugins, setCollectionPlugins] = useState<Plugin[]>([])
+
+  const {
+    plugins,
+    queryPlugins,
+    queryPluginsWithDebounced,
+    isLoading,
+  } = useMarketplacePlugins()
+
+  const getCollectionPlugins = useCallback(async () => {
+    const collectionPlugins = await getMarketplacePluginsByCollectionId('__model-settings-pinned-models')
+
+    setCollectionPlugins(collectionPlugins)
+  }, [])
+
+  useEffect(() => {
+    getCollectionPlugins()
+  }, [getCollectionPlugins])
+
+  useEffect(() => {
+    if (searchText) {
+      queryPluginsWithDebounced({
+        query: searchText,
+        category: PluginType.datasource,
+        exclude,
+        type: 'plugin',
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
+      })
+    }
+    else {
+      queryPlugins({
+        query: '',
+        category: PluginType.datasource,
+        type: 'plugin',
+        pageSize: 1000,
+        exclude,
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
+      })
+    }
+  }, [queryPlugins, queryPluginsWithDebounced, searchText, exclude])
+
+  const allPlugins = useMemo(() => {
+    const allPlugins = collectionPlugins.filter(plugin => !exclude.includes(plugin.plugin_id))
+
+    if (plugins?.length) {
+      for (let i = 0; i < plugins.length; i++) {
+        const plugin = plugins[i]
+
+        if (plugin.type !== 'bundle' && !allPlugins.find(p => p.plugin_id === plugin.plugin_id))
+          allPlugins.push(plugin)
+      }
+    }
+
+    return allPlugins
+  }, [plugins, collectionPlugins, exclude])
+
+  return {
+    plugins: allPlugins,
+    isLoading,
+  }
+}