Initial commit

api/controllers/console/__init__.py

@@ -0,0 +1,20 @@
+from flask import Blueprint
+
+from libs.external_api import ExternalApi
+
+bp = Blueprint('console', __name__, url_prefix='/console/api')
+api = ExternalApi(bp)
+
+# Import app controllers
+from .app import app, site, explore, completion, model_config, statistic, conversation, message
+
+# Import auth controllers
+from .auth import login, oauth
+
+# Import datasets controllers
+from .datasets import datasets, datasets_document, datasets_segments, file, hit_testing
+
+# Import other controllers
+from . import setup, version, apikey
+
+from .workspace import workspace, members, providers, account

api/controllers/console/apikey.py

@@ -0,0 +1,175 @@
+from flask_login import login_required, current_user
+import flask_restful
+from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
+from models.model import App, ApiToken
+from models.dataset import Dataset
+
+from . import api
+from .setup import setup_required
+from .wraps import account_initialization_required
+from libs.helper import TimestampField
+
+api_key_fields = {
+    'id': fields.String,
+    'type': fields.String,
+    'token': fields.String,
+    'last_used_at': TimestampField,
+    'created_at': TimestampField
+}
+
+api_key_list = {
+    'data': fields.List(fields.Nested(api_key_fields), attribute="items")
+}
+
+
+def _get_resource(resource_id, tenant_id, resource_model):
+    resource = resource_model.query.filter_by(
+        id=resource_id, tenant_id=tenant_id
+    ).first()
+
+    if resource is None:
+        flask_restful.abort(
+            404, message=f"{resource_model.__name__} not found.")
+
+    return resource
+
+
+class BaseApiKeyListResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type = None
+    resource_model = None
+    resource_id_field = None
+    token_prefix = None
+    max_keys = 10
+
+    @marshal_with(api_key_list)
+    def get(self, resource_id):
+        resource_id = str(resource_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+        keys = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id). \
+            all()
+        return {"items": keys}
+
+    @marshal_with(api_key_fields)
+    def post(self, resource_id):
+        resource_id = str(resource_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        current_key_count = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id). \
+            count()
+
+        if current_key_count >= self.max_keys:
+            flask_restful.abort(
+                400,
+                message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
+                code='max_keys_exceeded'
+            )
+
+        key = ApiToken.generate_api_key(self.token_prefix, 24)
+        api_token = ApiToken()
+        setattr(api_token, self.resource_id_field, resource_id)
+        api_token.token = key
+        api_token.type = self.resource_type
+        db.session.add(api_token)
+        db.session.commit()
+        return api_token, 201
+
+
+class BaseApiKeyResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type = None
+    resource_model = None
+    resource_id_field = None
+
+    def delete(self, resource_id, api_key_id):
+        resource_id = str(resource_id)
+        api_key_id = str(api_key_id)
+        _get_resource(resource_id, current_user.current_tenant_id,
+                      self.resource_model)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        key = db.session.query(ApiToken). \
+            filter(getattr(ApiToken, self.resource_id_field) == resource_id, ApiToken.type == self.resource_type, ApiToken.id == api_key_id). \
+            first()
+
+        if key is None:
+            flask_restful.abort(404, message='API key not found')
+
+        db.session.query(ApiToken).filter(ApiToken.id == api_key_id).delete()
+        db.session.commit()
+
+        return {'result': 'success'}, 204
+
+
+class AppApiKeyListResource(BaseApiKeyListResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'app'
+    resource_model = App
+    resource_id_field = 'app_id'
+    token_prefix = 'app-'
+
+
+class AppApiKeyResource(BaseApiKeyResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'app'
+    resource_model = App
+    resource_id_field = 'app_id'
+
+
+class DatasetApiKeyListResource(BaseApiKeyListResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+
+    resource_type = 'dataset'
+    resource_model = Dataset
+    resource_id_field = 'dataset_id'
+    token_prefix = 'ds-'
+
+
+class DatasetApiKeyResource(BaseApiKeyResource):
+
+    def after_request(self, resp):
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        resp.headers['Access-Control-Allow-Credentials'] = 'true'
+        return resp
+    resource_type = 'dataset'
+    resource_model = Dataset
+    resource_id_field = 'dataset_id'
+
+
+api.add_resource(AppApiKeyListResource, '/apps/<uuid:resource_id>/api-keys')
+api.add_resource(AppApiKeyResource,
+                 '/apps/<uuid:resource_id>/api-keys/<uuid:api_key_id>')
+api.add_resource(DatasetApiKeyListResource,
+                 '/datasets/<uuid:resource_id>/api-keys')
+api.add_resource(DatasetApiKeyResource,
+                 '/datasets/<uuid:resource_id>/api-keys/<uuid:api_key_id>')

api/controllers/console/app/__init__.py

@@ -0,0 +1,22 @@
+from flask_login import current_user
+from werkzeug.exceptions import NotFound
+
+from controllers.console.app.error import AppUnavailableError
+from extensions.ext_database import db
+from models.model import App
+
+
+def _get_app(app_id, mode=None):
+    app = db.session.query(App).filter(
+        App.id == app_id,
+        App.tenant_id == current_user.current_tenant_id,
+        App.status == 'normal'
+    ).first()
+
+    if not app:
+        raise NotFound("App not found")
+
+    if mode and app.mode != mode:
+        raise AppUnavailableError()
+
+    return app

api/controllers/console/app/app.py

@@ -0,0 +1,518 @@
+# -*- coding:utf-8 -*-
+import json
+from datetime import datetime
+
+import flask
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with, abort, inputs
+from werkzeug.exceptions import Unauthorized, Forbidden
+
+from constants.model_template import model_templates, demo_model_templates
+from controllers.console import api
+from controllers.console.app.error import AppNotFoundError, ProviderNotInitializeError, ProviderQuotaExceededError, \
+    CompletionRequestError, ProviderModelCurrentlyNotSupportError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.generator.llm_generator import LLMGenerator
+from core.llm.error import ProviderTokenNotInitError, QuotaExceededError, LLMBadRequestError, LLMAPIConnectionError, \
+    LLMAPIUnavailableError, LLMRateLimitError, LLMAuthorizationError, ModelCurrentlyNotSupportError
+from events.app_event import app_was_created, app_was_deleted
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.model import App, AppModelConfig, Site, InstalledApp
+from services.account_service import TenantService
+from services.app_model_config_service import AppModelConfigService
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
+    'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
+    'more_like_this': fields.Raw(attribute='more_like_this_dict'),
+    'model': fields.Raw(attribute='model_dict'),
+    'user_input_form': fields.Raw(attribute='user_input_form_list'),
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw(attribute='agent_mode_dict'),
+}
+
+app_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'api_rpm': fields.Integer,
+    'api_rph': fields.Integer,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+    'created_at': TimestampField
+}
+
+
+def _get_app(app_id, tenant_id):
+    app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first()
+    if not app:
+        raise AppNotFoundError
+    return app
+
+
+class AppListApi(Resource):
+    prompt_config_fields = {
+        'prompt_template': fields.String,
+    }
+
+    model_config_partial_fields = {
+        'model': fields.Raw(attribute='model_dict'),
+        'pre_prompt': fields.String,
+    }
+
+    app_partial_fields = {
+        'id': fields.String,
+        'name': fields.String,
+        'mode': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+        'enable_site': fields.Boolean,
+        'enable_api': fields.Boolean,
+        'is_demo': fields.Boolean,
+        'model_config': fields.Nested(model_config_partial_fields, attribute='app_model_config'),
+        'created_at': TimestampField
+    }
+
+    app_pagination_fields = {
+        'page': fields.Integer,
+        'limit': fields.Integer(attribute='per_page'),
+        'total': fields.Integer,
+        'has_more': fields.Boolean(attribute='has_next'),
+        'data': fields.List(fields.Nested(app_partial_fields), attribute='items')
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_pagination_fields)
+    def get(self):
+        """Get app list"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('page', type=inputs.int_range(1, 99999), required=False, default=1, location='args')
+        parser.add_argument('limit', type=inputs.int_range(1, 100), required=False, default=20, location='args')
+        args = parser.parse_args()
+
+        app_models = db.paginate(
+            db.select(App).where(App.tenant_id == current_user.current_tenant_id).order_by(App.created_at.desc()),
+            page=args['page'],
+            per_page=args['limit'],
+            error_out=False)
+
+        return app_models
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self):
+        """Create app"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, location='json')
+        parser.add_argument('mode', type=str, choices=['completion', 'chat'], location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        parser.add_argument('model_config', type=dict, location='json')
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        if args['model_config'] is not None:
+            # validate config
+            model_configuration = AppModelConfigService.validate_configuration(
+                account=current_user,
+                config=args['model_config'],
+                mode=args['mode']
+            )
+
+            app = App(
+                enable_site=True,
+                enable_api=True,
+                is_demo=False,
+                api_rpm=0,
+                api_rph=0,
+                status='normal'
+            )
+
+            app_model_config = AppModelConfig(
+                provider="",
+                model_id="",
+                configs={},
+                opening_statement=model_configuration['opening_statement'],
+                suggested_questions=json.dumps(model_configuration['suggested_questions']),
+                suggested_questions_after_answer=json.dumps(model_configuration['suggested_questions_after_answer']),
+                more_like_this=json.dumps(model_configuration['more_like_this']),
+                model=json.dumps(model_configuration['model']),
+                user_input_form=json.dumps(model_configuration['user_input_form']),
+                pre_prompt=model_configuration['pre_prompt'],
+                agent_mode=json.dumps(model_configuration['agent_mode']),
+            )
+        else:
+            if 'mode' not in args or args['mode'] is None:
+                abort(400, message="mode is required")
+
+            model_config_template = model_templates[args['mode'] + '_default']
+
+            app = App(**model_config_template['app'])
+            app_model_config = AppModelConfig(**model_config_template['model_config'])
+
+        app.name = args['name']
+        app.mode = args['mode']
+        app.icon = args['icon']
+        app.icon_background = args['icon_background']
+        app.tenant_id = current_user.current_tenant_id
+
+        db.session.add(app)
+        db.session.flush()
+
+        app_model_config.app_id = app.id
+        db.session.add(app_model_config)
+        db.session.flush()
+
+        app.app_model_config_id = app_model_config.id
+
+        account = current_user
+
+        site = Site(
+            app_id=app.id,
+            title=app.name,
+            default_language=account.interface_language,
+            customize_token_strategy='not_allow',
+            code=Site.generate_code(16)
+        )
+
+        db.session.add(site)
+        db.session.commit()
+
+        app_was_created.send(app)
+
+        return app, 201
+
+
+class AppTemplateApi(Resource):
+    template_fields = {
+        'name': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+        'description': fields.String,
+        'mode': fields.String,
+        'model_config': fields.Nested(model_config_fields),
+    }
+
+    template_list_fields = {
+        'data': fields.List(fields.Nested(template_fields)),
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(template_list_fields)
+    def get(self):
+        """Get app demo templates"""
+        account = current_user
+        interface_language = account.interface_language
+
+        return {'data': demo_model_templates.get(interface_language)}
+
+
+class AppApi(Resource):
+    site_fields = {
+        'access_token': fields.String(attribute='code'),
+        'code': fields.String,
+        'title': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+        'description': fields.String,
+        'default_language': fields.String,
+        'customize_domain': fields.String,
+        'copyright': fields.String,
+        'privacy_policy': fields.String,
+        'customize_token_strategy': fields.String,
+        'prompt_public': fields.Boolean,
+        'app_base_url': fields.String,
+    }
+
+    app_detail_fields_with_site = {
+        'id': fields.String,
+        'name': fields.String,
+        'mode': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+        'enable_site': fields.Boolean,
+        'enable_api': fields.Boolean,
+        'api_rpm': fields.Integer,
+        'api_rph': fields.Integer,
+        'is_demo': fields.Boolean,
+        'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+        'site': fields.Nested(site_fields),
+        'api_base_url': fields.String,
+        'created_at': TimestampField
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields_with_site)
+    def get(self, app_id):
+        """Get app detail"""
+        app_id = str(app_id)
+        app = _get_app(app_id, current_user.current_tenant_id)
+
+        return app
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, app_id):
+        """Delete app"""
+        app_id = str(app_id)
+        app = _get_app(app_id, current_user.current_tenant_id)
+
+        db.session.delete(app)
+        db.session.commit()
+
+        # todo delete related data??
+        # model_config, site, api_token, conversation, message, message_feedback, message_annotation
+
+        app_was_deleted.send(app)
+
+        return {'result': 'success'}, 204
+
+
+class AppNameApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        app = db.get_or_404(App, str(app_id))
+        if app.tenant_id != flask.session.get('tenant_id'):
+            raise Unauthorized()
+
+        app.name = args.get('name')
+        app.updated_at = datetime.utcnow()
+        db.session.commit()
+        return app
+
+
+class AppIconApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
+
+        app = db.get_or_404(App, str(app_id))
+        if app.tenant_id != flask.session.get('tenant_id'):
+            raise Unauthorized()
+
+        app.icon = args.get('icon')
+        app.icon_background = args.get('icon_background')
+        app.updated_at = datetime.utcnow()
+        db.session.commit()
+
+        return app
+
+
+class AppSiteStatus(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('enable_site', type=bool, required=True, location='json')
+        args = parser.parse_args()
+        app_id = str(app_id)
+        app = db.session.query(App).filter(App.id == app_id, App.tenant_id == current_user.current_tenant_id).first()
+        if not app:
+            raise AppNotFoundError
+
+        if args.get('enable_site') == app.enable_site:
+            return app
+
+        app.enable_site = args.get('enable_site')
+        app.updated_at = datetime.utcnow()
+        db.session.commit()
+        return app
+
+
+class AppApiStatus(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('enable_api', type=bool, required=True, location='json')
+        args = parser.parse_args()
+
+        app_id = str(app_id)
+        app = _get_app(app_id, current_user.current_tenant_id)
+
+        if args.get('enable_api') == app.enable_api:
+            return app
+
+        app.enable_api = args.get('enable_api')
+        app.updated_at = datetime.utcnow()
+        db.session.commit()
+        return app
+
+
+class AppRateLimit(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('api_rpm', type=inputs.natural, required=False, location='json')
+        parser.add_argument('api_rph', type=inputs.natural, required=False, location='json')
+        args = parser.parse_args()
+
+        app_id = str(app_id)
+        app = _get_app(app_id, current_user.current_tenant_id)
+
+        if args.get('api_rpm'):
+            app.api_rpm = args.get('api_rpm')
+        if args.get('api_rph'):
+            app.api_rph = args.get('api_rph')
+        app.updated_at = datetime.utcnow()
+        db.session.commit()
+        return app
+
+
+class AppCopy(Resource):
+    @staticmethod
+    def create_app_copy(app):
+        copy_app = App(
+            name=app.name + ' copy',
+            icon=app.icon,
+            icon_background=app.icon_background,
+            tenant_id=app.tenant_id,
+            mode=app.mode,
+            app_model_config_id=app.app_model_config_id,
+            enable_site=app.enable_site,
+            enable_api=app.enable_api,
+            api_rpm=app.api_rpm,
+            api_rph=app.api_rph
+        )
+        return copy_app
+
+    @staticmethod
+    def create_app_model_config_copy(app_config, copy_app_id):
+        copy_app_model_config = AppModelConfig(
+            app_id=copy_app_id,
+            provider=app_config.provider,
+            model_id=app_config.model_id,
+            configs=app_config.configs,
+            opening_statement=app_config.opening_statement,
+            suggested_questions=app_config.suggested_questions,
+            suggested_questions_after_answer=app_config.suggested_questions_after_answer,
+            more_like_this=app_config.more_like_this,
+            model=app_config.model,
+            user_input_form=app_config.user_input_form,
+            pre_prompt=app_config.pre_prompt,
+            agent_mode=app_config.agent_mode
+        )
+        return copy_app_model_config
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields)
+    def post(self, app_id):
+        app_id = str(app_id)
+        app = _get_app(app_id, current_user.current_tenant_id)
+
+        copy_app = self.create_app_copy(app)
+        db.session.add(copy_app)
+
+        app_config = db.session.query(AppModelConfig). \
+            filter(AppModelConfig.app_id == app_id). \
+            one_or_none()
+
+        if app_config:
+            copy_app_model_config = self.create_app_model_config_copy(app_config, copy_app.id)
+            db.session.add(copy_app_model_config)
+            db.session.commit()
+            copy_app.app_model_config_id = copy_app_model_config.id
+        db.session.commit()
+
+        return copy_app, 201
+
+
+class AppExport(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        # todo
+        pass
+
+
+class IntroductionGenerateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('prompt_template', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        account = current_user
+
+        try:
+            answer = LLMGenerator.generate_introduction(
+                account.current_tenant_id,
+                args['prompt_template']
+            )
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+
+        return {'introduction': answer}
+
+
+api.add_resource(AppListApi, '/apps')
+api.add_resource(AppTemplateApi, '/app-templates')
+api.add_resource(AppApi, '/apps/<uuid:app_id>')
+api.add_resource(AppCopy, '/apps/<uuid:app_id>/copy')
+api.add_resource(AppNameApi, '/apps/<uuid:app_id>/name')
+api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable')
+api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable')
+api.add_resource(AppRateLimit, '/apps/<uuid:app_id>/rate-limit')
+api.add_resource(IntroductionGenerateApi, '/introduction-generate')

api/controllers/console/app/completion.py

@@ -0,0 +1,206 @@
+# -*- coding:utf-8 -*-
+import json
+import logging
+from typing import Generator, Union
+
+import flask_login
+from flask import Response, stream_with_context
+from flask_login import login_required
+from werkzeug.exceptions import InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.app.error import ConversationCompletedError, AppUnavailableError, \
+    ProviderNotInitializeError, CompletionRequestError, ProviderQuotaExceededError, \
+    ProviderModelCurrentlyNotSupportError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.conversation_message_task import PubHandler
+from core.llm.error import LLMBadRequestError, LLMAPIUnavailableError, LLMAuthorizationError, LLMAPIConnectionError, \
+    LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from libs.helper import uuid_value
+from flask_restful import Resource, reqparse
+
+from services.completion_service import CompletionService
+
+
+# define completion message api for user
+class CompletionMessageApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app_model = _get_app(app_id, 'completion')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, location='json')
+        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('model_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        account = flask_login.current_user
+
+        try:
+            response = CompletionService.completion(
+                app_model=app_model,
+                user=account,
+                args=args,
+                from_source='console',
+                streaming=True,
+                is_model_config_override=True
+            )
+
+            return compact_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+class CompletionMessageStopApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id, task_id):
+        app_id = str(app_id)
+
+        # get app info
+        _get_app(app_id, 'completion')
+
+        account = flask_login.current_user
+
+        PubHandler.stop(account, task_id)
+
+        return {'result': 'success'}, 200
+
+
+class ChatMessageApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app_model = _get_app(app_id, 'chat')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('inputs', type=dict, required=True, location='json')
+        parser.add_argument('query', type=str, required=True, location='json')
+        parser.add_argument('model_config', type=dict, required=True, location='json')
+        parser.add_argument('conversation_id', type=uuid_value, location='json')
+        args = parser.parse_args()
+
+        account = flask_login.current_user
+
+        try:
+            response = CompletionService.completion(
+                app_model=app_model,
+                user=account,
+                args=args,
+                from_source='console',
+                streaming=True,
+                is_model_config_override=True
+            )
+
+            return compact_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logging.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+def compact_response(response: Union[dict | Generator]) -> Response:
+    if isinstance(response, dict):
+        return Response(response=json.dumps(response), status=200, mimetype='application/json')
+    else:
+        def generate() -> Generator:
+            try:
+                for chunk in response:
+                    yield chunk
+            except services.errors.conversation.ConversationNotExistsError:
+                yield "data: " + json.dumps(api.handle_error(NotFound("Conversation Not Exists.")).get_json()) + "\n\n"
+            except services.errors.conversation.ConversationCompletedError:
+                yield "data: " + json.dumps(api.handle_error(ConversationCompletedError()).get_json()) + "\n\n"
+            except services.errors.app_model_config.AppModelConfigBrokenError:
+                logging.exception("App model config broken.")
+                yield "data: " + json.dumps(api.handle_error(AppUnavailableError()).get_json()) + "\n\n"
+            except ProviderTokenNotInitError:
+                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError()).get_json()) + "\n\n"
+            except QuotaExceededError:
+                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
+            except ModelCurrentlyNotSupportError:
+                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
+            except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                    LLMRateLimitError, LLMAuthorizationError) as e:
+                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(str(e))).get_json()) + "\n\n"
+            except ValueError as e:
+                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
+            except Exception:
+                logging.exception("internal server error.")
+                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
+
+        return Response(stream_with_context(generate()), status=200,
+                        mimetype='text/event-stream')
+
+
+class ChatMessageStopApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id, task_id):
+        app_id = str(app_id)
+
+        # get app info
+        _get_app(app_id, 'chat')
+
+        account = flask_login.current_user
+
+        PubHandler.stop(account, task_id)
+
+        return {'result': 'success'}, 200
+
+
+api.add_resource(CompletionMessageApi, '/apps/<uuid:app_id>/completion-messages')
+api.add_resource(CompletionMessageStopApi, '/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop')
+api.add_resource(ChatMessageApi, '/apps/<uuid:app_id>/chat-messages')
+api.add_resource(ChatMessageStopApi, '/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop')

api/controllers/console/app/conversation.py

@@ -0,0 +1,384 @@
+from datetime import datetime
+
+import pytz
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with
+from flask_restful.inputs import int_range
+from sqlalchemy import or_, func
+from sqlalchemy.orm import joinedload
+from werkzeug.exceptions import NotFound
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import TimestampField, datetime_string, uuid_value
+from extensions.ext_database import db
+from models.model import Message, MessageAnnotation, Conversation
+
+account_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'email': fields.String
+}
+
+feedback_fields = {
+    'rating': fields.String,
+    'content': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account': fields.Nested(account_fields, allow_null=True),
+}
+
+annotation_fields = {
+    'content': fields.String,
+    'account': fields.Nested(account_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+message_detail_fields = {
+    'id': fields.String,
+    'conversation_id': fields.String,
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'message': fields.Raw,
+    'message_tokens': fields.Integer,
+    'answer': fields.String,
+    'answer_tokens': fields.Integer,
+    'provider_response_latency': fields.Integer,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'feedbacks': fields.List(fields.Nested(feedback_fields)),
+    'annotation': fields.Nested(annotation_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+feedback_stat_fields = {
+    'like': fields.Integer,
+    'dislike': fields.Integer
+}
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw,
+    'model': fields.Raw,
+    'user_input_form': fields.Raw,
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw,
+}
+
+
+class CompletionConversationApi(Resource):
+    class MessageTextField(fields.Raw):
+        def format(self, value):
+            return value[0]['text'] if value else ''
+
+    simple_configs_fields = {
+        'prompt_template': fields.String,
+    }
+
+    simple_model_config_fields = {
+        'model': fields.Raw(attribute='model_dict'),
+        'pre_prompt': fields.String,
+    }
+
+    simple_message_detail_fields = {
+        'inputs': fields.Raw,
+        'query': fields.String,
+        'message': MessageTextField,
+        'answer': fields.String,
+    }
+
+    conversation_fields = {
+        'id': fields.String,
+        'status': fields.String,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account_id': fields.String,
+        'read_at': TimestampField,
+        'created_at': TimestampField,
+        'annotation': fields.Nested(annotation_fields, allow_null=True),
+        'model_config': fields.Nested(simple_model_config_fields),
+        'user_feedback_stats': fields.Nested(feedback_stat_fields),
+        'admin_feedback_stats': fields.Nested(feedback_stat_fields),
+        'message': fields.Nested(simple_message_detail_fields, attribute='first_message')
+    }
+
+    conversation_pagination_fields = {
+        'page': fields.Integer,
+        'limit': fields.Integer(attribute='per_page'),
+        'total': fields.Integer,
+        'has_more': fields.Boolean(attribute='has_next'),
+        'data': fields.List(fields.Nested(conversation_fields), attribute='items')
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(conversation_pagination_fields)
+    def get(self, app_id):
+        app_id = str(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('keyword', type=str, location='args')
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('annotation_status', type=str,
+                            choices=['annotated', 'not_annotated', 'all'], default='all', location='args')
+        parser.add_argument('page', type=int_range(1, 99999), default=1, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), default=20, location='args')
+        args = parser.parse_args()
+
+        # get app info
+        app = _get_app(app_id, 'completion')
+
+        query = db.select(Conversation).where(Conversation.app_id == app.id, Conversation.mode == 'completion')
+
+        if args['keyword']:
+            query = query.join(
+                Message, Message.conversation_id == Conversation.id
+            ).filter(
+                or_(
+                    Message.query.ilike('%{}%'.format(args['keyword'])),
+                    Message.answer.ilike('%{}%'.format(args['keyword']))
+                )
+            )
+
+        account = current_user
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            query = query.where(Conversation.created_at >= start_datetime_utc)
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            query = query.where(Conversation.created_at < end_datetime_utc)
+
+        if args['annotation_status'] == "annotated":
+            query = query.options(joinedload(Conversation.message_annotations)).join(
+                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+            )
+        elif args['annotation_status'] == "not_annotated":
+            query = query.outerjoin(
+                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+            ).group_by(Conversation.id).having(func.count(MessageAnnotation.id) == 0)
+
+        query = query.order_by(Conversation.created_at.desc())
+
+        conversations = db.paginate(
+            query,
+            page=args['page'],
+            per_page=args['limit'],
+            error_out=False
+        )
+
+        return conversations
+
+
+class CompletionConversationDetailApi(Resource):
+    conversation_detail_fields = {
+        'id': fields.String,
+        'status': fields.String,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account_id': fields.String,
+        'created_at': TimestampField,
+        'model_config': fields.Nested(model_config_fields),
+        'message': fields.Nested(message_detail_fields, attribute='first_message'),
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(conversation_detail_fields)
+    def get(self, app_id, conversation_id):
+        app_id = str(app_id)
+        conversation_id = str(conversation_id)
+
+        return _get_conversation(app_id, conversation_id, 'completion')
+
+
+class ChatConversationApi(Resource):
+    simple_configs_fields = {
+        'prompt_template': fields.String,
+    }
+
+    simple_model_config_fields = {
+        'model': fields.Raw(attribute='model_dict'),
+        'pre_prompt': fields.String,
+    }
+
+    conversation_fields = {
+        'id': fields.String,
+        'status': fields.String,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account_id': fields.String,
+        'summary': fields.String(attribute='summary_or_query'),
+        'read_at': TimestampField,
+        'created_at': TimestampField,
+        'annotated': fields.Boolean,
+        'model_config': fields.Nested(simple_model_config_fields),
+        'message_count': fields.Integer,
+        'user_feedback_stats': fields.Nested(feedback_stat_fields),
+        'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+    }
+
+    conversation_pagination_fields = {
+        'page': fields.Integer,
+        'limit': fields.Integer(attribute='per_page'),
+        'total': fields.Integer,
+        'has_more': fields.Boolean(attribute='has_next'),
+        'data': fields.List(fields.Nested(conversation_fields), attribute='items')
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(conversation_pagination_fields)
+    def get(self, app_id):
+        app_id = str(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('keyword', type=str, location='args')
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('annotation_status', type=str,
+                            choices=['annotated', 'not_annotated', 'all'], default='all', location='args')
+        parser.add_argument('message_count_gte', type=int_range(1, 99999), required=False, location='args')
+        parser.add_argument('page', type=int_range(1, 99999), required=False, default=1, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
+        args = parser.parse_args()
+
+        # get app info
+        app = _get_app(app_id, 'chat')
+
+        query = db.select(Conversation).where(Conversation.app_id == app.id, Conversation.mode == 'chat')
+
+        if args['keyword']:
+            query = query.join(
+                Message, Message.conversation_id == Conversation.id
+            ).filter(
+                or_(
+                    Message.query.ilike('%{}%'.format(args['keyword'])),
+                    Message.answer.ilike('%{}%'.format(args['keyword'])),
+                    Conversation.name.ilike('%{}%'.format(args['keyword'])),
+                    Conversation.introduction.ilike('%{}%'.format(args['keyword'])),
+                ),
+
+            )
+
+        account = current_user
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            query = query.where(Conversation.created_at >= start_datetime_utc)
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            query = query.where(Conversation.created_at < end_datetime_utc)
+
+        if args['annotation_status'] == "annotated":
+            query = query.options(joinedload(Conversation.message_annotations)).join(
+                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+            )
+        elif args['annotation_status'] == "not_annotated":
+            query = query.outerjoin(
+                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+            ).group_by(Conversation.id).having(func.count(MessageAnnotation.id) == 0)
+
+        if args['message_count_gte'] and args['message_count_gte'] >= 1:
+            query = (
+                query.options(joinedload(Conversation.messages))
+                .join(Message, Message.conversation_id == Conversation.id)
+                .group_by(Conversation.id)
+                .having(func.count(Message.id) >= args['message_count_gte'])
+            )
+
+        query = query.order_by(Conversation.created_at.desc())
+
+        conversations = db.paginate(
+            query,
+            page=args['page'],
+            per_page=args['limit'],
+            error_out=False
+        )
+
+        return conversations
+
+
+class ChatConversationDetailApi(Resource):
+    conversation_detail_fields = {
+        'id': fields.String,
+        'status': fields.String,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account_id': fields.String,
+        'created_at': TimestampField,
+        'annotated': fields.Boolean,
+        'model_config': fields.Nested(model_config_fields),
+        'message_count': fields.Integer,
+        'user_feedback_stats': fields.Nested(feedback_stat_fields),
+        'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(conversation_detail_fields)
+    def get(self, app_id, conversation_id):
+        app_id = str(app_id)
+        conversation_id = str(conversation_id)
+
+        return _get_conversation(app_id, conversation_id, 'chat')
+
+
+
+
+api.add_resource(CompletionConversationApi, '/apps/<uuid:app_id>/completion-conversations')
+api.add_resource(CompletionConversationDetailApi, '/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>')
+api.add_resource(ChatConversationApi, '/apps/<uuid:app_id>/chat-conversations')
+api.add_resource(ChatConversationDetailApi, '/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>')
+
+
+def _get_conversation(app_id, conversation_id, mode):
+    # get app info
+    app = _get_app(app_id, mode)
+
+    conversation = db.session.query(Conversation) \
+        .filter(Conversation.id == conversation_id, Conversation.app_id == app.id).first()
+
+    if not conversation:
+        raise NotFound("Conversation Not Exists.")
+
+    if not conversation.read_at:
+        conversation.read_at = datetime.utcnow()
+        conversation.read_account_id = current_user.id
+        db.session.commit()
+
+    return conversation

api/controllers/console/app/error.py

@@ -0,0 +1,49 @@
+from libs.exception import BaseHTTPException
+
+
+class AppNotFoundError(BaseHTTPException):
+    error_code = 'app_not_found'
+    description = "App not found."
+    code = 404
+
+
+class ProviderNotInitializeError(BaseHTTPException):
+    error_code = 'provider_not_initialize'
+    description = "Provider Token not initialize."
+    code = 400
+
+
+class ProviderQuotaExceededError(BaseHTTPException):
+    error_code = 'provider_quota_exceeded'
+    description = "Provider quota exceeded."
+    code = 400
+
+
+class ProviderModelCurrentlyNotSupportError(BaseHTTPException):
+    error_code = 'model_currently_not_support'
+    description = "GPT-4 currently not support."
+    code = 400
+
+
+class ConversationCompletedError(BaseHTTPException):
+    error_code = 'conversation_completed'
+    description = "Conversation was completed."
+    code = 400
+
+
+class AppUnavailableError(BaseHTTPException):
+    error_code = 'app_unavailable'
+    description = "App unavailable."
+    code = 400
+
+
+class CompletionRequestError(BaseHTTPException):
+    error_code = 'completion_request_error'
+    description = "Completion request failed."
+    code = 400
+
+
+class AppMoreLikeThisDisabledError(BaseHTTPException):
+    error_code = 'app_more_like_this_disabled'
+    description = "More like this disabled."
+    code = 403

api/controllers/console/app/explore.py

@@ -0,0 +1,209 @@
+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with, abort, inputs
+from sqlalchemy import and_
+
+from controllers.console import api
+from extensions.ext_database import db
+from models.model import Tenant, App, InstalledApp, RecommendedApp
+from services.account_service import TenantService
+
+app_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String
+}
+
+installed_app_fields = {
+    'id': fields.String,
+    'app': fields.Nested(app_fields, attribute='app'),
+    'app_owner_tenant_id': fields.String,
+    'is_pinned': fields.Boolean,
+    'last_used_at': fields.DateTime,
+    'editable': fields.Boolean
+}
+
+installed_app_list_fields = {
+    'installed_apps': fields.List(fields.Nested(installed_app_fields))
+}
+
+recommended_app_fields = {
+    'app': fields.Nested(app_fields, attribute='app'),
+    'app_id': fields.String,
+    'description': fields.String(attribute='description'),
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'category': fields.String,
+    'position': fields.Integer,
+    'is_listed': fields.Boolean,
+    'install_count': fields.Integer,
+    'installed': fields.Boolean,
+    'editable': fields.Boolean
+}
+
+recommended_app_list_fields = {
+    'recommended_apps': fields.List(fields.Nested(recommended_app_fields)),
+    'categories': fields.List(fields.String)
+}
+
+
+class InstalledAppsListResource(Resource):
+    @login_required
+    @marshal_with(installed_app_list_fields)
+    def get(self):
+        current_tenant_id = Tenant.query.first().id
+        installed_apps = db.session.query(InstalledApp).filter(
+            InstalledApp.tenant_id == current_tenant_id
+        ).all()
+
+        current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
+        installed_apps = [
+            {
+                **installed_app,
+                "editable": current_user.role in ["owner", "admin"],
+            }
+            for installed_app in installed_apps
+        ]
+        installed_apps.sort(key=lambda app: (-app.is_pinned, app.last_used_at))
+
+        return {'installed_apps': installed_apps}
+
+    @login_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('app_id', type=str, required=True, help='Invalid app_id')
+        args = parser.parse_args()
+
+        current_tenant_id = Tenant.query.first().id
+        app = App.query.get(args['app_id'])
+        if app is None:
+            abort(404, message='App not found')
+        recommended_app = RecommendedApp.query.filter(RecommendedApp.app_id == args['app_id']).first()
+        if recommended_app is None:
+            abort(404, message='App not found')
+        if not app.is_public:
+            abort(403, message="You can't install a non-public app")
+
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.app_id == args['app_id'],
+            InstalledApp.tenant_id == current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            # todo: position
+            recommended_app.install_count += 1
+
+            new_installed_app = InstalledApp(
+                app_id=args['app_id'],
+                tenant_id=current_tenant_id,
+                is_pinned=False,
+                last_used_at=datetime.utcnow()
+            )
+            db.session.add(new_installed_app)
+            db.session.commit()
+
+        return {'message': 'App installed successfully'}
+
+
+class InstalledAppResource(Resource):
+
+    @login_required
+    def delete(self, installed_app_id):
+
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.id == str(installed_app_id),
+            InstalledApp.tenant_id == current_user.current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            abort(404, message='App not found')
+
+        if installed_app.app_owner_tenant_id == current_user.current_tenant_id:
+            abort(400, message="You can't uninstall an app owned by the current tenant")
+
+        db.session.delete(installed_app)
+        db.session.commit()
+
+        return {'result': 'success', 'message': 'App uninstalled successfully'}
+
+    @login_required
+    def patch(self, installed_app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('is_pinned', type=inputs.boolean)
+        args = parser.parse_args()
+
+        current_tenant_id = Tenant.query.first().id
+        installed_app = InstalledApp.query.filter(and_(
+            InstalledApp.id == str(installed_app_id),
+            InstalledApp.tenant_id == current_tenant_id
+        )).first()
+
+        if installed_app is None:
+            abort(404, message='Installed app not found')
+
+        commit_args = False
+        if 'is_pinned' in args:
+            installed_app.is_pinned = args['is_pinned']
+            commit_args = True
+
+        if commit_args:
+            db.session.commit()
+
+        return {'result': 'success', 'message': 'App info updated successfully'}
+
+
+class RecommendedAppsResource(Resource):
+    @login_required
+    @marshal_with(recommended_app_list_fields)
+    def get(self):
+        recommended_apps = db.session.query(RecommendedApp).filter(
+            RecommendedApp.is_listed == True
+        ).all()
+
+        categories = set()
+        current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
+        recommended_apps_result = []
+        for recommended_app in recommended_apps:
+            installed = db.session.query(InstalledApp).filter(
+                and_(
+                    InstalledApp.app_id == recommended_app.app_id,
+                    InstalledApp.tenant_id == current_user.current_tenant_id
+                )
+            ).first() is not None
+
+            language_prefix = current_user.interface_language.split('-')[0]
+            desc = None
+            if recommended_app.description:
+                if language_prefix in recommended_app.description:
+                    desc = recommended_app.description[language_prefix]
+                elif 'en' in recommended_app.description:
+                    desc = recommended_app.description['en']
+
+            recommended_app_result = {
+                'id': recommended_app.id,
+                'app': recommended_app.app,
+                'app_id': recommended_app.app_id,
+                'description': desc,
+                'copyright': recommended_app.copyright,
+                'privacy_policy': recommended_app.privacy_policy,
+                'category': recommended_app.category,
+                'position': recommended_app.position,
+                'is_listed': recommended_app.is_listed,
+                'install_count': recommended_app.install_count,
+                'installed': installed,
+                'editable': current_user.role in ['owner', 'admin'],
+            }
+            recommended_apps_result.append(recommended_app_result)
+
+            categories.add(recommended_app.category)  # add category to categories
+
+        return {'recommended_apps': recommended_apps_result, 'categories': list(categories)}
+
+
+api.add_resource(InstalledAppsListResource, '/installed-apps')
+api.add_resource(InstalledAppResource, '/installed-apps/<uuid:installed_app_id>')
+api.add_resource(RecommendedAppsResource, '/explore/apps')

api/controllers/console/app/message.py

@@ -0,0 +1,361 @@
+import json
+import logging
+from typing import Union, Generator
+
+from flask import Response, stream_with_context
+from flask_login import current_user, login_required
+from flask_restful import Resource, reqparse, marshal_with, fields
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.app.error import CompletionRequestError, ProviderNotInitializeError, \
+    AppMoreLikeThisDisabledError, ProviderQuotaExceededError, ProviderModelCurrentlyNotSupportError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.llm.error import LLMRateLimitError, LLMBadRequestError, LLMAuthorizationError, LLMAPIConnectionError, \
+    ProviderTokenNotInitError, LLMAPIUnavailableError, QuotaExceededError, ModelCurrentlyNotSupportError
+from libs.helper import uuid_value, TimestampField
+from libs.infinite_scroll_pagination import InfiniteScrollPagination
+from extensions.ext_database import db
+from models.model import MessageAnnotation, Conversation, Message, MessageFeedback
+from services.completion_service import CompletionService
+from services.errors.app import MoreLikeThisDisabledError
+from services.errors.conversation import ConversationNotExistsError
+from services.errors.message import MessageNotExistsError
+from services.message_service import MessageService
+
+
+class ChatMessageApi(Resource):
+    account_fields = {
+        'id': fields.String,
+        'name': fields.String,
+        'email': fields.String
+    }
+
+    feedback_fields = {
+        'rating': fields.String,
+        'content': fields.String,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account': fields.Nested(account_fields, allow_null=True),
+    }
+
+    annotation_fields = {
+        'content': fields.String,
+        'account': fields.Nested(account_fields, allow_null=True),
+        'created_at': TimestampField
+    }
+
+    message_detail_fields = {
+        'id': fields.String,
+        'conversation_id': fields.String,
+        'inputs': fields.Raw,
+        'query': fields.String,
+        'message': fields.Raw,
+        'message_tokens': fields.Integer,
+        'answer': fields.String,
+        'answer_tokens': fields.Integer,
+        'provider_response_latency': fields.Integer,
+        'from_source': fields.String,
+        'from_end_user_id': fields.String,
+        'from_account_id': fields.String,
+        'feedbacks': fields.List(fields.Nested(feedback_fields)),
+        'annotation': fields.Nested(annotation_fields, allow_null=True),
+        'created_at': TimestampField
+    }
+
+    message_infinite_scroll_pagination_fields = {
+        'limit': fields.Integer,
+        'has_more': fields.Boolean,
+        'data': fields.List(fields.Nested(message_detail_fields))
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(message_infinite_scroll_pagination_fields)
+    def get(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app = _get_app(app_id, 'chat')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('conversation_id', required=True, type=uuid_value, location='args')
+        parser.add_argument('first_id', type=uuid_value, location='args')
+        parser.add_argument('limit', type=int_range(1, 100), required=False, default=20, location='args')
+        args = parser.parse_args()
+
+        conversation = db.session.query(Conversation).filter(
+            Conversation.id == args['conversation_id'],
+            Conversation.app_id == app.id
+        ).first()
+
+        if not conversation:
+            raise NotFound("Conversation Not Exists.")
+
+        if args['first_id']:
+            first_message = db.session.query(Message) \
+                .filter(Message.conversation_id == conversation.id, Message.id == args['first_id']).first()
+
+            if not first_message:
+                raise NotFound("First message not found")
+
+            history_messages = db.session.query(Message).filter(
+                Message.conversation_id == conversation.id,
+                Message.created_at < first_message.created_at,
+                Message.id != first_message.id
+            ) \
+                .order_by(Message.created_at.desc()).limit(args['limit']).all()
+        else:
+            history_messages = db.session.query(Message).filter(Message.conversation_id == conversation.id) \
+                .order_by(Message.created_at.desc()).limit(args['limit']).all()
+
+        has_more = False
+        if len(history_messages) == args['limit']:
+            current_page_first_message = history_messages[-1]
+            rest_count = db.session.query(Message).filter(
+                Message.conversation_id == conversation.id,
+                Message.created_at < current_page_first_message.created_at,
+                Message.id != current_page_first_message.id
+            ).count()
+
+            if rest_count > 0:
+                has_more = True
+
+        history_messages = list(reversed(history_messages))
+
+        return InfiniteScrollPagination(
+            data=history_messages,
+            limit=args['limit'],
+            has_more=has_more
+        )
+
+
+class MessageFeedbackApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('message_id', required=True, type=uuid_value, location='json')
+        parser.add_argument('rating', type=str, choices=['like', 'dislike', None], location='json')
+        args = parser.parse_args()
+
+        message_id = str(args['message_id'])
+
+        message = db.session.query(Message).filter(
+            Message.id == message_id,
+            Message.app_id == app.id
+        ).first()
+
+        if not message:
+            raise NotFound("Message Not Exists.")
+
+        feedback = message.admin_feedback
+
+        if not args['rating'] and feedback:
+            db.session.delete(feedback)
+        elif args['rating'] and feedback:
+            feedback.rating = args['rating']
+        elif not args['rating'] and not feedback:
+            raise ValueError('rating cannot be None when feedback not exists')
+        else:
+            feedback = MessageFeedback(
+                app_id=app.id,
+                conversation_id=message.conversation_id,
+                message_id=message.id,
+                rating=args['rating'],
+                from_source='admin',
+                from_account_id=current_user.id
+            )
+            db.session.add(feedback)
+
+        db.session.commit()
+
+        return {'result': 'success'}
+
+
+class MessageAnnotationApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('message_id', required=True, type=uuid_value, location='json')
+        parser.add_argument('content', type=str, location='json')
+        args = parser.parse_args()
+
+        message_id = str(args['message_id'])
+
+        message = db.session.query(Message).filter(
+            Message.id == message_id,
+            Message.app_id == app.id
+        ).first()
+
+        if not message:
+            raise NotFound("Message Not Exists.")
+
+        annotation = message.annotation
+
+        if annotation:
+            annotation.content = args['content']
+        else:
+            annotation = MessageAnnotation(
+                app_id=app.id,
+                conversation_id=message.conversation_id,
+                message_id=message.id,
+                content=args['content'],
+                account_id=current_user.id
+            )
+            db.session.add(annotation)
+
+        db.session.commit()
+
+        return {'result': 'success'}
+
+
+class MessageAnnotationCountApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        app_id = str(app_id)
+
+        # get app info
+        app = _get_app(app_id)
+
+        count = db.session.query(MessageAnnotation).filter(
+            MessageAnnotation.app_id == app.id
+        ).count()
+
+        return {'count': count}
+
+
+class MessageMoreLikeThisApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id, message_id):
+        app_id = str(app_id)
+        message_id = str(message_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('response_mode', type=str, required=True, choices=['blocking', 'streaming'], location='args')
+        args = parser.parse_args()
+
+        streaming = args['response_mode'] == 'streaming'
+
+        # get app info
+        app_model = _get_app(app_id, 'completion')
+
+        try:
+            response = CompletionService.generate_more_like_this(app_model, current_user, message_id, streaming)
+            return compact_response(response)
+        except MessageNotExistsError:
+            raise NotFound("Message Not Exists.")
+        except MoreLikeThisDisabledError:
+            raise AppMoreLikeThisDisabledError()
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
+def compact_response(response: Union[dict | Generator]) -> Response:
+    if isinstance(response, dict):
+        return Response(response=json.dumps(response), status=200, mimetype='application/json')
+    else:
+        def generate() -> Generator:
+            try:
+                for chunk in response:
+                    yield chunk
+            except MessageNotExistsError:
+                yield "data: " + json.dumps(api.handle_error(NotFound("Message Not Exists.")).get_json()) + "\n\n"
+            except MoreLikeThisDisabledError:
+                yield "data: " + json.dumps(api.handle_error(AppMoreLikeThisDisabledError()).get_json()) + "\n\n"
+            except ProviderTokenNotInitError:
+                yield "data: " + json.dumps(api.handle_error(ProviderNotInitializeError()).get_json()) + "\n\n"
+            except QuotaExceededError:
+                yield "data: " + json.dumps(api.handle_error(ProviderQuotaExceededError()).get_json()) + "\n\n"
+            except ModelCurrentlyNotSupportError:
+                yield "data: " + json.dumps(api.handle_error(ProviderModelCurrentlyNotSupportError()).get_json()) + "\n\n"
+            except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                    LLMRateLimitError, LLMAuthorizationError) as e:
+                yield "data: " + json.dumps(api.handle_error(CompletionRequestError(str(e))).get_json()) + "\n\n"
+            except ValueError as e:
+                yield "data: " + json.dumps(api.handle_error(e).get_json()) + "\n\n"
+            except Exception:
+                logging.exception("internal server error.")
+                yield "data: " + json.dumps(api.handle_error(InternalServerError()).get_json()) + "\n\n"
+
+        return Response(stream_with_context(generate()), status=200,
+                        mimetype='text/event-stream')
+
+
+class MessageSuggestedQuestionApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id, message_id):
+        app_id = str(app_id)
+        message_id = str(message_id)
+
+        # get app info
+        app_model = _get_app(app_id, 'chat')
+
+        try:
+            questions = MessageService.get_suggested_questions_after_answer(
+                app_model=app_model,
+                user=current_user,
+                message_id=message_id,
+                check_enabled=False
+            )
+        except MessageNotExistsError:
+            raise NotFound("Message not found")
+        except ConversationNotExistsError:
+            raise NotFound("Conversation not found")
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+                LLMRateLimitError, LLMAuthorizationError) as e:
+            raise CompletionRequestError(str(e))
+        except Exception:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+        return {'data': questions}
+
+
+api.add_resource(MessageMoreLikeThisApi, '/apps/<uuid:app_id>/completion-messages/<uuid:message_id>/more-like-this')
+api.add_resource(MessageSuggestedQuestionApi, '/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions')
+api.add_resource(ChatMessageApi, '/apps/<uuid:app_id>/chat-messages', endpoint='chat_messages')
+api.add_resource(MessageFeedbackApi, '/apps/<uuid:app_id>/feedbacks')
+api.add_resource(MessageAnnotationApi, '/apps/<uuid:app_id>/annotations')
+api.add_resource(MessageAnnotationCountApi, '/apps/<uuid:app_id>/annotations/count')

api/controllers/console/app/model_config.py

@@ -0,0 +1,65 @@
+# -*- coding:utf-8 -*-
+import json
+
+from flask import request
+from flask_restful import Resource
+from flask_login import login_required, current_user
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from events.app_event import app_model_config_was_updated
+from extensions.ext_database import db
+from models.model import AppModelConfig
+from services.app_model_config_service import AppModelConfigService
+
+
+class ModelConfigResource(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        """Modify app model config"""
+        app_id = str(app_id)
+
+        app_model = _get_app(app_id)
+
+        # validate config
+        model_configuration = AppModelConfigService.validate_configuration(
+            account=current_user,
+            config=request.json,
+            mode=app_model.mode
+        )
+
+        new_app_model_config = AppModelConfig(
+            app_id=app_model.id,
+            provider="",
+            model_id="",
+            configs={},
+            opening_statement=model_configuration['opening_statement'],
+            suggested_questions=json.dumps(model_configuration['suggested_questions']),
+            suggested_questions_after_answer=json.dumps(model_configuration['suggested_questions_after_answer']),
+            more_like_this=json.dumps(model_configuration['more_like_this']),
+            model=json.dumps(model_configuration['model']),
+            user_input_form=json.dumps(model_configuration['user_input_form']),
+            pre_prompt=model_configuration['pre_prompt'],
+            agent_mode=json.dumps(model_configuration['agent_mode']),
+        )
+
+        db.session.add(new_app_model_config)
+        db.session.flush()
+
+        app_model.app_model_config_id = new_app_model_config.id
+        db.session.commit()
+
+        app_model_config_was_updated.send(
+            app_model,
+            app_model_config=new_app_model_config
+        )
+
+        return {'result': 'success'}
+
+
+api.add_resource(ModelConfigResource, '/apps/<uuid:app_id>/model-config')

api/controllers/console/app/site.py

@@ -0,0 +1,114 @@
+# -*- coding:utf-8 -*-
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with
+from werkzeug.exceptions import NotFound, Forbidden
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import supported_language
+from extensions.ext_database import db
+from models.model import Site
+
+app_site_fields = {
+    'app_id': fields.String,
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean
+}
+
+
+def parse_app_site_args():
+    parser = reqparse.RequestParser()
+    parser.add_argument('title', type=str, required=False, location='json')
+    parser.add_argument('icon', type=str, required=False, location='json')
+    parser.add_argument('icon_background', type=str, required=False, location='json')
+    parser.add_argument('description', type=str, required=False, location='json')
+    parser.add_argument('default_language', type=supported_language, required=False, location='json')
+    parser.add_argument('customize_domain', type=str, required=False, location='json')
+    parser.add_argument('copyright', type=str, required=False, location='json')
+    parser.add_argument('privacy_policy', type=str, required=False, location='json')
+    parser.add_argument('customize_token_strategy', type=str, choices=['must', 'allow', 'not_allow'],
+                        required=False,
+                        location='json')
+    parser.add_argument('prompt_public', type=bool, required=False, location='json')
+    return parser.parse_args()
+
+
+class AppSite(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_site_fields)
+    def post(self, app_id):
+        args = parse_app_site_args()
+
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        site = db.session.query(Site). \
+            filter(Site.app_id == app_model.id). \
+            one_or_404()
+
+        for attr_name in [
+            'title',
+            'icon',
+            'icon_background',
+            'description',
+            'default_language',
+            'customize_domain',
+            'copyright',
+            'privacy_policy',
+            'customize_token_strategy',
+            'prompt_public'
+        ]:
+            value = args.get(attr_name)
+            if value is not None:
+                setattr(site, attr_name, value)
+
+        db.session.commit()
+
+        return site
+
+
+class AppSiteAccessTokenReset(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_site_fields)
+    def post(self, app_id):
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        site = db.session.query(Site).filter(Site.app_id == app_model.id).first()
+
+        if not site:
+            raise NotFound
+
+        site.code = Site.generate_code(16)
+        db.session.commit()
+
+        return site
+
+
+api.add_resource(AppSite, '/apps/<uuid:app_id>/site')
+api.add_resource(AppSiteAccessTokenReset, '/apps/<uuid:app_id>/site/access-token-reset')

api/controllers/console/app/statistic.py

@@ -0,0 +1,202 @@
+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+import pytz
+from flask import jsonify
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app import _get_app
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import datetime_string
+from extensions.ext_database import db
+
+
+class DailyConversationStatistic(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+        SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.conversation_id) AS conversation_count
+            FROM messages where app_id = :app_id 
+        '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'conversation_count': i.conversation_count
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+class DailyTerminalsStatistic(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, count(distinct messages.from_end_user_id) AS terminal_count
+                    FROM messages where app_id = :app_id 
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'terminal_count': i.terminal_count
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+class DailyTokenCostStatistic(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        account = current_user
+        app_id = str(app_id)
+        app_model = _get_app(app_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        parser.add_argument('end', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
+        args = parser.parse_args()
+
+        sql_query = '''
+                SELECT date(DATE_TRUNC('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE :tz )) AS date, 
+                    (sum(messages.message_tokens) + sum(messages.answer_tokens)) as token_count,
+                    sum(total_price) as total_price
+                    FROM messages where app_id = :app_id 
+                '''
+        arg_dict = {'tz': account.timezone, 'app_id': app_model.id}
+
+        timezone = pytz.timezone(account.timezone)
+        utc_timezone = pytz.utc
+
+        if args['start']:
+            start_datetime = datetime.strptime(args['start'], '%Y-%m-%d %H:%M')
+            start_datetime = start_datetime.replace(second=0)
+
+            start_datetime_timezone = timezone.localize(start_datetime)
+            start_datetime_utc = start_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at >= :start'
+            arg_dict['start'] = start_datetime_utc
+
+        if args['end']:
+            end_datetime = datetime.strptime(args['end'], '%Y-%m-%d %H:%M')
+            end_datetime = end_datetime.replace(second=0)
+
+            end_datetime_timezone = timezone.localize(end_datetime)
+            end_datetime_utc = end_datetime_timezone.astimezone(utc_timezone)
+
+            sql_query += ' and created_at < :end'
+            arg_dict['end'] = end_datetime_utc
+
+        sql_query += ' GROUP BY date order by date'
+        rs = db.session.execute(sql_query, arg_dict)
+
+        response_date = []
+
+        for i in rs:
+            response_date.append({
+                'date': str(i.date),
+                'token_count': i.token_count,
+                'total_price': i.total_price,
+                'currency': 'USD'
+            })
+
+        return jsonify({
+            'data': response_date
+        })
+
+
+api.add_resource(DailyConversationStatistic, '/apps/<uuid:app_id>/statistics/daily-conversations')
+api.add_resource(DailyTerminalsStatistic, '/apps/<uuid:app_id>/statistics/daily-end-users')
+api.add_resource(DailyTokenCostStatistic, '/apps/<uuid:app_id>/statistics/token-costs')

api/controllers/console/auth/login.py

@@ -0,0 +1,109 @@
+# -*- coding:utf-8 -*-
+import flask
+import flask_login
+from flask import request, current_app
+from flask_restful import Resource, reqparse
+
+import services
+from controllers.console import api
+from controllers.console.error import AccountNotLinkTenantError
+from controllers.console.setup import setup_required
+from libs.helper import email
+from libs.password import valid_password
+from services.account_service import AccountService, TenantService
+
+
+class LoginApi(Resource):
+    """Resource for user login."""
+
+    @setup_required
+    def post(self):
+        """Authenticate user and login."""
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=email, required=True, location='json')
+        parser.add_argument('password', type=valid_password, required=True, location='json')
+        parser.add_argument('remember_me', type=bool, required=False, default=False, location='json')
+        args = parser.parse_args()
+
+        # todo: Verify the recaptcha
+
+        try:
+            account = AccountService.authenticate(args['email'], args['password'])
+        except services.errors.account.AccountLoginError:
+            return {'code': 'unauthorized', 'message': 'Invalid email or password'}, 401
+
+        try:
+            TenantService.switch_tenant(account)
+        except Exception:
+            raise AccountNotLinkTenantError("Account not link tenant")
+
+        flask_login.login_user(account, remember=args['remember_me'])
+        AccountService.update_last_login(account, request)
+
+        # todo: return the user info
+
+        return {'result': 'success'}
+
+
+class LogoutApi(Resource):
+
+    @setup_required
+    def get(self):
+        flask.session.pop('workspace_id', None)
+        flask_login.logout_user()
+        return {'result': 'success'}
+
+
+class ResetPasswordApi(Resource):
+    @setup_required
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=email, required=True, location='json')
+        args = parser.parse_args()
+
+        # import mailchimp_transactional as MailchimpTransactional
+        # from mailchimp_transactional.api_client import ApiClientError
+
+        account = {'email': args['email']}
+        # account = AccountService.get_by_email(args['email'])
+        # if account is None:
+        #     raise ValueError('Email not found')
+        # new_password = AccountService.generate_password()
+        # AccountService.update_password(account, new_password)
+
+        # todo: Send email
+        MAILCHIMP_API_KEY = current_app.config['MAILCHIMP_TRANSACTIONAL_API_KEY']
+        # mailchimp = MailchimpTransactional(MAILCHIMP_API_KEY)
+
+        message = {
+            'from_email': 'noreply@example.com',
+            'to': [{'email': account.email}],
+            'subject': 'Reset your Dify password',
+            'html': """
+                <p>Dear User,</p>
+                <p>The Dify team has generated a new password for you, details as follows:</p> 
+                <p><strong>{new_password}</strong></p>
+                <p>Please change your password to log in as soon as possible.</p>
+                <p>Regards,</p>
+                <p>The Dify Team</p> 
+            """
+        }
+
+        # response = mailchimp.messages.send({
+        #     'message': message,
+        #     # required for transactional email
+        #     ' settings': {
+        #         'sandbox_mode': current_app.config['MAILCHIMP_SANDBOX_MODE'],
+        #     },
+        # })
+
+        # Check if MSG was sent
+        # if response.status_code != 200:
+        #     # handle error
+        #     pass
+
+        return {'result': 'success'}
+
+
+api.add_resource(LoginApi, '/login')
+api.add_resource(LogoutApi, '/logout')

api/controllers/console/auth/oauth.py

@@ -0,0 +1,126 @@
+import logging
+from datetime import datetime
+from typing import Optional
+
+import flask_login
+import requests
+from flask import request, redirect, current_app, session
+from flask_restful import Resource
+
+from libs.oauth import OAuthUserInfo, GitHubOAuth, GoogleOAuth
+from extensions.ext_database import db
+from models.account import Account, AccountStatus
+from services.account_service import AccountService, RegisterService
+from .. import api
+
+
+def get_oauth_providers():
+    with current_app.app_context():
+        github_oauth = GitHubOAuth(client_id=current_app.config.get('GITHUB_CLIENT_ID'),
+                                   client_secret=current_app.config.get(
+                                       'GITHUB_CLIENT_SECRET'),
+                                   redirect_uri=current_app.config.get(
+                                       'CONSOLE_URL') + '/console/api/oauth/authorize/github')
+
+        google_oauth = GoogleOAuth(client_id=current_app.config.get('GOOGLE_CLIENT_ID'),
+                                   client_secret=current_app.config.get(
+                                       'GOOGLE_CLIENT_SECRET'),
+                                   redirect_uri=current_app.config.get(
+                                       'CONSOLE_URL') + '/console/api/oauth/authorize/google')
+
+        OAUTH_PROVIDERS = {
+            'github': github_oauth,
+            'google': google_oauth
+        }
+        return OAUTH_PROVIDERS
+
+
+class OAuthLogin(Resource):
+    def get(self, provider: str):
+        OAUTH_PROVIDERS = get_oauth_providers()
+        with current_app.app_context():
+            oauth_provider = OAUTH_PROVIDERS.get(provider)
+            print(vars(oauth_provider))
+        if not oauth_provider:
+            return {'error': 'Invalid provider'}, 400
+
+        auth_url = oauth_provider.get_authorization_url()
+        return redirect(auth_url)
+
+
+class OAuthCallback(Resource):
+    def get(self, provider: str):
+        OAUTH_PROVIDERS = get_oauth_providers()
+        with current_app.app_context():
+            oauth_provider = OAUTH_PROVIDERS.get(provider)
+        if not oauth_provider:
+            return {'error': 'Invalid provider'}, 400
+
+        code = request.args.get('code')
+        try:
+            token = oauth_provider.get_access_token(code)
+            user_info = oauth_provider.get_user_info(token)
+        except requests.exceptions.HTTPError as e:
+            logging.exception(
+                f"An error occurred during the OAuth process with {provider}: {e.response.text}")
+            return {'error': 'OAuth process failed'}, 400
+
+        account = _generate_account(provider, user_info)
+        # Check account status
+        if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
+            return {'error': 'Account is banned or closed.'}, 403
+
+        if account.status == AccountStatus.PENDING.value:
+            account.status = AccountStatus.ACTIVE.value
+            account.initialized_at = datetime.utcnow()
+            db.session.commit()
+
+        # login user
+        session.clear()
+        flask_login.login_user(account, remember=True)
+        AccountService.update_last_login(account, request)
+
+        return redirect(f'{current_app.config.get("CONSOLE_URL")}?oauth_login=success')
+
+
+def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:
+    account = Account.get_by_openid(provider, user_info.id)
+
+    if not account:
+        account = Account.query.filter_by(email=user_info.email).first()
+
+    return account
+
+
+def _generate_account(provider: str, user_info: OAuthUserInfo):
+    # Get account by openid or email.
+    account = _get_account_by_openid_or_email(provider, user_info)
+
+    if not account:
+        # Create account
+        account_name = user_info.name if user_info.name else 'Dify'
+        account = RegisterService.register(
+            email=user_info.email,
+            name=account_name,
+            password=None,
+            open_id=user_info.id,
+            provider=provider
+        )
+
+        # Set interface language
+        preferred_lang = request.accept_languages.best_match(['zh', 'en'])
+        if preferred_lang == 'zh':
+            interface_language = 'zh-Hans'
+        else:
+            interface_language = 'en-US'
+        account.interface_language = interface_language
+        db.session.commit()
+
+    # Link account
+    AccountService.link_account_integrate(provider, user_info.id, account)
+
+    return account
+
+
+api.add_resource(OAuthLogin, '/oauth/login/<provider>')
+api.add_resource(OAuthCallback, '/oauth/authorize/<provider>')

api/controllers/console/datasets/datasets.py

@@ -0,0 +1,281 @@
+# -*- coding:utf-8 -*-
+from flask import request
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal, marshal_with
+from werkzeug.exceptions import NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.indexing_runner import IndexingRunner
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.model import UploadFile
+from services.dataset_service import DatasetService
+
+dataset_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'provider': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'app_count': fields.Integer,
+    'document_count': fields.Integer,
+    'word_count': fields.Integer,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'updated_by': fields.String,
+    'updated_at': TimestampField,
+}
+
+dataset_query_detail_fields = {
+    "id": fields.String,
+    "content": fields.String,
+    "source": fields.String,
+    "source_app_id": fields.String,
+    "created_by_role": fields.String,
+    "created_by": fields.String,
+    "created_at": TimestampField
+}
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 40:
+        raise ValueError('Name must be between 1 to 40 characters.')
+    return name
+
+
+def _validate_description_length(description):
+    if len(description) > 200:
+        raise ValueError('Description cannot exceed 200 characters.')
+    return description
+
+
+class DatasetListApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+        ids = request.args.getlist('ids')
+        provider = request.args.get('provider', default="vendor")
+        if ids:
+            datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
+        else:
+            datasets, total = DatasetService.get_datasets(page, limit, provider,
+                                                          current_user.current_tenant_id, current_user)
+
+        response = {
+            'data': marshal(datasets, dataset_detail_fields),
+            'has_more': len(datasets) == limit,
+            'limit': limit,
+            'total': total,
+            'page': page
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', nullable=False, required=True,
+                            help='type is required. Name must be between 1 to 40 characters.',
+                            type=_validate_name)
+        parser.add_argument('indexing_technique', type=str, location='json',
+                            choices=('high_quality', 'economy'),
+                            help='Invalid indexing technique.')
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        try:
+            dataset = DatasetService.create_empty_dataset(
+                tenant_id=current_user.current_tenant_id,
+                name=args['name'],
+                indexing_technique=args['indexing_technique'],
+                account=current_user
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 201
+
+
+class DatasetApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(
+                dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', nullable=False,
+                            help='type is required. Name must be between 1 to 40 characters.',
+                            type=_validate_name)
+        parser.add_argument('description',
+                            location='json', store_missing=False,
+                            type=_validate_description_length)
+        parser.add_argument('indexing_technique', type=str, location='json',
+                            choices=('high_quality', 'economy'),
+                            help='Invalid indexing technique.')
+        parser.add_argument('permission', type=str, location='json', choices=(
+            'only_me', 'all_team_members'), help='Invalid permission.')
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        dataset = DatasetService.update_dataset(
+            dataset_id_str, args, current_user)
+
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        if DatasetService.delete_dataset(dataset_id_str, current_user):
+            return {'result': 'success'}, 204
+        else:
+            raise NotFound("Dataset not found.")
+
+
+class DatasetQueryApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+
+        dataset_queries, total = DatasetService.get_dataset_queries(
+            dataset_id=dataset.id,
+            page=page,
+            per_page=limit
+        )
+
+        response = {
+            'data': marshal(dataset_queries, dataset_query_detail_fields),
+            'has_more': len(dataset_queries) == limit,
+            'limit': limit,
+            'total': total,
+            'page': page
+        }
+        return response, 200
+
+
+class DatasetIndexingEstimateApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        segment_rule = request.get_json()
+        file_detail = db.session.query(UploadFile).filter(
+            UploadFile.tenant_id == current_user.current_tenant_id,
+            UploadFile.id == segment_rule["file_id"]
+        ).first()
+
+        if file_detail is None:
+            raise NotFound("File not found.")
+
+        indexing_runner = IndexingRunner()
+        response = indexing_runner.indexing_estimate(file_detail, segment_rule['process_rule'])
+        return response, 200
+
+
+class DatasetRelatedAppListApi(Resource):
+    app_detail_kernel_fields = {
+        'id': fields.String,
+        'name': fields.String,
+        'mode': fields.String,
+        'icon': fields.String,
+        'icon_background': fields.String,
+    }
+
+    related_app_list = {
+        'data': fields.List(fields.Nested(app_detail_kernel_fields)),
+        'total': fields.Integer,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(related_app_list)
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        app_dataset_joins = DatasetService.get_related_apps(dataset.id)
+
+        related_apps = []
+        for app_dataset_join in app_dataset_joins:
+            app_model = app_dataset_join.app
+            if app_model:
+                related_apps.append(app_model)
+
+        return {
+            'data': related_apps,
+            'total': len(related_apps)
+        }, 200
+
+
+api.add_resource(DatasetListApi, '/datasets')
+api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
+api.add_resource(DatasetQueryApi, '/datasets/<uuid:dataset_id>/queries')
+api.add_resource(DatasetIndexingEstimateApi, '/datasets/file-indexing-estimate')
+api.add_resource(DatasetRelatedAppListApi, '/datasets/<uuid:dataset_id>/related-apps')

api/controllers/console/datasets/datasets_document.py

@@ -0,0 +1,682 @@
+# -*- coding:utf-8 -*-
+import random
+from datetime import datetime
+
+from flask import request
+from flask_login import login_required, current_user
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from sqlalchemy import desc, asc
+from werkzeug.exceptions import NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.app.error import ProviderNotInitializeError
+from controllers.console.datasets.error import DocumentAlreadyFinishedError, InvalidActionError, DocumentIndexingError, \
+    InvalidMetadataError, ArchivedDocumentImmutableError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.indexing_runner import IndexingRunner
+from core.llm.error import ProviderTokenNotInitError
+from extensions.ext_redis import redis_client
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.dataset import DatasetProcessRule, Dataset
+from models.dataset import Document, DocumentSegment
+from models.model import UploadFile
+from services.dataset_service import DocumentService, DatasetService
+from tasks.add_document_to_index_task import add_document_to_index_task
+from tasks.remove_document_from_index_task import remove_document_from_index_task
+
+dataset_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+}
+
+document_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'data_source_type': fields.String,
+    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
+    'dataset_process_rule_id': fields.String,
+    'name': fields.String,
+    'created_from': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'tokens': fields.Integer,
+    'indexing_status': fields.String,
+    'error': fields.String,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'archived': fields.Boolean,
+    'display_status': fields.String,
+    'word_count': fields.Integer,
+    'hit_count': fields.Integer,
+}
+
+
+class DocumentResource(Resource):
+    def get_document(self, dataset_id: str, document_id: str) -> Document:
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        document = DocumentService.get_document(dataset_id, document_id)
+
+        if not document:
+            raise NotFound('Document not found.')
+
+        if document.tenant_id != current_user.current_tenant_id:
+            raise Forbidden('No permission.')
+
+        return document
+
+
+class GetProcessRuleApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        req_data = request.args
+
+        document_id = req_data.get('document_id')
+        if document_id:
+            # get the latest process rule
+            document = Document.query.get_or_404(document_id)
+
+            dataset = DatasetService.get_dataset(document.dataset_id)
+
+            if not dataset:
+                raise NotFound('Dataset not found.')
+
+            try:
+                DatasetService.check_dataset_permission(dataset, current_user)
+            except services.errors.account.NoPermissionError as e:
+                raise Forbidden(str(e))
+
+            # get the latest process rule
+            dataset_process_rule = db.session.query(DatasetProcessRule). \
+                filter(DatasetProcessRule.dataset_id == document.dataset_id). \
+                order_by(DatasetProcessRule.created_at.desc()). \
+                limit(1). \
+                one_or_none()
+            mode = dataset_process_rule.mode
+            rules = dataset_process_rule.rules_dict
+        else:
+            mode = DocumentService.DEFAULT_RULES['mode']
+            rules = DocumentService.DEFAULT_RULES['rules']
+
+        return {
+            'mode': mode,
+            'rules': rules
+        }
+
+
+class DatasetDocumentListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id = str(dataset_id)
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+        search = request.args.get('search', default=None, type=str)
+        sort = request.args.get('sort', default='-created_at', type=str)
+
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        query = Document.query.filter_by(
+            dataset_id=str(dataset_id), tenant_id=current_user.current_tenant_id)
+
+        if search:
+            search = f'%{search}%'
+            query = query.filter(Document.name.like(search))
+
+        if sort.startswith('-'):
+            sort_logic = desc
+            sort = sort[1:]
+        else:
+            sort_logic = asc
+
+        if sort == 'hit_count':
+            sub_query = db.select(DocumentSegment.document_id,
+                                  db.func.sum(DocumentSegment.hit_count).label("total_hit_count")) \
+                .group_by(DocumentSegment.document_id) \
+                .subquery()
+
+            query = query.outerjoin(sub_query, sub_query.c.document_id == Document.id) \
+                .order_by(sort_logic(db.func.coalesce(sub_query.c.total_hit_count, 0)))
+        elif sort == 'created_at':
+            query = query.order_by(sort_logic(Document.created_at))
+        else:
+            query = query.order_by(desc(Document.created_at))
+
+        paginated_documents = query.paginate(
+            page=page, per_page=limit, max_per_page=100, error_out=False)
+        documents = paginated_documents.items
+
+        response = {
+            'data': marshal(documents, document_fields),
+            'has_more': len(documents) == limit,
+            'limit': limit,
+            'total': paginated_documents.total,
+            'page': page
+        }
+
+        return response
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(document_fields)
+    def post(self, dataset_id):
+        dataset_id = str(dataset_id)
+
+        dataset = DatasetService.get_dataset(dataset_id)
+
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('indexing_technique', type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False,
+                            location='json')
+        parser.add_argument('data_source', type=dict, required=True, nullable=True, location='json')
+        parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
+        parser.add_argument('duplicate', type=bool, nullable=False, location='json')
+        args = parser.parse_args()
+
+        if not dataset.indexing_technique and not args['indexing_technique']:
+            raise ValueError('indexing_technique is required.')
+
+        # validate args
+        DocumentService.document_create_args_validate(args)
+
+        try:
+            document = DocumentService.save_document_with_dataset_id(dataset, args, current_user)
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+
+        return document
+
+
+class DatasetInitApi(Resource):
+    dataset_and_document_fields = {
+        'dataset': fields.Nested(dataset_fields),
+        'document': fields.Nested(document_fields)
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(dataset_and_document_fields)
+    def post(self):
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('indexing_technique', type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, required=True,
+                            nullable=False, location='json')
+        parser.add_argument('data_source', type=dict, required=True, nullable=True, location='json')
+        parser.add_argument('process_rule', type=dict, required=True, nullable=True, location='json')
+        args = parser.parse_args()
+
+        # validate args
+        DocumentService.document_create_args_validate(args)
+
+        try:
+            dataset, document = DocumentService.save_document_without_dataset_id(
+                tenant_id=current_user.current_tenant_id,
+                document_data=args,
+                account=current_user
+            )
+        except ProviderTokenNotInitError:
+            raise ProviderNotInitializeError()
+
+        response = {
+            'dataset': dataset,
+            'document': document
+        }
+
+        return response
+
+
+class DocumentIndexingEstimateApi(DocumentResource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        if document.indexing_status in ['completed', 'error']:
+            raise DocumentAlreadyFinishedError()
+
+        data_process_rule = document.dataset_process_rule
+        data_process_rule_dict = data_process_rule.to_dict()
+
+        response = {
+            "tokens": 0,
+            "total_price": 0,
+            "currency": "USD",
+            "total_segments": 0,
+            "preview": []
+        }
+
+        if document.data_source_type == 'upload_file':
+            data_source_info = document.data_source_info_dict
+            if data_source_info and 'upload_file_id' in data_source_info:
+                file_id = data_source_info['upload_file_id']
+
+                file = db.session.query(UploadFile).filter(
+                    UploadFile.tenant_id == document.tenant_id,
+                    UploadFile.id == file_id
+                ).first()
+
+                # raise error if file not found
+                if not file:
+                    raise NotFound('File not found.')
+
+                indexing_runner = IndexingRunner()
+                response = indexing_runner.indexing_estimate(file, data_process_rule_dict)
+
+        return response
+
+
+class DocumentIndexingStatusApi(DocumentResource):
+    document_status_fields = {
+        'id': fields.String,
+        'indexing_status': fields.String,
+        'processing_started_at': TimestampField,
+        'parsing_completed_at': TimestampField,
+        'cleaning_completed_at': TimestampField,
+        'splitting_completed_at': TimestampField,
+        'completed_at': TimestampField,
+        'paused_at': TimestampField,
+        'error': fields.String,
+        'stopped_at': TimestampField,
+        'completed_segments': fields.Integer,
+        'total_segments': fields.Integer,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        completed_segments = DocumentSegment.query \
+            .filter(DocumentSegment.completed_at.isnot(None),
+                    DocumentSegment.document_id == str(document_id)) \
+            .count()
+        total_segments = DocumentSegment.query \
+            .filter_by(document_id=str(document_id)) \
+            .count()
+
+        document.completed_segments = completed_segments
+        document.total_segments = total_segments
+
+        return marshal(document, self.document_status_fields)
+
+
+class DocumentDetailApi(DocumentResource):
+    METADATA_CHOICES = {'all', 'only', 'without'}
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        metadata = request.args.get('metadata', 'all')
+        if metadata not in self.METADATA_CHOICES:
+            raise InvalidMetadataError(f'Invalid metadata value: {metadata}')
+
+        if metadata == 'only':
+            response = {
+                'id': document.id,
+                'doc_type': document.doc_type,
+                'doc_metadata': document.doc_metadata
+            }
+        elif metadata == 'without':
+            process_rules = DatasetService.get_process_rules(dataset_id)
+            data_source_info = document.data_source_detail_dict
+            response = {
+                'id': document.id,
+                'position': document.position,
+                'data_source_type': document.data_source_type,
+                'data_source_info': data_source_info,
+                'dataset_process_rule_id': document.dataset_process_rule_id,
+                'dataset_process_rule': process_rules,
+                'name': document.name,
+                'created_from': document.created_from,
+                'created_by': document.created_by,
+                'created_at': document.created_at.timestamp(),
+                'tokens': document.tokens,
+                'indexing_status': document.indexing_status,
+                'completed_at': int(document.completed_at.timestamp()) if document.completed_at else None,
+                'updated_at': int(document.updated_at.timestamp()) if document.updated_at else None,
+                'indexing_latency': document.indexing_latency,
+                'error': document.error,
+                'enabled': document.enabled,
+                'disabled_at': int(document.disabled_at.timestamp()) if document.disabled_at else None,
+                'disabled_by': document.disabled_by,
+                'archived': document.archived,
+                'segment_count': document.segment_count,
+                'average_segment_length':   document.average_segment_length,
+                'hit_count': document.hit_count,
+                'display_status': document.display_status
+            }
+        else:
+            process_rules = DatasetService.get_process_rules(dataset_id)
+            data_source_info = document.data_source_detail_dict_()
+            response = {
+                'id': document.id,
+                'position': document.position,
+                'data_source_type': document.data_source_type,
+                'data_source_info': data_source_info,
+                'dataset_process_rule_id': document.dataset_process_rule_id,
+                'dataset_process_rule': process_rules,
+                'name': document.name,
+                'created_from': document.created_from,
+                'created_by': document.created_by,
+                'created_at': document.created_at.timestamp(),
+                'tokens': document.tokens,
+                'indexing_status': document.indexing_status,
+                'completed_at': int(document.completed_at.timestamp())if document.completed_at else None,
+                'updated_at': int(document.updated_at.timestamp()) if document.updated_at else None,
+                'indexing_latency': document.indexing_latency,
+                'error': document.error,
+                'enabled': document.enabled,
+                'disabled_at': int(document.disabled_at.timestamp()) if document.disabled_at else None,
+                'disabled_by': document.disabled_by,
+                'archived': document.archived,
+                'doc_type': document.doc_type,
+                'doc_metadata': document.doc_metadata,
+                'segment_count': document.segment_count,
+                'average_segment_length': document.average_segment_length,
+                'hit_count': document.hit_count,
+                'display_status': document.display_status
+            }
+
+        return response, 200
+
+
+class DocumentProcessingApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id, document_id, action):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        if action == "pause":
+            if document.indexing_status != "indexing":
+                raise InvalidActionError('Document not in indexing state.')
+
+            document.paused_by = current_user.id
+            document.paused_at = datetime.utcnow()
+            document.is_paused = True
+            db.session.commit()
+
+        elif action == "resume":
+            if document.indexing_status not in ["paused", "error"]:
+                raise InvalidActionError('Document not in paused or error state.')
+
+            document.paused_by = None
+            document.paused_at = None
+            document.is_paused = False
+            db.session.commit()
+        else:
+            raise InvalidActionError()
+
+        return {'result': 'success'}, 200
+
+
+class DocumentDeleteApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        try:
+            DocumentService.delete_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError('Cannot delete document during indexing.')
+
+        return {'result': 'success'}, 204
+
+
+class DocumentMetadataApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def put(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        req_data = request.get_json()
+
+        doc_type = req_data.get('doc_type')
+        doc_metadata = req_data.get('doc_metadata')
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        if doc_type is None or doc_metadata is None:
+            raise ValueError('Both doc_type and doc_metadata must be provided.')
+
+        if doc_type not in DocumentService.DOCUMENT_METADATA_SCHEMA:
+            raise ValueError('Invalid doc_type.')
+
+        if not isinstance(doc_metadata, dict):
+            raise ValueError('doc_metadata must be a dictionary.')
+
+        metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[doc_type]
+
+        document.doc_metadata = {}
+
+        for key, value_type in metadata_schema.items():
+            value = doc_metadata.get(key)
+            if value is not None and isinstance(value, value_type):
+                document.doc_metadata[key] = value
+
+        document.doc_type = doc_type
+        document.updated_at = datetime.utcnow()
+        db.session.commit()
+
+        return {'result': 'success', 'message': 'Document metadata updated.'}, 200
+
+
+class DocumentStatusApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id, document_id, action):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        document = self.get_document(dataset_id, document_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        indexing_cache_key = 'document_{}_indexing'.format(document.id)
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Document is being indexed, please try again later")
+
+        if action == "enable":
+            if document.enabled:
+                raise InvalidActionError('Document already enabled.')
+
+            document.enabled = True
+            document.disabled_at = None
+            document.disabled_by = None
+            document.updated_at = datetime.utcnow()
+            db.session.commit()
+
+            # Set cache to prevent indexing the same document multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            add_document_to_index_task.delay(document_id)
+
+            return {'result': 'success'}, 200
+
+        elif action == "disable":
+            if not document.enabled:
+                raise InvalidActionError('Document already disabled.')
+
+            document.enabled = False
+            document.disabled_at = datetime.utcnow()
+            document.disabled_by = current_user.id
+            document.updated_at = datetime.utcnow()
+            db.session.commit()
+
+            # Set cache to prevent indexing the same document multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            remove_document_from_index_task.delay(document_id)
+
+            return {'result': 'success'}, 200
+
+        elif action == "archive":
+            if document.archived:
+                raise InvalidActionError('Document already archived.')
+
+            document.archived = True
+            document.archived_at = datetime.utcnow()
+            document.archived_by = current_user.id
+            document.updated_at = datetime.utcnow()
+            db.session.commit()
+
+            if document.enabled:
+                # Set cache to prevent indexing the same document multiple times
+                redis_client.setex(indexing_cache_key, 600, 1)
+
+                remove_document_from_index_task.delay(document_id)
+
+            return {'result': 'success'}, 200
+        else:
+            raise InvalidActionError()
+
+
+class DocumentPauseApi(DocumentResource):
+    def patch(self, dataset_id, document_id):
+        """pause document."""
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        document = DocumentService.get_document(dataset.id, document_id)
+
+        # 404 if document not found
+        if document is None:
+            raise NotFound("Document Not Exists.")
+
+        # 403 if document is archived
+        if DocumentService.check_archived(document):
+            raise ArchivedDocumentImmutableError()
+
+        try:
+            # pause document
+            DocumentService.pause_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError('Cannot pause completed document.')
+
+        return {'result': 'success'}, 204
+
+
+class DocumentRecoverApi(DocumentResource):
+    def patch(self, dataset_id, document_id):
+        """recover document."""
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+        document = DocumentService.get_document(dataset.id, document_id)
+
+        # 404 if document not found
+        if document is None:
+            raise NotFound("Document Not Exists.")
+
+        # 403 if document is archived
+        if DocumentService.check_archived(document):
+            raise ArchivedDocumentImmutableError()
+        try:
+            # pause document
+            DocumentService.recover_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError('Document is not in paused status.')
+
+        return {'result': 'success'}, 204
+
+
+api.add_resource(GetProcessRuleApi, '/datasets/process-rule')
+api.add_resource(DatasetDocumentListApi,
+                 '/datasets/<uuid:dataset_id>/documents')
+api.add_resource(DatasetInitApi,
+                 '/datasets/init')
+api.add_resource(DocumentIndexingEstimateApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-estimate')
+api.add_resource(DocumentIndexingStatusApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-status')
+api.add_resource(DocumentDetailApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>')
+api.add_resource(DocumentProcessingApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/<string:action>')
+api.add_resource(DocumentDeleteApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>')
+api.add_resource(DocumentMetadataApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata')
+api.add_resource(DocumentStatusApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/status/<string:action>')
+api.add_resource(DocumentPauseApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause')
+api.add_resource(DocumentRecoverApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/resume')

api/controllers/console/datasets/datasets_segments.py

@@ -0,0 +1,203 @@
+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal
+from werkzeug.exceptions import NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import InvalidActionError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.dataset import DocumentSegment
+
+from libs.helper import TimestampField
+from services.dataset_service import DatasetService, DocumentService
+from tasks.add_segment_to_index_task import add_segment_to_index_task
+from tasks.remove_segment_from_index_task import remove_segment_from_index_task
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField
+}
+
+segment_list_response = {
+    'data': fields.List(fields.Nested(segment_fields)),
+    'has_more': fields.Boolean,
+    'limit': fields.Integer
+}
+
+
+class DatasetDocumentSegmentListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        document = DocumentService.get_document(dataset_id, document_id)
+
+        if not document:
+            raise NotFound('Document not found.')
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('last_id', type=str, default=None, location='args')
+        parser.add_argument('limit', type=int, default=20, location='args')
+        parser.add_argument('status', type=str,
+                            action='append', default=[], location='args')
+        parser.add_argument('hit_count_gte', type=int,
+                            default=None, location='args')
+        parser.add_argument('enabled', type=str, default='all', location='args')
+        args = parser.parse_args()
+
+        last_id = args['last_id']
+        limit = min(args['limit'], 100)
+        status_list = args['status']
+        hit_count_gte = args['hit_count_gte']
+
+        query = DocumentSegment.query.filter(
+            DocumentSegment.document_id == str(document_id),
+            DocumentSegment.tenant_id == current_user.current_tenant_id
+        )
+
+        if last_id is not None:
+            last_segment = DocumentSegment.query.get(str(last_id))
+            if last_segment:
+                query = query.filter(
+                    DocumentSegment.position > last_segment.position)
+            else:
+                return {'data': [], 'has_more': False, 'limit': limit}, 200
+
+        if status_list:
+            query = query.filter(DocumentSegment.status.in_(status_list))
+
+        if hit_count_gte is not None:
+            query = query.filter(DocumentSegment.hit_count >= hit_count_gte)
+
+        if args['enabled'].lower() != 'all':
+            if args['enabled'].lower() == 'true':
+                query = query.filter(DocumentSegment.enabled == True)
+            elif args['enabled'].lower() == 'false':
+                query = query.filter(DocumentSegment.enabled == False)
+
+        total = query.count()
+        segments = query.order_by(DocumentSegment.position).limit(limit + 1).all()
+
+        has_more = False
+        if len(segments) > limit:
+            has_more = True
+            segments = segments[:-1]
+
+        return {
+            'data': marshal(segments, segment_fields),
+            'has_more': has_more,
+            'limit': limit,
+            'total': total
+        }, 200
+
+
+class DatasetDocumentSegmentApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, dataset_id, segment_id, action):
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        segment = DocumentSegment.query.filter(
+            DocumentSegment.id == str(segment_id),
+            DocumentSegment.tenant_id == current_user.current_tenant_id
+        ).first()
+
+        if not segment:
+            raise NotFound('Segment not found.')
+
+        document_indexing_cache_key = 'document_{}_indexing'.format(segment.document_id)
+        cache_result = redis_client.get(document_indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Document is being indexed, please try again later")
+
+        indexing_cache_key = 'segment_{}_indexing'.format(segment.id)
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Segment is being indexed, please try again later")
+
+        if action == "enable":
+            if segment.enabled:
+                raise InvalidActionError("Segment is already enabled.")
+
+            segment.enabled = True
+            segment.disabled_at = None
+            segment.disabled_by = None
+            db.session.commit()
+
+            # Set cache to prevent indexing the same segment multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            add_segment_to_index_task.delay(segment.id)
+
+            return {'result': 'success'}, 200
+        elif action == "disable":
+            if not segment.enabled:
+                raise InvalidActionError("Segment is already disabled.")
+
+            segment.enabled = False
+            segment.disabled_at = datetime.utcnow()
+            segment.disabled_by = current_user.id
+            db.session.commit()
+
+            # Set cache to prevent indexing the same segment multiple times
+            redis_client.setex(indexing_cache_key, 600, 1)
+
+            remove_segment_from_index_task.delay(segment.id)
+
+            return {'result': 'success'}, 200
+        else:
+            raise InvalidActionError()
+
+
+api.add_resource(DatasetDocumentSegmentListApi,
+                 '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments')
+api.add_resource(DatasetDocumentSegmentApi,
+                 '/datasets/<uuid:dataset_id>/segments/<uuid:segment_id>/<string:action>')

api/controllers/console/datasets/error.py

@@ -0,0 +1,73 @@
+from libs.exception import BaseHTTPException
+
+
+class NoFileUploadedError(BaseHTTPException):
+    error_code = 'no_file_uploaded'
+    description = "No file uploaded."
+    code = 400
+
+
+class TooManyFilesError(BaseHTTPException):
+    error_code = 'too_many_files'
+    description = "Only one file is allowed."
+    code = 400
+
+
+class FileTooLargeError(BaseHTTPException):
+    error_code = 'file_too_large'
+    description = "File size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = 'unsupported_file_type'
+    description = "File type not allowed."
+    code = 415
+
+
+class HighQualityDatasetOnlyError(BaseHTTPException):
+    error_code = 'high_quality_dataset_only'
+    description = "High quality dataset only."
+    code = 400
+
+
+class DatasetNotInitializedError(BaseHTTPException):
+    error_code = 'dataset_not_initialized'
+    description = "Dataset not initialized."
+    code = 400
+
+
+class ArchivedDocumentImmutableError(BaseHTTPException):
+    error_code = 'archived_document_immutable'
+    description = "Cannot process an archived document."
+    code = 403
+
+
+class DatasetNameDuplicateError(BaseHTTPException):
+    error_code = 'dataset_name_duplicate'
+    description = "Dataset name already exists."
+    code = 409
+
+
+class InvalidActionError(BaseHTTPException):
+    error_code = 'invalid_action'
+    description = "Invalid action."
+    code = 400
+
+
+class DocumentAlreadyFinishedError(BaseHTTPException):
+    error_code = 'document_already_finished'
+    description = "Document already finished."
+    code = 400
+
+
+class DocumentIndexingError(BaseHTTPException):
+    error_code = 'document_indexing'
+    description = "Document indexing."
+    code = 400
+
+
+class InvalidMetadataError(BaseHTTPException):
+    error_code = 'invalid_metadata'
+    description = "Invalid metadata."
+    code = 400

api/controllers/console/datasets/file.py

@@ -0,0 +1,147 @@
+import datetime
+import hashlib
+import tempfile
+import time
+import uuid
+from pathlib import Path
+
+from cachetools import TTLCache
+from flask import request, current_app
+from flask_login import login_required, current_user
+from flask_restful import Resource, marshal_with, fields
+from werkzeug.exceptions import NotFound
+
+from controllers.console import api
+from controllers.console.datasets.error import NoFileUploadedError, TooManyFilesError, FileTooLargeError, \
+    UnsupportedFileTypeError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.index.readers.html_parser import HTMLParser
+from core.index.readers.pdf_parser import PDFParser
+from extensions.ext_storage import storage
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.model import UploadFile
+
+cache = TTLCache(maxsize=None, ttl=30)
+
+FILE_SIZE_LIMIT = 15 * 1024 * 1024  # 15MB
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm']
+PREVIEW_WORDS_LIMIT = 3000
+
+
+class FileApi(Resource):
+    file_fields = {
+        'id': fields.String,
+        'name': fields.String,
+        'size': fields.Integer,
+        'extension': fields.String,
+        'mime_type': fields.String,
+        'created_by': fields.String,
+        'created_at': TimestampField,
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(file_fields)
+    def post(self):
+
+        # get file from request
+        file = request.files['file']
+
+        # check file
+        if 'file' not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+
+        file_content = file.read()
+        file_size = len(file_content)
+
+        if file_size > FILE_SIZE_LIMIT:
+            message = "({file_size} > {FILE_SIZE_LIMIT})"
+            raise FileTooLargeError(message)
+
+        extension = file.filename.split('.')[-1]
+        if extension not in ALLOWED_EXTENSIONS:
+            raise UnsupportedFileTypeError()
+
+        # user uuid as file name
+        file_uuid = str(uuid.uuid4())
+        file_key = 'upload_files/' + current_user.current_tenant_id + '/' + file_uuid + '.' + extension
+
+        # save file to storage
+        storage.save(file_key, file_content)
+
+        # save file to db
+        config = current_app.config
+        upload_file = UploadFile(
+            tenant_id=current_user.current_tenant_id,
+            storage_type=config['STORAGE_TYPE'],
+            key=file_key,
+            name=file.filename,
+            size=file_size,
+            extension=extension,
+            mime_type=file.mimetype,
+            created_by=current_user.id,
+            created_at=datetime.datetime.utcnow(),
+            used=False,
+            hash=hashlib.sha3_256(file_content).hexdigest()
+        )
+
+        db.session.add(upload_file)
+        db.session.commit()
+
+        return upload_file, 201
+
+
+class FilePreviewApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, file_id):
+        file_id = str(file_id)
+
+        key = file_id + request.path
+        cached_response = cache.get(key)
+        if cached_response and time.time() - cached_response['timestamp'] < cache.ttl:
+            return cached_response['response']
+
+        upload_file = db.session.query(UploadFile) \
+            .filter(UploadFile.id == file_id) \
+            .first()
+
+        if not upload_file:
+            raise NotFound("File not found")
+
+        # extract text from file
+        extension = upload_file.extension
+        if extension not in ALLOWED_EXTENSIONS:
+            raise UnsupportedFileTypeError()
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            suffix = Path(upload_file.key).suffix
+            filepath = f"{temp_dir}/{next(tempfile._get_candidate_names())}{suffix}"
+            storage.download(upload_file.key, filepath)
+
+            if extension == 'pdf':
+                parser = PDFParser({'upload_file': upload_file})
+                text = parser.parse_file(Path(filepath))
+            elif extension in ['html', 'htm']:
+                # Use BeautifulSoup to extract text
+                parser = HTMLParser()
+                text = parser.parse_file(Path(filepath))
+            else:
+                # ['txt', 'markdown', 'md']
+                with open(filepath, "rb") as fp:
+                    data = fp.read()
+                    text = data.decode(encoding='utf-8').strip() if data else ''
+
+        text = text[0:PREVIEW_WORDS_LIMIT] if text else ''
+        return {'content': text}
+
+
+api.add_resource(FileApi, '/files/upload')
+api.add_resource(FilePreviewApi, '/files/<uuid:file_id>/preview')

api/controllers/console/datasets/hit_testing.py

@@ -0,0 +1,100 @@
+import logging
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, marshal, fields
+from werkzeug.exceptions import InternalServerError, NotFound, Forbidden
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import HighQualityDatasetOnlyError, DatasetNotInitializedError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import TimestampField
+from services.dataset_service import DatasetService
+from services.hit_testing_service import HitTestingService
+
+document_fields = {
+    'id': fields.String,
+    'data_source_type': fields.String,
+    'name': fields.String,
+    'doc_type': fields.String,
+}
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField,
+    'document': fields.Nested(document_fields),
+}
+
+hit_testing_record_fields = {
+    'segment': fields.Nested(segment_fields),
+    'score': fields.Float,
+    'tsne_position': fields.Raw
+}
+
+
+class HitTestingApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        # only high quality dataset can be used for hit testing
+        if dataset.indexing_technique != 'high_quality':
+            raise HighQualityDatasetOnlyError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('query', type=str, location='json')
+        args = parser.parse_args()
+
+        query = args['query']
+
+        if not query or len(query) > 250:
+            raise ValueError('Query is required and cannot exceed 250 characters')
+
+        try:
+            response = HitTestingService.retrieve(
+                dataset=dataset,
+                query=query,
+                account=current_user,
+                limit=10,
+            )
+
+            return {"query": response['query'], 'records': marshal(response['records'], hit_testing_record_fields)}
+        except services.errors.index.IndexNotInitializedError:
+            raise DatasetNotInitializedError()
+        except Exception as e:
+            logging.exception("Hit testing failed.")
+            raise InternalServerError(str(e))
+
+
+api.add_resource(HitTestingApi, '/datasets/<uuid:dataset_id>/hit-testing')

api/controllers/console/error.py

@@ -0,0 +1,19 @@
+from libs.exception import BaseHTTPException
+
+
+class AlreadySetupError(BaseHTTPException):
+    error_code = 'already_setup'
+    description = "Application already setup."
+    code = 403
+
+
+class NotSetupError(BaseHTTPException):
+    error_code = 'not_setup'
+    description = "Application not setup."
+    code = 401
+
+
+class AccountNotLinkTenantError(BaseHTTPException):
+    error_code = 'account_not_link_tenant'
+    description = "Account not link tenant."
+    code = 403

api/controllers/console/setup.py

@@ -0,0 +1,93 @@
+# -*- coding:utf-8 -*-
+from functools import wraps
+
+import flask_login
+from flask import request, current_app
+from flask_restful import Resource, reqparse
+
+from extensions.ext_database import db
+from models.model import DifySetup
+from services.account_service import AccountService, TenantService, RegisterService
+
+from libs.helper import email, str_len
+from libs.password import valid_password
+
+from . import api
+from .error import AlreadySetupError, NotSetupError
+from .wraps import only_edition_self_hosted
+
+
+class SetupApi(Resource):
+
+    @only_edition_self_hosted
+    def get(self):
+        setup_status = get_setup_status()
+        if setup_status:
+            return {
+                'step': 'finished',
+                'setup_at': setup_status.setup_at.isoformat()
+            }  
+        return {'step': 'not_start'}
+
+    @only_edition_self_hosted
+    def post(self):
+        # is set up
+        if get_setup_status():
+            raise AlreadySetupError()
+
+        # is tenant created
+        tenant_count = TenantService.get_tenant_count()
+        if tenant_count > 0:
+            raise AlreadySetupError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=email,
+                            required=True, location='json')
+        parser.add_argument('name', type=str_len(
+            30), required=True, location='json')
+        parser.add_argument('password', type=valid_password,
+                            required=True, location='json')
+        args = parser.parse_args()
+
+        # Register
+        account = RegisterService.register(
+            email=args['email'],
+            name=args['name'],
+            password=args['password']
+        )
+
+        setup()
+
+        # Login
+        flask_login.login_user(account)
+        AccountService.update_last_login(account, request)
+
+        return {'result': 'success'}, 201
+
+
+def setup():
+    dify_setup = DifySetup(
+        version=current_app.config['CURRENT_VERSION']
+    )
+    db.session.add(dify_setup)
+
+
+def setup_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        # check setup
+        if not get_setup_status():
+            raise NotSetupError()
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
+def get_setup_status():
+    if current_app.config['EDITION'] == 'SELF_HOSTED':
+        return DifySetup.query.first()
+    else:
+        return True
+
+api.add_resource(SetupApi, '/setup')

api/controllers/console/version.py

@@ -0,0 +1,39 @@
+# -*- coding:utf-8 -*-
+
+import json
+import logging
+
+import requests
+from flask import current_app
+from flask_restful import reqparse, Resource
+from werkzeug.exceptions import InternalServerError
+
+from . import api
+
+
+class VersionApi(Resource):
+
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('current_version', type=str, required=True, location='args')
+        args = parser.parse_args()
+        check_update_url = current_app.config['CHECK_UPDATE_URL']
+
+        try:
+            response = requests.get(check_update_url, {
+                'current_version': args.get('current_version')
+            })
+        except Exception as error:
+            logging.exception("Check update error.")
+            raise InternalServerError()
+
+        content = json.loads(response.content)
+        return {
+            'version': content['version'],
+            'release_date': content['releaseDate'],
+            'release_notes': content['releaseNotes'],
+            'can_auto_update': content['canAutoUpdate']
+        }
+
+
+api.add_resource(VersionApi, '/version')

api/controllers/console/workspace/account.py

@@ -0,0 +1,263 @@
+# -*- coding:utf-8 -*-
+from datetime import datetime
+
+import pytz
+from flask import current_app, request
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, fields, marshal_with
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.workspace.error import AccountAlreadyInitedError, InvalidInvitationCodeError, \
+    RepeatPasswordNotMatchError
+from controllers.console.wraps import account_initialization_required
+from libs.helper import TimestampField, supported_language, timezone
+from extensions.ext_database import db
+from models.account import InvitationCode, AccountIntegrate
+from services.account_service import AccountService
+
+
+account_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'avatar': fields.String,
+    'email': fields.String,
+    'interface_language': fields.String,
+    'interface_theme': fields.String,
+    'timezone': fields.String,
+    'last_login_at': TimestampField,
+    'last_login_ip': fields.String,
+    'created_at': TimestampField
+}
+
+
+class AccountInitApi(Resource):
+
+    @setup_required
+    @login_required
+    def post(self):
+        account = current_user
+
+        if account.status == 'active':
+            raise AccountAlreadyInitedError()
+
+        parser = reqparse.RequestParser()
+
+        if current_app.config['EDITION'] == 'CLOUD':
+            parser.add_argument('invitation_code', type=str, location='json')
+
+        parser.add_argument(
+            'interface_language', type=supported_language, required=True, location='json')
+        parser.add_argument('timezone', type=timezone,
+                            required=True, location='json')
+        args = parser.parse_args()
+
+        if current_app.config['EDITION'] == 'CLOUD':
+            if not args['invitation_code']:
+                raise ValueError('invitation_code is required')
+
+            # check invitation code
+            invitation_code = db.session.query(InvitationCode).filter(
+                InvitationCode.code == args['invitation_code'],
+                InvitationCode.status == 'unused',
+            ).first()
+
+            if not invitation_code:
+                raise InvalidInvitationCodeError()
+
+            invitation_code.status = 'used'
+            invitation_code.used_at = datetime.utcnow()
+            invitation_code.used_by_tenant_id = account.current_tenant_id
+            invitation_code.used_by_account_id = account.id
+
+        account.interface_language = args['interface_language']
+        account.timezone = args['timezone']
+        account.interface_theme = 'light'
+        account.status = 'active'
+        account.initialized_at = datetime.utcnow()
+        db.session.commit()
+
+        return {'result': 'success'}
+
+
+class AccountProfileApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def get(self):
+        return current_user
+
+
+class AccountNameApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        # Validate account name length
+        if len(args['name']) < 3 or len(args['name']) > 30:
+            raise ValueError(
+                "Account name must be between 3 and 30 characters.")
+
+        updated_account = AccountService.update_account(current_user, name=args['name'])
+
+        return updated_account
+
+
+class AccountAvatarApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('avatar', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        updated_account = AccountService.update_account(current_user, avatar=args['avatar'])
+
+        return updated_account
+
+
+class AccountInterfaceLanguageApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            'interface_language', type=supported_language, required=True, location='json')
+        args = parser.parse_args()
+
+        updated_account = AccountService.update_account(current_user, interface_language=args['interface_language'])
+
+        return updated_account
+
+
+class AccountInterfaceThemeApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('interface_theme', type=str, choices=[
+            'light', 'dark'], required=True, location='json')
+        args = parser.parse_args()
+
+        updated_account = AccountService.update_account(current_user, interface_theme=args['interface_theme'])
+
+        return updated_account
+
+
+class AccountTimezoneApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('timezone', type=str,
+                            required=True, location='json')
+        args = parser.parse_args()
+
+        # Validate timezone string, e.g. America/New_York, Asia/Shanghai
+        if args['timezone'] not in pytz.all_timezones:
+            raise ValueError("Invalid timezone string.")
+
+        updated_account = AccountService.update_account(current_user, timezone=args['timezone'])
+
+        return updated_account
+
+
+class AccountPasswordApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('password', type=str,
+                            required=False, location='json')
+        parser.add_argument('new_password', type=str,
+                            required=True, location='json')
+        parser.add_argument('repeat_new_password', type=str,
+                            required=True, location='json')
+        args = parser.parse_args()
+
+        if args['new_password'] != args['repeat_new_password']:
+            raise RepeatPasswordNotMatchError()
+
+        AccountService.update_account_password(
+            current_user, args['password'], args['new_password'])
+
+        return {"result": "success"}
+
+
+class AccountIntegrateApi(Resource):
+    integrate_fields = {
+        'provider': fields.String,
+        'created_at': TimestampField,
+        'is_bound': fields.Boolean,
+        'link': fields.String
+    }
+
+    integrate_list_fields = {
+        'data': fields.List(fields.Nested(integrate_fields)),
+    }
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(integrate_list_fields)
+    def get(self):
+        account = current_user
+
+        account_integrates = db.session.query(AccountIntegrate).filter(
+            AccountIntegrate.account_id == account.id).all()
+
+        base_url = request.url_root.rstrip('/')
+        oauth_base_path = "/console/api/oauth/login"
+        providers = ["github", "google"]
+
+        integrate_data = []
+        for provider in providers:
+            existing_integrate = next((ai for ai in account_integrates if ai.provider == provider), None)
+            if existing_integrate:
+                integrate_data.append({
+                    'id': existing_integrate.id,
+                    'provider': provider,
+                    'created_at': existing_integrate.created_at,
+                    'is_bound': True,
+                    'link': None
+                })
+            else:
+                integrate_data.append({
+                    'id': None,
+                    'provider': provider,
+                    'created_at': None,
+                    'is_bound': False,
+                    'link': f'{base_url}{oauth_base_path}/{provider}'
+                })
+
+        return {'data': integrate_data}
+
+
+# Register API resources
+api.add_resource(AccountInitApi, '/account/init')
+api.add_resource(AccountProfileApi, '/account/profile')
+api.add_resource(AccountNameApi, '/account/name')
+api.add_resource(AccountAvatarApi, '/account/avatar')
+api.add_resource(AccountInterfaceLanguageApi, '/account/interface-language')
+api.add_resource(AccountInterfaceThemeApi, '/account/interface-theme')
+api.add_resource(AccountTimezoneApi, '/account/timezone')
+api.add_resource(AccountPasswordApi, '/account/password')
+api.add_resource(AccountIntegrateApi, '/account/integrates')
+# api.add_resource(AccountEmailApi, '/account/email')
+# api.add_resource(AccountEmailVerifyApi, '/account/email-verify')

api/controllers/console/workspace/error.py

@@ -0,0 +1,31 @@
+from libs.exception import BaseHTTPException
+
+
+class RepeatPasswordNotMatchError(BaseHTTPException):
+    error_code = 'repeat_password_not_match'
+    description = "New password and repeat password does not match."
+    code = 400
+
+
+class ProviderRequestFailedError(BaseHTTPException):
+    error_code = 'provider_request_failed'
+    description = None
+    code = 400
+
+
+class InvalidInvitationCodeError(BaseHTTPException):
+    error_code = 'invalid_invitation_code'
+    description = "Invalid invitation code."
+    code = 400
+
+
+class AccountAlreadyInitedError(BaseHTTPException):
+    error_code = 'account_already_inited'
+    description = "Account already inited."
+    code = 400
+
+
+class AccountNotInitializedError(BaseHTTPException):
+    error_code = 'account_not_initialized'
+    description = "Account not initialized."
+    code = 400

api/controllers/console/workspace/members.py

@@ -0,0 +1,141 @@
+# -*- coding:utf-8 -*-
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, marshal_with, abort, fields, marshal
+
+import services
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.account import Account, TenantAccountJoin
+from services.account_service import TenantService, RegisterService
+
+account_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'avatar': fields.String,
+    'email': fields.String,
+    'last_login_at': TimestampField,
+    'created_at': TimestampField,
+    'role': fields.String,
+    'status': fields.String,
+}
+
+account_list_fields = {
+    'accounts': fields.List(fields.Nested(account_fields))
+}
+
+
+class MemberListApi(Resource):
+    """List all members of current tenant."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_list_fields)
+    def get(self):
+        members = TenantService.get_tenant_members(current_user.current_tenant)
+        return {'result': 'success', 'accounts': members}, 200
+
+
+class MemberInviteEmailApi(Resource):
+    """Invite a new member by email."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=str, required=True, location='json')
+        parser.add_argument('role', type=str, required=True, default='admin', location='json')
+        args = parser.parse_args()
+
+        invitee_email = args['email']
+        invitee_role = args['role']
+        if invitee_role not in ['admin', 'normal']:
+            return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
+
+        inviter = current_user
+
+        try:
+            RegisterService.invite_new_member(inviter.current_tenant, invitee_email, role=invitee_role, inviter=inviter)
+            account = db.session.query(Account, TenantAccountJoin.role).join(
+                TenantAccountJoin, Account.id == TenantAccountJoin.account_id
+            ).filter(Account.email == args['email']).first()
+            account, role = account
+            account = marshal(account, account_fields)
+            account['role'] = role
+        except services.errors.account.CannotOperateSelfError as e:
+            return {'code': 'cannot-operate-self', 'message': str(e)}, 400
+        except services.errors.account.NoPermissionError as e:
+            return {'code': 'forbidden', 'message': str(e)}, 403
+        except services.errors.account.AccountAlreadyInTenantError as e:
+            return {'code': 'email-taken', 'message': str(e)}, 409
+        except Exception as e:
+            return {'code': 'unexpected-error', 'message': str(e)}, 500
+
+        # todo:413
+
+        return {'result': 'success', 'account': account}, 201
+
+
+class MemberCancelInviteApi(Resource):
+    """Cancel an invitation by member id."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, member_id):
+        member = Account.query.get(str(member_id))
+        if not member:
+            abort(404)
+
+        try:
+            TenantService.remove_member_from_tenant(current_user.current_tenant, member, current_user)
+        except services.errors.account.CannotOperateSelfError as e:
+            return {'code': 'cannot-operate-self', 'message': str(e)}, 400
+        except services.errors.account.NoPermissionError as e:
+            return {'code': 'forbidden', 'message': str(e)}, 403
+        except services.errors.account.MemberNotInTenantError as e:
+            return {'code': 'member-not-found', 'message': str(e)}, 404
+        except Exception as e:
+            raise ValueError(str(e))
+
+        return {'result': 'success'}, 204
+
+
+class MemberUpdateRoleApi(Resource):
+    """Update member role."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def put(self, member_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('role', type=str, required=True, location='json')
+        args = parser.parse_args()
+        new_role = args['role']
+
+        if new_role not in ['admin', 'normal', 'owner']:
+            return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
+
+        member = Account.query.get(str(member_id))
+        if not member:
+            abort(404)
+
+        try:
+            TenantService.update_member_role(current_user.current_tenant, member, new_role, current_user)
+        except Exception as e:
+            raise ValueError(str(e))
+
+        # todo: 403
+
+        return {'result': 'success'}
+
+
+api.add_resource(MemberListApi, '/workspaces/current/members')
+api.add_resource(MemberInviteEmailApi, '/workspaces/current/members/invite-email')
+api.add_resource(MemberCancelInviteApi, '/workspaces/current/members/<uuid:member_id>')
+api.add_resource(MemberUpdateRoleApi, '/workspaces/current/members/<uuid:member_id>/update-role')

api/controllers/console/workspace/providers.py

@@ -0,0 +1,246 @@
+# -*- coding:utf-8 -*-
+import base64
+import json
+import logging
+
+from flask_login import login_required, current_user
+from flask_restful import Resource, reqparse, abort
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.llm.provider.errors import ValidateFailedError
+from extensions.ext_database import db
+from libs import rsa
+from models.provider import Provider, ProviderType, ProviderName
+from services.provider_service import ProviderService
+
+
+class ProviderListApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+
+        """
+        If the type is AZURE_OPENAI, decode and return the four fields of azure_api_type, azure_api_version:, 
+        azure_api_base, azure_api_key as an object, where azure_api_key displays the first 6 bits in plaintext, and the 
+        rest is replaced by * and the last two bits are displayed in plaintext
+        
+        If the type is other, decode and return the Token field directly, the field displays the first 6 bits in 
+        plaintext, the rest is replaced by * and the last two bits are displayed in plaintext
+        """
+
+        ProviderService.init_supported_provider(current_user.current_tenant, "cloud")
+        providers = Provider.query.filter_by(tenant_id=tenant_id).all()
+
+        provider_list = [
+            {
+                'provider_name': p.provider_name,
+                'provider_type': p.provider_type,
+                'is_valid': p.is_valid,
+                'last_used': p.last_used,
+                'is_enabled': p.is_enabled,
+                **({
+                       'quota_type': p.quota_type,
+                       'quota_limit': p.quota_limit,
+                       'quota_used': p.quota_used
+                   } if p.provider_type == ProviderType.SYSTEM.value else {}),
+                'token': ProviderService.get_obfuscated_api_key(current_user.current_tenant,
+                                                                ProviderName(p.provider_name))
+            }
+            for p in providers
+        ]
+
+        return provider_list
+
+
+class ProviderTokenApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, provider):
+        if provider not in [p.value for p in ProviderName]:
+            abort(404)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            logging.log(logging.ERROR,
+                        f'User {current_user.id} is not authorized to update provider token, current_role is {current_user.current_tenant.current_role}')
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+
+        parser.add_argument('token', type=ProviderService.get_token_type(
+            tenant=current_user.current_tenant,
+            provider_name=ProviderName(provider)
+        ), required=True, nullable=False, location='json')
+
+        args = parser.parse_args()
+
+        if not args['token']:
+            raise ValueError('Token is empty')
+
+        try:
+            ProviderService.validate_provider_configs(
+                tenant=current_user.current_tenant,
+                provider_name=ProviderName(provider),
+                configs=args['token']
+            )
+            token_is_valid = True
+        except ValidateFailedError:
+            token_is_valid = False
+
+        tenant = current_user.current_tenant
+
+        base64_encrypted_token = ProviderService.get_encrypted_token(
+            tenant=current_user.current_tenant,
+            provider_name=ProviderName(provider),
+            configs=args['token']
+        )
+
+        provider_model = Provider.query.filter_by(tenant_id=tenant.id, provider_name=provider,
+                                                  provider_type=ProviderType.CUSTOM.value).first()
+
+        # Only allow updating token for CUSTOM provider type
+        if provider_model:
+            provider_model.encrypted_config = base64_encrypted_token
+            provider_model.is_valid = token_is_valid
+        else:
+            provider_model = Provider(tenant_id=tenant.id, provider_name=provider,
+                                      provider_type=ProviderType.CUSTOM.value,
+                                      encrypted_config=base64_encrypted_token,
+                                      is_valid=token_is_valid)
+            db.session.add(provider_model)
+
+        db.session.commit()
+
+        if provider in [ProviderName.ANTHROPIC.value, ProviderName.AZURE_OPENAI.value, ProviderName.COHERE.value,
+                        ProviderName.HUGGINGFACEHUB.value]:
+            return {'result': 'success', 'warning': 'MOCK: This provider is not supported yet.'}, 201
+
+        return {'result': 'success'}, 201
+
+
+class ProviderTokenValidateApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, provider):
+        if provider not in [p.value for p in ProviderName]:
+            abort(404)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('token', type=ProviderService.get_token_type(
+            tenant=current_user.current_tenant,
+            provider_name=ProviderName(provider)
+        ), required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        # todo: remove this when the provider is supported
+        if provider in [ProviderName.ANTHROPIC.value, ProviderName.AZURE_OPENAI.value, ProviderName.COHERE.value,
+                        ProviderName.HUGGINGFACEHUB.value]:
+            return {'result': 'success', 'warning': 'MOCK: This provider is not supported yet.'}
+
+        result = True
+        error = None
+
+        try:
+            ProviderService.validate_provider_configs(
+                tenant=current_user.current_tenant,
+                provider_name=ProviderName(provider),
+                configs=args['token']
+            )
+        except ValidateFailedError as e:
+            result = False
+            error = str(e)
+
+        response = {'result': 'success' if result else 'error'}
+
+        if not result:
+            response['error'] = error
+
+        return response
+
+
+class ProviderSystemApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def put(self, provider):
+        if provider not in [p.value for p in ProviderName]:
+            abort(404)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('is_enabled', type=bool, required=True, location='json')
+        args = parser.parse_args()
+
+        tenant = current_user.current_tenant_id
+
+        provider_model = Provider.query.filter_by(tenant_id=tenant.id, provider_name=provider).first()
+
+        if provider_model and provider_model.provider_type == ProviderType.SYSTEM.value:
+            provider_model.is_valid = args['is_enabled']
+            db.session.commit()
+        elif not provider_model:
+            ProviderService.create_system_provider(tenant, provider, args['is_enabled'])
+        else:
+            abort(403)
+
+        return {'result': 'success'}
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider):
+        if provider not in [p.value for p in ProviderName]:
+            abort(404)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        provider_model = db.session.query(Provider).filter(Provider.tenant_id == current_user.current_tenant_id,
+                                                           Provider.provider_name == provider,
+                                                           Provider.provider_type == ProviderType.SYSTEM.value).first()
+
+        system_model = None
+        if provider_model:
+            system_model = {
+                'result': 'success',
+                'provider': {
+                    'provider_name': provider_model.provider_name,
+                    'provider_type': provider_model.provider_type,
+                    'is_valid': provider_model.is_valid,
+                    'last_used': provider_model.last_used,
+                    'is_enabled': provider_model.is_enabled,
+                    'quota_type': provider_model.quota_type,
+                    'quota_limit': provider_model.quota_limit,
+                    'quota_used': provider_model.quota_used
+                }
+            }
+        else:
+            abort(404)
+
+        return system_model
+
+
+api.add_resource(ProviderTokenApi, '/providers/<provider>/token',
+                 endpoint='current_providers_token')  # Deprecated
+api.add_resource(ProviderTokenValidateApi, '/providers/<provider>/token-validate',
+                 endpoint='current_providers_token_validate')  # Deprecated
+
+api.add_resource(ProviderTokenApi, '/workspaces/current/providers/<provider>/token',
+                 endpoint='workspaces_current_providers_token')  # PUT for updating provider token
+api.add_resource(ProviderTokenValidateApi, '/workspaces/current/providers/<provider>/token-validate',
+                 endpoint='workspaces_current_providers_token_validate')  # POST for validating provider token
+
+api.add_resource(ProviderListApi, '/workspaces/current/providers')  # GET for getting providers list
+api.add_resource(ProviderSystemApi, '/workspaces/current/providers/<provider>/system',
+                 endpoint='workspaces_current_providers_system')  # GET for getting provider quota, PUT for updating provider status

api/controllers/console/workspace/workspace.py

@@ -0,0 +1,97 @@
+# -*- coding:utf-8 -*-
+import logging
+
+from flask import request
+from flask_login import login_required, current_user
+from flask_restful import Resource, fields, marshal_with, reqparse, marshal
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.error import AccountNotLinkTenantError
+from controllers.console.wraps import account_initialization_required
+from libs.helper import TimestampField
+from extensions.ext_database import db
+from models.account import Tenant
+from services.account_service import TenantService
+from services.workspace_service import WorkspaceService
+
+provider_fields = {
+    'provider_name': fields.String,
+    'provider_type': fields.String,
+    'is_valid': fields.Boolean,
+    'token_is_set': fields.Boolean,
+}
+
+tenant_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'plan': fields.String,
+    'status': fields.String,
+    'created_at': TimestampField,
+    'role': fields.String,
+    'providers': fields.List(fields.Nested(provider_fields)),
+    'in_trail': fields.Boolean,
+    'trial_end_reason': fields.String,
+}
+
+tenants_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'plan': fields.String,
+    'status': fields.String,
+    'created_at': TimestampField,
+    'current': fields.Boolean
+}
+
+
+class TenantListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        tenants = TenantService.get_join_tenants(current_user)
+
+        for tenant in tenants:
+            if tenant.id == current_user.current_tenant_id:
+                tenant.current = True  # Set current=True for current tenant
+        return {'workspaces': marshal(tenants, tenants_fields)}, 200
+
+
+class TenantApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(tenant_fields)
+    def get(self):
+        if request.path == '/info':
+            logging.warning('Deprecated URL /info was used.')
+
+        tenant = current_user.current_tenant
+
+        return WorkspaceService.get_tenant_info(tenant), 200
+
+
+class SwitchWorkspaceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('tenant_id', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        # check if tenant_id is valid, 403 if not
+        try:
+            TenantService.switch_tenant(current_user, args['tenant_id'])
+        except Exception:
+            raise AccountNotLinkTenantError("Account not link tenant")
+
+        new_tenant = db.session.query(Tenant).get(args['tenant_id'])  # Get new tenant
+
+        return {'result': 'success', 'new_tenant': marshal(WorkspaceService.get_tenant_info(new_tenant), tenant_fields)}
+
+
+api.add_resource(TenantListApi, '/workspaces')  # GET for getting all tenants
+api.add_resource(TenantApi, '/workspaces/current', endpoint='workspaces_current')  # GET for getting current tenant info
+api.add_resource(TenantApi, '/info', endpoint='info')  # Deprecated
+api.add_resource(SwitchWorkspaceApi, '/workspaces/switch')  # POST for switching tenant

api/controllers/console/wraps.py

@@ -0,0 +1,43 @@
+# -*- coding:utf-8 -*-
+from functools import wraps
+
+from flask import current_app, abort
+from flask_login import current_user
+
+from controllers.console.workspace.error import AccountNotInitializedError
+
+
+def account_initialization_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        # check account initialization
+        account = current_user
+
+        if account.status == 'uninitialized':
+            raise AccountNotInitializedError()
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
+def only_edition_cloud(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        if current_app.config['EDITION'] != 'CLOUD':
+            abort(404)
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
+def only_edition_self_hosted(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        if current_app.config['EDITION'] != 'SELF_HOSTED':
+            abort(404)
+
+        return view(*args, **kwargs)
+
+    return decorated