feat: add datasource support to PluginDeclaration and PluginCategory

api/controllers/console/auth/oauth.py

@@ -2,17 +2,18 @@ import logging
 from datetime import UTC, datetime
 from typing import Optional
 
-from flask_login import current_user
 import requests
 from flask import current_app, redirect, request
+from flask_login import current_user
 from flask_restful import Resource
 from sqlalchemy import select
 from sqlalchemy.orm import Session
-from werkzeug.exceptions import Unauthorized, Forbidden, NotFound
+from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from configs import dify_config
 from constants.languages import languages
 from controllers.console.wraps import account_initialization_required, setup_required
+from core.plugin.impl.oauth import OAuthHandler
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
 from libs.helper import extract_remote_ip
@@ -25,7 +26,7 @@ from services.account_service import AccountService, RegisterService, TenantServ
 from services.errors.account import AccountNotFoundError, AccountRegisterError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError
 from services.feature_service import FeatureService
-from core.plugin.impl.oauth import OAuthHandler
+
 from .. import api
 
 

api/controllers/console/datasets/rag_pipeline/datasource_oauth.py

@@ -1,24 +1,15 @@
-from typing import cast
 
 from flask_login import current_user  # type: ignore
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
+from flask_restful import Resource  # type: ignore
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.datasets.wraps import get_rag_pipeline
 from controllers.console.wraps import (
     account_initialization_required,
     setup_required,
 )
 from core.plugin.impl.datasource import PluginDatasourceManager
-from extensions.ext_database import db
-from fields.rag_pipeline_fields import pipeline_import_check_dependencies_fields, pipeline_import_fields
 from libs.login import login_required
-from models import Account
-from models.dataset import Pipeline
-from services.app_dsl_service import ImportStatus
-from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService
 
 
 class DatasourcePluginOauthApi(Resource):

api/core/app/apps/pipeline/pipeline_generator.py

@@ -30,7 +30,7 @@ from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository
 from core.workflow.repository.workflow_node_execution_repository import WorkflowNodeExecutionRepository
 from extensions.ext_database import db
-from models import Account, App, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom
+from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom
 from models.dataset import Document, Pipeline
 from models.model import AppMode
 from services.dataset_service import DocumentService

api/core/datasource/website_crawl/website_crawl_provider.py

@@ -10,7 +10,11 @@ class WebsiteCrawlDatasourcePluginProviderController(DatasourcePluginProviderCon
     plugin_unique_identifier: str
 
     def __init__(
-        self, entity: DatasourceProviderEntityWithPlugin | None, plugin_id: str, plugin_unique_identifier: str, tenant_id: str
+        self,
+        entity: DatasourceProviderEntityWithPlugin | None,
+        plugin_id: str,
+        plugin_unique_identifier: str,
+        tenant_id: str,
     ) -> None:
         super().__init__(entity, tenant_id)
         self.plugin_id = plugin_id

api/core/plugin/entities/plugin.py

@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, model_validator
 from werkzeug.exceptions import NotFound
 
 from core.agent.plugin_entities import AgentStrategyProviderEntity
+from core.datasource.entities.datasource_entities import DatasourceProviderEntity
 from core.model_runtime.entities.provider_entities import ProviderEntity
 from core.plugin.entities.base import BasePluginEntity
 from core.plugin.entities.endpoint import EndpointProviderDeclaration
@@ -62,6 +63,7 @@ class PluginCategory(enum.StrEnum):
     Model = "model"
     Extension = "extension"
     AgentStrategy = "agent-strategy"
+    Datasource = "datasource"
 
 
 class PluginDeclaration(BaseModel):
@@ -69,6 +71,7 @@ class PluginDeclaration(BaseModel):
         tools: Optional[list[str]] = Field(default_factory=list[str])
         models: Optional[list[str]] = Field(default_factory=list[str])
         endpoints: Optional[list[str]] = Field(default_factory=list[str])
+        datasources: Optional[list[str]] = Field(default_factory=list[str])
 
     class Meta(BaseModel):
         minimum_dify_version: Optional[str] = Field(default=None, pattern=r"^\d{1,4}(\.\d{1,4}){1,3}(-\w{1,16})?$")
@@ -90,6 +93,7 @@ class PluginDeclaration(BaseModel):
     model: Optional[ProviderEntity] = None
     endpoint: Optional[EndpointProviderDeclaration] = None
     agent_strategy: Optional[AgentStrategyProviderEntity] = None
+    datasource: Optional[DatasourceProviderEntity] = None
     meta: Meta
 
     @model_validator(mode="before")
@@ -100,6 +104,8 @@ class PluginDeclaration(BaseModel):
             values["category"] = PluginCategory.Tool
         elif values.get("model"):
             values["category"] = PluginCategory.Model
+        elif values.get("datasource"):
+            values["category"] = PluginCategory.Datasource
         elif values.get("agent_strategy"):
             values["category"] = PluginCategory.AgentStrategy
         else:

api/core/plugin/impl/datasource.py

@@ -236,7 +236,7 @@ class PluginDatasourceManager(BasePluginClient):
 
         response = self._request_with_plugin_daemon_response_stream(
             "GET",
-            f"plugin/datasource/oauth",
+            "plugin/datasource/oauth",
             PluginBasicBooleanResponse,
             params={"page": 1, "page_size": 256},
             headers={

api/core/workflow/nodes/datasource/datasource_node.py

@@ -1,4 +1,4 @@
-from collections.abc import Generator, Mapping, Sequence
+from collections.abc import Mapping, Sequence
 from typing import Any, cast
 
 from core.datasource.entities.datasource_entities import (

api/models/oauth.py

@@ -1,20 +1,11 @@
 
 from datetime import datetime
-from json import JSONDecodeError
-from typing import Any, cast
 
-from sqlalchemy import func
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped
 
-from configs import dify_config
-from extensions.ext_storage import storage
-from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule
-
-from .account import Account
 from .base import Base
 from .engine import db
-from .model import App, Tag, TagBinding, UploadFile
 from .types import StringUUID