diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json index 5287b6a30f8..444f523b6c0 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/reverseIngestionPipeline.json @@ -57,7 +57,7 @@ }, "required": [ "id", - "entity", + "entityLink", "type", "parameters" ], @@ -70,9 +70,9 @@ "$ref": "#/definitions/reverseIngestionType", "default": "ReverseIngestion" }, - "serviceId": { - "description": "The id of the database service to be modified", - "$ref": "../type/basic.json#/definitions/uuid" + "service": { + "description": "Service to be modified", + "$ref": "../type/entityReference.json" }, "operations": { "description": "List of operations to be performed on the service", @@ -84,7 +84,8 @@ }, "additionalProperties": false, "required": [ - "serviceId", + "type", + "service", "operations" ] } \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index 62f53299c03..a5d3ad00b57 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -33,7 +33,7 @@ export interface CreateWorkflow { /** * Owners of this workflow. */ - owners?: EntityReference[]; + owners?: OwnerElement[]; /** * Request body for a specific workflow type */ @@ -65,7 +65,7 @@ export interface CreateWorkflow { * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. */ -export interface EntityReference { +export interface OwnerElement { /** * If true the entity referred to has been soft-deleted. */ @@ -112,6 +112,8 @@ export interface EntityReference { * Request body for a specific workflow type * * Test Service Connection to test user provided configuration is valid or not. + * + * Apply a set of operations on a service */ export interface TestServiceConnectionRequest { /** @@ -134,6 +136,18 @@ export interface TestServiceConnectionRequest { * Type of service such as Database, Dashboard, Messaging, etc. */ serviceType?: ServiceType; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: ServiceClass; + /** + * Pipeline type + */ + type?: ReverseIngestionType; } /** @@ -3554,6 +3568,216 @@ export enum RESTType { Vertica = "Vertica", } +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: OwnerElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: OwnerElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Description for the tag label. + */ + description?: string; + /** + * Display Name that identifies this tag. + */ + displayName?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType: LabelType; + /** + * Name of the tag or glossary term. + */ + name?: string; + /** + * Label is from Tags or Glossary. + */ + source: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + style?: Style; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ +export enum LabelType { + Automated = "Automated", + Derived = "Derived", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Label is from Tags or Glossary. + */ +export enum TagSource { + Classification = "Classification", + Glossary = "Glossary", +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = "Confirmed", + Suggested = "Suggested", +} + +/** + * UI Style is used to associate a color code and/or icon to entity to customize the look of + * that entity in UI. + */ +export interface Style { + /** + * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. + */ + color?: string; + /** + * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. + */ + iconURL?: string; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ServiceClass { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + /** * Type of service such as Database, Dashboard, Messaging, etc. * @@ -3571,11 +3795,22 @@ export enum ServiceType { Storage = "Storage", } +/** + * Pipeline type + * + * Reverse Ingestion Config Pipeline type + */ +export enum ReverseIngestionType { + ReverseIngestion = "ReverseIngestion", +} + /** * Response to the request. * * TestConnectionResult is the definition that will encapsulate result of running the test * connection steps. + * + * Apply a set of operations on a service */ export interface TestConnectionResult { /** @@ -3589,7 +3824,40 @@ export interface TestConnectionResult { /** * Steps to test the connection. Order matters. */ - steps: TestConnectionStepResult[]; + steps?: TestConnectionStepResult[]; + /** + * Error message in case of failure + */ + message?: string; + /** + * List of operations to be performed on the service + */ + results?: ReverseIngestionOperationResult[]; + /** + * The id of the service to be modified + */ + serviceId?: string; + /** + * Whether the workflow was successful. Failure indicates a critical failure such as + * connection issues. + */ + success?: boolean; +} + +export interface ReverseIngestionOperationResult { + /** + * The id of the operation + */ + id: string; + /** + * Error message in case of failure + */ + message?: string; + /** + * Whether the specific operation was successful + */ + success: boolean; + [property: string]: any; } /** @@ -3649,5 +3917,6 @@ export enum WorkflowStatus { * This enum defines the type for which this workflow applies to. */ export enum WorkflowType { + ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index 61c9f1216ab..eb2fa024719 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -30,7 +30,7 @@ export interface CreateIngestionPipeline { /** * The ingestion agent responsible for executing the ingestion pipeline. */ - ingestionAgent?: EntityReference; + ingestionAgent?: IngestionAgentElement; /** * Set the logging level for the workflow. */ @@ -42,12 +42,12 @@ export interface CreateIngestionPipeline { /** * Owner of this Ingestion Pipeline. */ - owners?: EntityReference[]; + owners?: IngestionAgentElement[]; pipelineType: PipelineType; /** * Link to the service for which ingestion pipeline is ingesting the metadata. */ - service: EntityReference; + service: IngestionAgentElement; sourceConfig: SourceConfig; } @@ -132,7 +132,7 @@ export interface AirflowConfig { * * Domain to apply */ -export interface EntityReference { +export interface IngestionAgentElement { /** * If true the entity referred to has been soft-deleted. */ @@ -242,6 +242,8 @@ export interface SourceConfig { * Application Pipeline Configuration. * * ApiService Metadata Pipeline Configuration. + * + * Apply a set of operations on a service */ export interface Pipeline { /** @@ -670,6 +672,14 @@ export interface Pipeline { * like endpoints, etc., with that collection will be deleted */ markDeletedApiCollections?: boolean; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: ConfigService; } /** @@ -728,6 +738,8 @@ export interface FilterPattern { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -767,7 +779,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -822,6 +835,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -915,7 +930,7 @@ export interface Action { /** * Domain to apply */ - domain?: EntityReference; + domain?: IngestionAgentElement; /** * Description to apply */ @@ -933,7 +948,7 @@ export interface Action { /** * Owners to apply */ - owners?: EntityReference[]; + owners?: IngestionAgentElement[]; /** * Propagate the metadata to columns via column-level lineage. */ @@ -1127,6 +1142,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -1166,6 +1258,7 @@ export enum CollateAIAppConfigType { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } @@ -1543,6 +1636,78 @@ export interface LineageInformation { [property: string]: any; } +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: IngestionAgentElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: IngestionAgentElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + /** * Type of Profile Sample (percentage or rows) */ @@ -1559,6 +1724,57 @@ export enum SamplingMethodType { System = "SYSTEM", } +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ConfigService { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + /** * Service connections available for the logical test suite. */ @@ -4857,6 +5073,8 @@ export interface StorageMetadataBucketDetails { * Pipeline Source Config For Application Pipeline type. Nothing is required. * * Api Source Config Metadata Pipeline type + * + * Reverse Ingestion Config Pipeline type */ export enum ConfigType { APIMetadata = "ApiMetadata", @@ -4873,6 +5091,7 @@ export enum ConfigType { MlModelMetadata = "MlModelMetadata", PipelineMetadata = "PipelineMetadata", Profiler = "Profiler", + ReverseIngestion = "ReverseIngestion", SearchMetadata = "SearchMetadata", StorageMetadata = "StorageMetadata", TestSuite = "TestSuite", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts index 80d6121a614..e3028f82aca 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/app.ts @@ -185,6 +185,8 @@ export interface App { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -224,7 +226,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -279,6 +282,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -647,6 +652,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -686,6 +768,7 @@ export enum Type { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts index ad773387b45..a8abf23fbc0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/appMarketPlaceDefinition.ts @@ -170,6 +170,8 @@ export interface AppMarketPlaceDefinition { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -209,7 +211,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -264,6 +267,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -630,6 +635,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -669,6 +751,7 @@ export enum Type { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts index 6437f2b702b..fc25d6047c4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.ts @@ -127,6 +127,8 @@ export interface CreateAppMarketPlaceDefinitionReq { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -166,7 +168,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -221,6 +224,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -584,6 +589,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -623,6 +705,7 @@ export enum Type { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 627a575781e..7c3ed66d85d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -21,7 +21,7 @@ export interface Workflow { /** * List of data products this entity is part of. */ - dataProducts?: EntityReference[]; + dataProducts?: DomainElement[]; /** * When `true` indicates the entity has been soft deleted. */ @@ -38,7 +38,7 @@ export interface Workflow { * Domain the asset belongs to. When not set, the asset inherits the domain from the parent * it belongs to. */ - domain?: EntityReference; + domain?: DomainElement; /** * FullyQualifiedName same as `name`. */ @@ -63,7 +63,7 @@ export interface Workflow { /** * Owners of this workflow. */ - owners?: EntityReference[]; + owners?: DomainElement[]; /** * Request body for a specific workflow type */ @@ -152,7 +152,7 @@ export interface FieldChange { * Domain the asset belongs to. When not set, the asset inherits the domain from the parent * it belongs to. */ -export interface EntityReference { +export interface DomainElement { /** * If true the entity referred to has been soft-deleted. */ @@ -452,6 +452,8 @@ export enum VerifySSL { * Request body for a specific workflow type * * Test Service Connection to test user provided configuration is valid or not. + * + * Apply a set of operations on a service */ export interface TestServiceConnectionRequest { /** @@ -474,6 +476,18 @@ export interface TestServiceConnectionRequest { * Type of service such as Database, Dashboard, Messaging, etc. */ serviceType?: ServiceType; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: ServiceClass; + /** + * Pipeline type + */ + type?: ReverseIngestionType; } /** @@ -3790,6 +3804,216 @@ export enum RESTType { Vertica = "Vertica", } +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: DomainElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: DomainElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Description for the tag label. + */ + description?: string; + /** + * Display Name that identifies this tag. + */ + displayName?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType: LabelType; + /** + * Name of the tag or glossary term. + */ + name?: string; + /** + * Label is from Tags or Glossary. + */ + source: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + style?: Style; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ +export enum LabelType { + Automated = "Automated", + Derived = "Derived", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Label is from Tags or Glossary. + */ +export enum TagSource { + Classification = "Classification", + Glossary = "Glossary", +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = "Confirmed", + Suggested = "Suggested", +} + +/** + * UI Style is used to associate a color code and/or icon to entity to customize the look of + * that entity in UI. + */ +export interface Style { + /** + * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. + */ + color?: string; + /** + * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. + */ + iconURL?: string; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ServiceClass { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + /** * Type of service such as Database, Dashboard, Messaging, etc. * @@ -3807,11 +4031,22 @@ export enum ServiceType { Storage = "Storage", } +/** + * Pipeline type + * + * Reverse Ingestion Config Pipeline type + */ +export enum ReverseIngestionType { + ReverseIngestion = "ReverseIngestion", +} + /** * Response to the request. * * TestConnectionResult is the definition that will encapsulate result of running the test * connection steps. + * + * Apply a set of operations on a service */ export interface TestConnectionResult { /** @@ -3825,7 +4060,40 @@ export interface TestConnectionResult { /** * Steps to test the connection. Order matters. */ - steps: TestConnectionStepResult[]; + steps?: TestConnectionStepResult[]; + /** + * Error message in case of failure + */ + message?: string; + /** + * List of operations to be performed on the service + */ + results?: ReverseIngestionOperationResult[]; + /** + * The id of the service to be modified + */ + serviceId?: string; + /** + * Whether the workflow was successful. Failure indicates a critical failure such as + * connection issues. + */ + success?: boolean; +} + +export interface ReverseIngestionOperationResult { + /** + * The id of the operation + */ + id: string; + /** + * Error message in case of failure + */ + message?: string; + /** + * Whether the specific operation was successful + */ + success: boolean; + [property: string]: any; } /** @@ -3885,5 +4153,6 @@ export enum WorkflowStatus { * This enum defines the type for which this workflow applies to. */ export enum WorkflowType { + ReverseIngestion = "REVERSE_INGESTION", TestConnection = "TEST_CONNECTION", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/dashboard/mstrConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/dashboard/mstrConnection.ts deleted file mode 100644 index 568b914b856..00000000000 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/dashboard/mstrConnection.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2025 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* - * Copyright 2021 Collate - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - /** - * Mstr Connection Config - */ -export interface MstrConnection { - /** - * Host and Port of the Metabase instance. - */ - hostPort: string; - /** - * Password to connect to MSTR. - */ - password?: string; - /** - * MSTR Project Name - */ - projectName?: string; - supportsMetadataExtraction?: boolean; - /** - * Service Type - */ - type?: MstrType; - /** - * Username to connect to MSTR. This user should have privileges to read all the metadata in - * MSTR. - */ - username: string; -} - -/** - * Service Type - * - * Mstr service type - */ -export enum MstrType { - Mstr = "Mstr", -} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index 13b4f7f3dfd..49510f9bf6b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -44,7 +44,7 @@ export interface IngestionPipeline { * Domain the asset belongs to. When not set, the asset inherits the domain from the parent * it belongs to. */ - domain?: EntityReference; + domain?: DomainElement; /** * True if the pipeline is ready to be run in the next schedule. False if it is paused. */ @@ -68,7 +68,7 @@ export interface IngestionPipeline { /** * The ingestion agent responsible for executing the ingestion pipeline. */ - ingestionAgent?: EntityReference; + ingestionAgent?: DomainElement; /** * Set the logging level for the workflow. */ @@ -81,7 +81,7 @@ export interface IngestionPipeline { /** * Owners of this Pipeline. */ - owners?: EntityReference[]; + owners?: DomainElement[]; /** * Last of executions and status for the Pipeline. */ @@ -92,7 +92,7 @@ export interface IngestionPipeline { * Link to the service (such as database, messaging, storage services, etc. for which this * ingestion pipeline ingests the metadata from. */ - service?: EntityReference; + service?: DomainElement; sourceConfig: SourceConfig; /** * Last update time corresponding to the new version of the entity in Unix epoch time @@ -237,7 +237,7 @@ export interface FieldChange { * * Domain to apply */ -export interface EntityReference { +export interface DomainElement { /** * If true the entity referred to has been soft-deleted. */ @@ -711,6 +711,8 @@ export interface SourceConfig { * Application Pipeline Configuration. * * ApiService Metadata Pipeline Configuration. + * + * Apply a set of operations on a service */ export interface Pipeline { /** @@ -1139,6 +1141,14 @@ export interface Pipeline { * like endpoints, etc., with that collection will be deleted */ markDeletedApiCollections?: boolean; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: ConfigService; } /** @@ -1197,6 +1207,8 @@ export interface FilterPattern { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -1236,7 +1248,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -1291,6 +1304,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -1384,7 +1399,7 @@ export interface Action { /** * Domain to apply */ - domain?: EntityReference; + domain?: DomainElement; /** * Description to apply */ @@ -1402,7 +1417,7 @@ export interface Action { /** * Owners to apply */ - owners?: EntityReference[]; + owners?: DomainElement[]; /** * Propagate the metadata to columns via column-level lineage. */ @@ -1596,6 +1611,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -1635,6 +1727,7 @@ export enum CollateAIAppConfigType { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } @@ -2012,6 +2105,78 @@ export interface LineageInformation { [property: string]: any; } +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: DomainElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: DomainElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + /** * Type of Profile Sample (percentage or rows) */ @@ -2028,6 +2193,57 @@ export enum SamplingMethodType { System = "SYSTEM", } +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ConfigService { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + /** * Service connections available for the logical test suite. */ @@ -5224,6 +5440,8 @@ export interface StorageMetadataBucketDetails { * Pipeline Source Config For Application Pipeline type. Nothing is required. * * Api Source Config Metadata Pipeline type + * + * Reverse Ingestion Config Pipeline type */ export enum ConfigType { APIMetadata = "ApiMetadata", @@ -5240,6 +5458,7 @@ export enum ConfigType { MlModelMetadata = "MlModelMetadata", PipelineMetadata = "PipelineMetadata", Profiler = "Profiler", + ReverseIngestion = "ReverseIngestion", SearchMetadata = "SearchMetadata", StorageMetadata = "StorageMetadata", TestSuite = "TestSuite", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts index 26cc8ee9367..6805a06a2d6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/application.ts @@ -56,6 +56,8 @@ export interface Application { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -95,7 +97,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -150,6 +153,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -513,6 +518,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -552,6 +634,7 @@ export enum Type { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts index 102a8d14439..88a6af98799 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/applicationPipeline.ts @@ -45,6 +45,8 @@ export interface ApplicationPipeline { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -84,7 +86,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -139,6 +142,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -502,6 +507,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -541,6 +623,7 @@ export enum Type { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseIngestionPipeline.ts new file mode 100644 index 00000000000..aa999565797 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/reverseIngestionPipeline.ts @@ -0,0 +1,304 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Apply a set of operations on a service + */ +export interface ReverseIngestionPipeline { + /** + * List of operations to be performed on the service + */ + operations: Operation[]; + /** + * Service to be modified + */ + service: ServiceClass; + /** + * Pipeline type + */ + type: ReverseIngestionType; +} + +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: AddedOwnerElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: AddedOwnerElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * Added owners to be applied + * + * This schema defines the EntityReferenceList type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface AddedOwnerElement { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + +/** + * This schema defines the type for labeling an entity with a Tag. + */ +export interface TagLabel { + /** + * Description for the tag label. + */ + description?: string; + /** + * Display Name that identifies this tag. + */ + displayName?: string; + /** + * Link to the tag resource. + */ + href?: string; + /** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ + labelType: LabelType; + /** + * Name of the tag or glossary term. + */ + name?: string; + /** + * Label is from Tags or Glossary. + */ + source: TagSource; + /** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ + state: State; + style?: Style; + tagFQN: string; +} + +/** + * Label type describes how a tag label was applied. 'Manual' indicates the tag label was + * applied by a person. 'Derived' indicates a tag label was derived using the associated tag + * relationship (see Classification.json for more details). 'Propagated` indicates a tag + * label was propagated from upstream based on lineage. 'Automated' is used when a tool was + * used to determine the tag label. + */ +export enum LabelType { + Automated = "Automated", + Derived = "Derived", + Manual = "Manual", + Propagated = "Propagated", +} + +/** + * Label is from Tags or Glossary. + */ +export enum TagSource { + Classification = "Classification", + Glossary = "Glossary", +} + +/** + * 'Suggested' state is used when a tag label is suggested by users or tools. Owner of the + * entity must confirm the suggested labels before it is marked as 'Confirmed'. + */ +export enum State { + Confirmed = "Confirmed", + Suggested = "Suggested", +} + +/** + * UI Style is used to associate a color code and/or icon to entity to customize the look of + * that entity in UI. + */ +export interface Style { + /** + * Hex Color Code to mark an entity such as GlossaryTerm, Tag, Domain or Data Product. + */ + color?: string; + /** + * An icon to associate with GlossaryTerm, Tag, Domain or Data Product. + */ + iconURL?: string; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ServiceClass { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + +/** + * Pipeline type + * + * Reverse Ingestion Config Pipeline type + */ +export enum ReverseIngestionType { + ReverseIngestion = "ReverseIngestion", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index 65b770fed44..03b75fedf8c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -3558,6 +3558,8 @@ export interface SourceConfig { * Application Pipeline Configuration. * * ApiService Metadata Pipeline Configuration. + * + * Apply a set of operations on a service */ export interface Pipeline { /** @@ -3986,6 +3988,14 @@ export interface Pipeline { * like endpoints, etc., with that collection will be deleted */ markDeletedApiCollections?: boolean; + /** + * List of operations to be performed on the service + */ + operations?: Operation[]; + /** + * Service to be modified + */ + service?: ServiceClass; } /** @@ -4044,6 +4054,8 @@ export interface FilterPattern { * Search Indexing App. * * Configuration for the Collate AI Quality Agent. + * + * Configuration for the Day One Experience Flow. */ export interface CollateAIAppConfig { /** @@ -4083,7 +4095,8 @@ export interface CollateAIAppConfig { * * Maximum number of events sent in a batch (Default 100). */ - batchSize?: number; + batchSize?: number; + moduleConfiguration?: ModuleConfiguration; /** * Recreates the DataAssets index on DataInsights. Useful if you changed a Custom Property * Type and are facing errors. Bear in mind that recreating the index will delete your @@ -4138,6 +4151,8 @@ export interface CollateAIAppConfig { searchIndexMappingLanguage?: SearchIndexMappingLanguage; /** * Whether the suggested tests should be active or not upon suggestion + * + * Whether the Day One Experience flow should be active or not. */ active?: boolean; /** @@ -4231,7 +4246,7 @@ export interface Action { /** * Domain to apply */ - domain?: EntityReference; + domain?: DomainElement; /** * Description to apply */ @@ -4249,7 +4264,7 @@ export interface Action { /** * Owners to apply */ - owners?: EntityReference[]; + owners?: DomainElement[]; /** * Propagate the metadata to columns via column-level lineage. */ @@ -4295,7 +4310,7 @@ export interface Action { * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. */ -export interface EntityReference { +export interface DomainElement { /** * If true the entity referred to has been soft-deleted. */ @@ -4501,6 +4516,83 @@ export interface BackfillConfiguration { [property: string]: any; } +/** + * Different Module Configurations + */ +export interface ModuleConfiguration { + /** + * App Analytics Module configuration + */ + appAnalytics: AppAnalyticsConfig; + /** + * Cost Analysis Insights Module configuration + */ + costAnalysis: CostAnalysisConfig; + /** + * Data Assets Insights Module configuration + */ + dataAssets: DataAssetsConfig; + /** + * Data Quality Insights Module configuration + */ + dataQuality: DataQualityConfig; +} + +/** + * App Analytics Module configuration + */ +export interface AppAnalyticsConfig { + /** + * If Enabled, App Analytics insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Cost Analysis Insights Module configuration + */ +export interface CostAnalysisConfig { + /** + * If Enabled, Cost Analysis insights will be populated when the App runs. + */ + enabled: boolean; +} + +/** + * Data Assets Insights Module configuration + */ +export interface DataAssetsConfig { + /** + * If Enabled, Data Asset insights will be populated when the App runs. + */ + enabled: boolean; + /** + * List of Entities to Reindex + */ + entities?: string[]; + /** + * Defines the number of days the Data Assets Insights information will be kept. After it + * they will be deleted. + */ + retention?: number; + serviceFilter?: ServiceFilter; +} + +export interface ServiceFilter { + serviceName?: string; + serviceType?: string; +} + +/** + * Data Quality Insights Module configuration + */ +export interface DataQualityConfig { + /** + * If Enabled, Data Quality insights will be populated when the App runs. + */ + enabled: boolean; +} + /** * Entities selected to run the automation. */ @@ -4529,6 +4621,7 @@ export enum CollateAIAppConfigType { CollateAIQualityAgent = "CollateAIQualityAgent", DataInsights = "DataInsights", DataInsightsReport = "DataInsightsReport", + DayOneExperienceWorkflow = "DayOneExperienceWorkflow", SearchIndexing = "SearchIndexing", } @@ -4719,6 +4812,78 @@ export interface LineageInformation { [property: string]: any; } +/** + * Operation to be performed on the entity + */ +export interface Operation { + /** + * Entity to be modified + */ + entityLink: string; + /** + * The id of the operation + */ + id: string; + /** + * The configuration for the operation to be applied + */ + parameters: ReverseIngestionConfig; + /** + * Templated SQL command to be used for the operation. Context parameters will be populated + * based on the event type. + */ + SQLTemplate?: string; + /** + * Type of operation to perform + */ + type: Type; +} + +/** + * The configuration for the operation to be applied + * + * Configuration for updating descriptions + * + * Configuration for updating owners + * + * Configuration for updating tags + */ +export interface ReverseIngestionConfig { + /** + * New description of the service + */ + newDescription?: string; + /** + * Previous description of the service + */ + previousDescription?: string; + /** + * Added owners to be applied + */ + addedOwners?: DomainElement[]; + /** + * Removed owners from the entity + */ + removedOwners?: DomainElement[]; + /** + * Added tags to be applied + */ + addedTags?: TagLabel[]; + /** + * Removed tags of the entity + */ + removedTags?: TagLabel[]; +} + +/** + * Type of operation to perform + */ +export enum Type { + UpdateDescription = "UPDATE_DESCRIPTION", + UpdateOwner = "UPDATE_OWNER", + UpdateTags = "UPDATE_TAGS", +} + /** * Type of Profile Sample (percentage or rows) */ @@ -4735,6 +4900,57 @@ export enum SamplingMethodType { System = "SYSTEM", } +/** + * Service to be modified + * + * This schema defines the EntityReference type used for referencing an entity. + * EntityReference is used for capturing relationships from one entity to another. For + * example, a table has an attribute called database of type EntityReference that captures + * the relationship of a table `belongs to a` database. + */ +export interface ServiceClass { + /** + * If true the entity referred to has been soft-deleted. + */ + deleted?: boolean; + /** + * Optional description of entity. + */ + description?: string; + /** + * Display Name that identifies this entity. + */ + displayName?: string; + /** + * Fully qualified name of the entity instance. For entities such as tables, databases + * fullyQualifiedName is returned in this field. For entities that don't have name hierarchy + * such as `user` and `team` this will be same as the `name` field. + */ + fullyQualifiedName?: string; + /** + * Link to the entity resource. + */ + href?: string; + /** + * Unique identifier that identifies an entity instance. + */ + id: string; + /** + * If true the relationship indicated by this entity reference is inherited from the parent + * entity. + */ + inherited?: boolean; + /** + * Name of the entity instance. + */ + name?: string; + /** + * Entity type/class name - Examples: `database`, `table`, `metrics`, `databaseService`, + * `dashboardService`... + */ + type: string; +} + /** * Service connections available for the logical test suite. */ @@ -4813,6 +5029,8 @@ export interface StorageMetadataBucketDetails { * Pipeline Source Config For Application Pipeline type. Nothing is required. * * Api Source Config Metadata Pipeline type + * + * Reverse Ingestion Config Pipeline type */ export enum ConfigType { APIMetadata = "ApiMetadata", @@ -4829,6 +5047,7 @@ export enum ConfigType { MlModelMetadata = "MlModelMetadata", PipelineMetadata = "PipelineMetadata", Profiler = "Profiler", + ReverseIngestion = "ReverseIngestion", SearchMetadata = "SearchMetadata", StorageMetadata = "StorageMetadata", TestSuite = "TestSuite",