diff --git a/.gitignore b/.gitignore index 7b945ff74ac..a695f328d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -88,9 +88,7 @@ openmetadata-ui/src/main/resources/ui/playwright/.auth openmetadata-ui/src/main/resources/ui/blob-report #UI - Dereferenced Schemas -openmetadata-ui/src/main/resources/ui/src/jsons/connectionSchemas -openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas -openmetadata-ui/src/main/resources/ui/src/jsons/governanceSchemas +openmetadata-ui/src/main/resources/ui/src/jsons/* #vscode diff --git a/CLAUDE.md b/CLAUDE.md index d33331d9722..9582302e8ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,9 +29,12 @@ make yarn_install_cache # Install UI dependencies cd openmetadata-ui/src/main/resources/ui yarn start # Start development server on localhost:3000 yarn test # Run Jest unit tests +yarn test path/to/test.spec.ts # Run a specific test file +yarn test:watch # Run tests in watch mode yarn playwright:run # Run E2E tests yarn lint # ESLint check yarn lint:fix # ESLint with auto-fix +yarn build # Production build ``` ### Backend Development @@ -76,9 +79,15 @@ OpenMetadata uses a schema-first approach with JSON Schema definitions driving c make generate # Generate all models from schemas make py_antlr # Generate Python ANTLR parsers make js_antlr # Generate JavaScript ANTLR parsers -yarn parse-schema # Parse JSON schemas for frontend +yarn parse-schema # Parse JSON schemas for frontend (connection and ingestion schemas) ``` +### Schema Architecture +- **Source schemas** in `openmetadata-spec/` define the canonical data models +- **Connection schemas** are pre-processed at build time via `parseSchemas.js` to resolve all `$ref` references +- **Application schemas** in `openmetadata-ui/.../ApplicationSchemas/` are resolved at runtime using `schemaResolver.ts` +- JSON schemas with `$ref` references to external files require resolution before use in forms + ## Key Directories - `openmetadata-service/` - Core Java backend services and REST APIs @@ -92,11 +101,54 @@ yarn parse-schema # Parse JSON schemas for frontend ## Development Workflow 1. **Schema Changes**: Modify JSON schemas in `openmetadata-spec/`, then run `mvn clean install` on openmetadata-spec to update models -2. **Backend**: Develop in Java using Dropwizard patterns, test with `mvn test` +2. **Backend**: Develop in Java using Dropwizard patterns, test with `mvn test`, format with `mvn spotless:apply` 3. **Frontend**: Use React/TypeScript with Ant Design components, test with Jest/Playwright 4. **Ingestion**: Python connectors follow plugin pattern, use `make install_dev_env` for development 5. **Full Testing**: Use `make run_e2e_tests` before major changes +## Frontend Architecture Patterns + +### React Component Patterns +- **File Naming**: Components use `ComponentName.component.tsx`, interfaces use `ComponentName.interface.ts` +- **State Management**: Use `useState` with proper typing, avoid `any` +- **Side Effects**: Use `useEffect` with proper dependency arrays +- **Performance**: Use `useCallback` for event handlers, `useMemo` for expensive computations +- **Custom Hooks**: Prefix with `use`, place in `src/hooks/`, return typed objects +- **Internationalization**: Use `useTranslation` hook from react-i18next, access with `t('key')` +- **Component Structure**: Functional components only, no class components +- **Props**: Define interfaces for all component props, place in `.interface.ts` files +- **Loading States**: Use object state for multiple loading states: `useState>({})` +- **Error Handling**: Use `showErrorToast` and `showSuccessToast` utilities from ToastUtils +- **Navigation**: Use `useNavigate` from react-router-dom, not direct history manipulation +- **Data Fetching**: Async functions with try-catch blocks, update loading states appropriately + +### State Management +- Use Zustand stores for global state (e.g., `useLimitStore`, `useWelcomeStore`) +- Keep component state local when possible with `useState` +- Use context providers for feature-specific shared state (e.g., `ApplicationsProvider`) + +### Styling +- Use Ant Design components as the primary UI library +- Custom styles in `.less` files with component-specific naming +- Follow BEM naming convention for custom CSS classes +- Use CSS modules where appropriate + +### Application Configuration +- Applications use `ApplicationsClassBase` for schema loading and configuration +- Dynamic imports handle application-specific schemas and assets +- Form schemas use React JSON Schema Form (RJSF) with custom UI widgets + +### Service Utilities +- Each service type has dedicated utility files (e.g., `DatabaseServiceUtils.tsx`) +- Connection schemas are imported statically and pre-resolved +- Service configurations use switch statements to map types to schemas + +### Type Safety +- All API responses have generated TypeScript interfaces in `generated/` +- Custom types extend base interfaces when needed +- Avoid type assertions unless absolutely necessary +- Use discriminated unions for action types and state variants + ## Database and Migrations - Flyway handles schema migrations in `bootstrap/sql/migrations/` @@ -128,6 +180,19 @@ yarn parse-schema # Parse JSON schemas for frontend - Follow existing project patterns and conventions - Generate production-ready code, not tutorial code +### TypeScript/Frontend Code Requirements +- **NEVER use `any` type** in TypeScript code - always use proper types +- Use `unknown` when the type is truly unknown and add type guards +- Import types from existing type definitions (e.g., `RJSFSchema` from `@rjsf/utils`) +- Follow ESLint rules strictly - the project enforces no-console, proper formatting +- Add `// eslint-disable-next-line` comments only when absolutely necessary +- **Import Organization** (in order): + 1. External libraries (React, Ant Design, etc.) + 2. Internal absolute imports from `generated/`, `constants/`, `hooks/`, etc. + 3. Relative imports for utilities and components + 4. Asset imports (SVGs, styles) + 5. Type imports grouped separately when needed + ### Response Format - Provide clean code blocks without unnecessary explanations - Assume readers are experienced developers diff --git a/openmetadata-ui/src/main/resources/ui/parseSchemas.js b/openmetadata-ui/src/main/resources/ui/parseSchemas.js index 845d014ae4c..211886dcb40 100644 --- a/openmetadata-ui/src/main/resources/ui/parseSchemas.js +++ b/openmetadata-ui/src/main/resources/ui/parseSchemas.js @@ -130,6 +130,57 @@ async function main(rootDir, srcDir, destDir, shouldDereference = false) { } } +// Function to parse Application schemas +async function parseApplicationSchemas() { + const appSchemaDir = 'src/utils/ApplicationSchemas'; + const destDir = 'src/jsons/applicationSchemas'; + + try { + // Create destination directory if it doesn't exist + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } else { + // Clean existing destination directory + fs.rmSync(destDir, { recursive: true }); + fs.mkdirSync(destDir, { recursive: true }); + } + + // Get all JSON files in ApplicationSchemas directory + const files = fs.readdirSync(appSchemaDir).filter(file => file.endsWith('.json')); + + for (const file of files) { + const filePath = path.join(appSchemaDir, file); + const destPath = path.join(destDir, file); + + try { + // Change to the source directory for relative path resolution + const fileDir = path.dirname(filePath); + const originalCwd = process.cwd(); + process.chdir(fileDir); + + // Parse and dereference the schema + let parsedSchema = await parser.parse(file); + parsedSchema = await parser.dereference(parsedSchema); + + // Remove $id fields + const updatedSchema = removeObjectByKey(parsedSchema, '$id'); + + // Change back to original directory + process.chdir(originalCwd); + + // Write the processed schema to destination + fs.writeFileSync(destPath, JSON.stringify(updatedSchema, null, 2)); + console.log(`Processed ApplicationSchema: ${file}`); + } catch (err) { + console.error(`Error processing ${file}:`, err.message); + process.chdir(cwd); + } + } + } catch (err) { + console.error('Error parsing Application schemas:', err); + } +} + // Execute the parsing for connection and ingestion schemas async function runParsers() { // For connection schemas @@ -152,6 +203,9 @@ async function runParsers() { 'schema/governance/workflows/elements/nodes', 'src/jsons/governanceSchemas' ); + + // Parse Application schemas + await parseApplicationSchemas(); } runParsers(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx index aa35e1f600c..02a85e0f163 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx @@ -109,7 +109,7 @@ const AppDetails = () => { const schema = await applicationsClassBase.importSchema(fqn); - setJsonSchema(schema.default); + setJsonSchema(schema); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -325,15 +325,15 @@ const AppDetails = () => { } }; - // Check if there's a plugin configuration component for this app - const pluginConfigComponent = useMemo(() => { + // Check if there's a plugin app details component for this app + const pluginAppDetailsComponent = useMemo(() => { if (!appData?.name || !plugins.length) { return null; } const plugin = plugins.find((p) => p.name === appData.name); - return plugin?.getConfigComponent?.(appData) || null; + return plugin?.getAppDetails?.(appData) || null; }, [appData?.name, plugins]); const tabs = useMemo(() => { @@ -348,11 +348,7 @@ const AppDetails = () => { /> ), key: ApplicationTabs.CONFIGURATION, - children: pluginConfigComponent ? ( - // Use plugin configuration component if available - React.createElement(pluginConfigComponent) - ) : ( - // Fall back to default ApplicationConfiguration + children: ( { - + {pluginAppDetailsComponent ? ( + // Render plugin's custom app details component + React.createElement(pluginAppDetailsComponent) + ) : ( + // Render default tabs interface + + )} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.test.ts new file mode 100644 index 00000000000..0e51e5939e8 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.test.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AppType } from '../../../../generated/entity/applications/app'; +import applicationsClassBase from './ApplicationsClassBase'; + +describe('ApplicationsClassBase', () => { + describe('importSchema', () => { + it('should import pre-parsed schema', async () => { + // Mock the dynamic import + jest.doMock( + '../../../../jsons/applicationSchemas/SearchIndexingApplication.json', + () => ({ + type: 'object', + properties: { + type: { + type: 'string', + default: 'SearchIndexing', + }, + cacheSize: { + type: 'integer', + default: 100, + }, + }, + }), + { virtual: true } + ); + + const schema = await applicationsClassBase.importSchema( + 'SearchIndexingApplication' + ); + + expect(schema).toBeDefined(); + expect(schema.type).toBe('object'); + expect(schema.properties).toBeDefined(); + // Should not contain any $ref since schemas are pre-parsed + expect(JSON.stringify(schema)).not.toContain('$ref'); + }); + }); + + describe('getJSONUISchema', () => { + it('should return UI schema configuration', () => { + const uiSchema = applicationsClassBase.getJSONUISchema(); + + expect(uiSchema).toBeDefined(); + expect(uiSchema.moduleConfiguration?.dataAssets?.serviceFilter).toEqual({ + 'ui:widget': 'hidden', + }); + expect(uiSchema.entityLink).toEqual({ + 'ui:widget': 'hidden', + }); + expect(uiSchema.type).toEqual({ + 'ui:widget': 'hidden', + }); + }); + }); + + describe('getScheduleOptionsForApp', () => { + it('should return week schedule for DataInsightsReportApplication', () => { + const options = applicationsClassBase.getScheduleOptionsForApp( + 'DataInsightsReportApplication', + AppType.Internal + ); + + expect(options).toEqual(['week']); + }); + + it('should return day schedule for External apps', () => { + const options = applicationsClassBase.getScheduleOptionsForApp( + 'SomeExternalApp', + AppType.External + ); + + expect(options).toEqual(['day']); + }); + + it('should return undefined when no schedules provided for other apps', () => { + const options = applicationsClassBase.getScheduleOptionsForApp( + 'SomeApp', + AppType.Internal + ); + + expect(options).toBeUndefined(); + }); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts index 1f18ee6fa98..ef11f52c1f5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts @@ -17,8 +17,13 @@ import { getScheduleOptionsFromSchedules } from '../../../../utils/SchedularUtil import { AppPlugin } from '../plugins/AppPlugin'; class ApplicationsClassBase { - public importSchema(fqn: string) { - return import(`../../../../utils/ApplicationSchemas/${fqn}.json`); + public async importSchema(fqn: string) { + const module = await import( + `../../../../jsons/applicationSchemas/${fqn}.json` + ); + const schema = module.default || module; + + return schema; } public getJSONUISchema() { return { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/plugins/AppPlugin.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/plugins/AppPlugin.ts index 00b847a9f99..a692c2d4147 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/plugins/AppPlugin.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/plugins/AppPlugin.ts @@ -13,6 +13,7 @@ import { FC } from 'react'; import { RouteProps } from 'react-router-dom'; import { App } from '../../../../generated/entity/applications/app'; +import { AppMarketPlaceDefinition } from '../../../../generated/entity/applications/marketplace/appMarketPlaceDefinition'; import { LeftSidebarItem } from '../../../MyData/LeftSidebar/LeftSidebar.interface'; export interface LeftSidebarItemExample extends LeftSidebarItem { @@ -39,14 +40,15 @@ export interface AppPlugin { isInstalled: boolean; /** - * Optional method that returns a React component for plugin configuration. - * It is the responsibility of this component to update application data using patchApplication API + * Optional method that returns a React component for the entire app details view. + * If provided, this component will replace the default tabs interface. + * It is the responsibility of this component to handle all app details functionality. * * @param app - The App entity containing application details and configuration - * @returns A React functional component for plugin settings/configuration, - * or null if no configuration is needed. + * @returns A React functional component for the complete app details view, + * or null if the default tabs interface should be used. */ - getConfigComponent?(app: App): FC | null; + getAppDetails?(app: App): FC | null; /** * Optional method that provides custom routes for the plugin. @@ -63,4 +65,15 @@ export interface AppPlugin { * left sidebar when the plugin is active. */ getSidebarActions?(): Array; + + /** + * Optional method that returns a React component for the app installation page. + * If provided, this component will replace the default stepper and tabs interface + * in the App Installation page. + * + * @param app - The AppMarketPlaceDefinition entity containing application details and configuration + * @returns A React functional component for the complete app installation view, + * or null if the default installation interface should be used. + */ + getAppInstallComponent?(app: AppMarketPlaceDefinition): FC | null; } diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx index cce6b29845b..21d044efe8e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/AppInstall/AppInstall.component.tsx @@ -15,7 +15,7 @@ import { RJSFSchema } from '@rjsf/utils'; import { Col, Row, Typography } from 'antd'; import { AxiosError } from 'axios'; import { isEmpty } from 'lodash'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { FC, useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useNavigate } from 'react-router-dom'; import ErrorPlaceHolder from '../../components/common/ErrorWithPlaceholder/ErrorPlaceHolder'; @@ -27,6 +27,7 @@ import { } from '../../components/Settings/Applications/AppDetails/ApplicationsClassBase'; import AppInstallVerifyCard from '../../components/Settings/Applications/AppInstallVerifyCard/AppInstallVerifyCard.component'; import ApplicationConfiguration from '../../components/Settings/Applications/ApplicationConfiguration/ApplicationConfiguration'; +import { AppPlugin } from '../../components/Settings/Applications/plugins/AppPlugin'; import ScheduleInterval from '../../components/Settings/Services/AddIngestion/Steps/ScheduleInterval'; import { WorkflowExtraConfig } from '../../components/Settings/Services/AddIngestion/Steps/ScheduleInterval.interface'; import IngestionStepper from '../../components/Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; @@ -65,6 +66,7 @@ const AppInstall = () => { const [activeServiceStep, setActiveServiceStep] = useState(1); const [appConfiguration, setAppConfiguration] = useState(); const [jsonSchema, setJsonSchema] = useState(); + const [pluginComponent, setPluginComponent] = useState(null); const { config, getResourceLimit } = useLimitStore(); const { pipelineSchedules } = @@ -112,6 +114,20 @@ const AppInstall = () => { const schema = await applicationSchemaClassBase.importSchema(fqn); setJsonSchema(schema); + + // Check if this app has a plugin with a custom install component + if (data.name) { + const PluginClass = applicationsClassBase.appPluginRegistry[data.name]; + if (PluginClass) { + const pluginInstance: AppPlugin = new PluginClass(data.name, false); + if (pluginInstance.getAppInstallComponent) { + const Component = pluginInstance.getAppInstallComponent(data); + if (Component) { + setPluginComponent(() => Component); + } + } + } + } } catch (_) { showErrorToast( t('message.no-application-schema-found', { appName: fqn }) @@ -260,17 +276,22 @@ const AppInstall = () => { - - - - - - {renderSelectedTab} - - + {pluginComponent ? ( + // Render plugin's custom app details component + React.createElement(pluginComponent) + ) : ( + + + + + + {renderSelectedTab} + + + )} ); };