From cee45565982baee73f046546d5af0394cd9622b5 Mon Sep 17 00:00:00 2001 From: Karan Hotchandani <33024356+karanh37@users.noreply.github.com> Date: Thu, 21 Aug 2025 17:23:45 +0530 Subject: [PATCH] fix(ui): app schema ref resolve (#23016) * resolve application schema refs * add details and install comps for plugin * add tests * use parse schema (cherry picked from commit 20486c23c5a19b24c38a4e18130483f49b2e907b) --- .gitignore | 4 +- CLAUDE.md | 69 ++++++++++++- .../src/main/resources/ui/parseSchemas.js | 54 +++++++++++ .../AppDetails/AppDetails.component.tsx | 17 ++-- .../AppDetails/ApplicationsClassBase.test.ts | 96 +++++++++++++++++++ .../AppDetails/ApplicationsClassBase.ts | 9 +- .../pages/AppInstall/AppInstall.component.tsx | 3 +- 7 files changed, 238 insertions(+), 14 deletions(-) create mode 100644 openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.test.ts diff --git a/.gitignore b/.gitignore index bdb3426dc57..a5af85c9a57 100644 --- a/.gitignore +++ b/.gitignore @@ -89,9 +89,7 @@ openmetadata-ui/src/main/resources/ui/playwright/.auth openmetadata-ui/src/main/resources/ui/blob-report #UI - Dereferenced Schemas -openmetadata-ui/src/main/resources/ui/src/jsons/connectionSchemas -openmetadata-ui/src/main/resources/ui/src/jsons/ingestionSchemas -openmetadata-ui/src/main/resources/ui/src/jsons/governanceSchemas +openmetadata-ui/src/main/resources/ui/src/jsons/* #vscode diff --git a/CLAUDE.md b/CLAUDE.md index d33331d9722..9582302e8ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,9 +29,12 @@ make yarn_install_cache # Install UI dependencies cd openmetadata-ui/src/main/resources/ui yarn start # Start development server on localhost:3000 yarn test # Run Jest unit tests +yarn test path/to/test.spec.ts # Run a specific test file +yarn test:watch # Run tests in watch mode yarn playwright:run # Run E2E tests yarn lint # ESLint check yarn lint:fix # ESLint with auto-fix +yarn build # Production build ``` ### Backend Development @@ -76,9 +79,15 @@ OpenMetadata uses a schema-first approach with JSON Schema definitions driving c make generate # Generate all models from schemas make py_antlr # Generate Python ANTLR parsers make js_antlr # Generate JavaScript ANTLR parsers -yarn parse-schema # Parse JSON schemas for frontend +yarn parse-schema # Parse JSON schemas for frontend (connection and ingestion schemas) ``` +### Schema Architecture +- **Source schemas** in `openmetadata-spec/` define the canonical data models +- **Connection schemas** are pre-processed at build time via `parseSchemas.js` to resolve all `$ref` references +- **Application schemas** in `openmetadata-ui/.../ApplicationSchemas/` are resolved at runtime using `schemaResolver.ts` +- JSON schemas with `$ref` references to external files require resolution before use in forms + ## Key Directories - `openmetadata-service/` - Core Java backend services and REST APIs @@ -92,11 +101,54 @@ yarn parse-schema # Parse JSON schemas for frontend ## Development Workflow 1. **Schema Changes**: Modify JSON schemas in `openmetadata-spec/`, then run `mvn clean install` on openmetadata-spec to update models -2. **Backend**: Develop in Java using Dropwizard patterns, test with `mvn test` +2. **Backend**: Develop in Java using Dropwizard patterns, test with `mvn test`, format with `mvn spotless:apply` 3. **Frontend**: Use React/TypeScript with Ant Design components, test with Jest/Playwright 4. **Ingestion**: Python connectors follow plugin pattern, use `make install_dev_env` for development 5. **Full Testing**: Use `make run_e2e_tests` before major changes +## Frontend Architecture Patterns + +### React Component Patterns +- **File Naming**: Components use `ComponentName.component.tsx`, interfaces use `ComponentName.interface.ts` +- **State Management**: Use `useState` with proper typing, avoid `any` +- **Side Effects**: Use `useEffect` with proper dependency arrays +- **Performance**: Use `useCallback` for event handlers, `useMemo` for expensive computations +- **Custom Hooks**: Prefix with `use`, place in `src/hooks/`, return typed objects +- **Internationalization**: Use `useTranslation` hook from react-i18next, access with `t('key')` +- **Component Structure**: Functional components only, no class components +- **Props**: Define interfaces for all component props, place in `.interface.ts` files +- **Loading States**: Use object state for multiple loading states: `useState>({})` +- **Error Handling**: Use `showErrorToast` and `showSuccessToast` utilities from ToastUtils +- **Navigation**: Use `useNavigate` from react-router-dom, not direct history manipulation +- **Data Fetching**: Async functions with try-catch blocks, update loading states appropriately + +### State Management +- Use Zustand stores for global state (e.g., `useLimitStore`, `useWelcomeStore`) +- Keep component state local when possible with `useState` +- Use context providers for feature-specific shared state (e.g., `ApplicationsProvider`) + +### Styling +- Use Ant Design components as the primary UI library +- Custom styles in `.less` files with component-specific naming +- Follow BEM naming convention for custom CSS classes +- Use CSS modules where appropriate + +### Application Configuration +- Applications use `ApplicationsClassBase` for schema loading and configuration +- Dynamic imports handle application-specific schemas and assets +- Form schemas use React JSON Schema Form (RJSF) with custom UI widgets + +### Service Utilities +- Each service type has dedicated utility files (e.g., `DatabaseServiceUtils.tsx`) +- Connection schemas are imported statically and pre-resolved +- Service configurations use switch statements to map types to schemas + +### Type Safety +- All API responses have generated TypeScript interfaces in `generated/` +- Custom types extend base interfaces when needed +- Avoid type assertions unless absolutely necessary +- Use discriminated unions for action types and state variants + ## Database and Migrations - Flyway handles schema migrations in `bootstrap/sql/migrations/` @@ -128,6 +180,19 @@ yarn parse-schema # Parse JSON schemas for frontend - Follow existing project patterns and conventions - Generate production-ready code, not tutorial code +### TypeScript/Frontend Code Requirements +- **NEVER use `any` type** in TypeScript code - always use proper types +- Use `unknown` when the type is truly unknown and add type guards +- Import types from existing type definitions (e.g., `RJSFSchema` from `@rjsf/utils`) +- Follow ESLint rules strictly - the project enforces no-console, proper formatting +- Add `// eslint-disable-next-line` comments only when absolutely necessary +- **Import Organization** (in order): + 1. External libraries (React, Ant Design, etc.) + 2. Internal absolute imports from `generated/`, `constants/`, `hooks/`, etc. + 3. Relative imports for utilities and components + 4. Asset imports (SVGs, styles) + 5. Type imports grouped separately when needed + ### Response Format - Provide clean code blocks without unnecessary explanations - Assume readers are experienced developers diff --git a/openmetadata-ui/src/main/resources/ui/parseSchemas.js b/openmetadata-ui/src/main/resources/ui/parseSchemas.js index 845d014ae4c..211886dcb40 100644 --- a/openmetadata-ui/src/main/resources/ui/parseSchemas.js +++ b/openmetadata-ui/src/main/resources/ui/parseSchemas.js @@ -130,6 +130,57 @@ async function main(rootDir, srcDir, destDir, shouldDereference = false) { } } +// Function to parse Application schemas +async function parseApplicationSchemas() { + const appSchemaDir = 'src/utils/ApplicationSchemas'; + const destDir = 'src/jsons/applicationSchemas'; + + try { + // Create destination directory if it doesn't exist + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } else { + // Clean existing destination directory + fs.rmSync(destDir, { recursive: true }); + fs.mkdirSync(destDir, { recursive: true }); + } + + // Get all JSON files in ApplicationSchemas directory + const files = fs.readdirSync(appSchemaDir).filter(file => file.endsWith('.json')); + + for (const file of files) { + const filePath = path.join(appSchemaDir, file); + const destPath = path.join(destDir, file); + + try { + // Change to the source directory for relative path resolution + const fileDir = path.dirname(filePath); + const originalCwd = process.cwd(); + process.chdir(fileDir); + + // Parse and dereference the schema + let parsedSchema = await parser.parse(file); + parsedSchema = await parser.dereference(parsedSchema); + + // Remove $id fields + const updatedSchema = removeObjectByKey(parsedSchema, '$id'); + + // Change back to original directory + process.chdir(originalCwd); + + // Write the processed schema to destination + fs.writeFileSync(destPath, JSON.stringify(updatedSchema, null, 2)); + console.log(`Processed ApplicationSchema: ${file}`); + } catch (err) { + console.error(`Error processing ${file}:`, err.message); + process.chdir(cwd); + } + } + } catch (err) { + console.error('Error parsing Application schemas:', err); + } +} + // Execute the parsing for connection and ingestion schemas async function runParsers() { // For connection schemas @@ -152,6 +203,9 @@ async function runParsers() { 'schema/governance/workflows/elements/nodes', 'src/jsons/governanceSchemas' ); + + // Parse Application schemas + await parseApplicationSchemas(); } runParsers(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx index f8505daf582..24d16f085db 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/AppDetails.component.tsx @@ -107,7 +107,7 @@ const AppDetails = () => { const schema = await applicationsClassBase.importSchema(fqn); - setJsonSchema(schema.default); + setJsonSchema(schema); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -436,7 +436,8 @@ const AppDetails = () => { return ( + pageTitle={t('label.application-plural')} + >