mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-25 17:37:57 +00:00
fix(test): ingestion related playwright (#17674)
* fix(test): ingestion related playwright * fix failing tests * update status logic to avoid conflict * fix double run issue for ingestion * fixing ingestion specs * fix postgres and ingestion * running ingestion specs * fix airflow and argo gaps * revert config * fix redshift dbt tests
This commit is contained in:
parent
123f222026
commit
1848be697b
@ -30,7 +30,7 @@ const uniqueID = uuid();
|
|||||||
|
|
||||||
export const REDSHIFT = {
|
export const REDSHIFT = {
|
||||||
serviceType: 'Redshift',
|
serviceType: 'Redshift',
|
||||||
serviceName: `redshift-ct-test-${uniqueID}`,
|
serviceName: `redshift-ct-test-with-%-${uniqueID}`,
|
||||||
tableName: 'raw_payments',
|
tableName: 'raw_payments',
|
||||||
DBTTable: 'customers',
|
DBTTable: 'customers',
|
||||||
description: `This is Redshift-ct-test-${uniqueID} description`,
|
description: `This is Redshift-ct-test-${uniqueID} description`,
|
||||||
@ -38,7 +38,7 @@ export const REDSHIFT = {
|
|||||||
|
|
||||||
export const POSTGRES = {
|
export const POSTGRES = {
|
||||||
serviceType: 'Postgres',
|
serviceType: 'Postgres',
|
||||||
serviceName: `pw-postgres-test-${uuid()}`,
|
serviceName: `pw-postgres-test-with-%-${uniqueID}`,
|
||||||
tableName: 'order_items',
|
tableName: 'order_items',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -11,7 +11,9 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import test from '@playwright/test';
|
import test, { expect } from '@playwright/test';
|
||||||
|
import { POSTGRES, REDSHIFT } from '../../constant/service';
|
||||||
|
import { GlobalSettingOptions } from '../../constant/settings';
|
||||||
import AirflowIngestionClass from '../../support/entity/ingestion/AirflowIngestionClass';
|
import AirflowIngestionClass from '../../support/entity/ingestion/AirflowIngestionClass';
|
||||||
import BigQueryIngestionClass from '../../support/entity/ingestion/BigQueryIngestionClass';
|
import BigQueryIngestionClass from '../../support/entity/ingestion/BigQueryIngestionClass';
|
||||||
import KafkaIngestionClass from '../../support/entity/ingestion/KafkaIngestionClass';
|
import KafkaIngestionClass from '../../support/entity/ingestion/KafkaIngestionClass';
|
||||||
@ -23,8 +25,8 @@ import RedshiftWithDBTIngestionClass from '../../support/entity/ingestion/Redshi
|
|||||||
import S3IngestionClass from '../../support/entity/ingestion/S3IngestionClass';
|
import S3IngestionClass from '../../support/entity/ingestion/S3IngestionClass';
|
||||||
import SnowflakeIngestionClass from '../../support/entity/ingestion/SnowflakeIngestionClass';
|
import SnowflakeIngestionClass from '../../support/entity/ingestion/SnowflakeIngestionClass';
|
||||||
import SupersetIngestionClass from '../../support/entity/ingestion/SupersetIngestionClass';
|
import SupersetIngestionClass from '../../support/entity/ingestion/SupersetIngestionClass';
|
||||||
import { redirectToHomePage } from '../../utils/common';
|
import { INVALID_NAMES, redirectToHomePage } from '../../utils/common';
|
||||||
import { settingClick } from '../../utils/sidebar';
|
import { settingClick, SettingOptionsType } from '../../utils/sidebar';
|
||||||
|
|
||||||
const services = [
|
const services = [
|
||||||
S3IngestionClass,
|
S3IngestionClass,
|
||||||
@ -44,19 +46,26 @@ if (process.env.PLAYWRIGHT_IS_OSS) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// use the admin user to login
|
// use the admin user to login
|
||||||
test.use({ storageState: 'playwright/.auth/admin.json', trace: 'off' });
|
test.use({
|
||||||
|
storageState: 'playwright/.auth/admin.json',
|
||||||
|
trace: process.env.PLAYWRIGHT_IS_OSS ? 'off' : 'on-first-retry',
|
||||||
|
});
|
||||||
|
|
||||||
services.forEach((ServiceClass) => {
|
services.forEach((ServiceClass) => {
|
||||||
const service = new ServiceClass();
|
const service = new ServiceClass();
|
||||||
|
|
||||||
test.describe.configure({
|
test.describe.configure({
|
||||||
timeout: 300000,
|
// 11 minutes max for ingestion tests
|
||||||
|
timeout: 11 * 60 * 1000,
|
||||||
});
|
});
|
||||||
|
|
||||||
test.describe.serial(service.serviceType, { tag: '@ingestion' }, async () => {
|
test.describe.serial(service.serviceType, { tag: '@ingestion' }, async () => {
|
||||||
test.beforeEach('Visit entity details page', async ({ page }) => {
|
test.beforeEach('Visit entity details page', async ({ page }) => {
|
||||||
await redirectToHomePage(page);
|
await redirectToHomePage(page);
|
||||||
await settingClick(page, service.category);
|
await settingClick(
|
||||||
|
page,
|
||||||
|
service.category as unknown as SettingOptionsType
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test(`Create & Ingest ${service.serviceType} service`, async ({ page }) => {
|
test(`Create & Ingest ${service.serviceType} service`, async ({ page }) => {
|
||||||
@ -73,12 +82,48 @@ services.forEach((ServiceClass) => {
|
|||||||
await service.updateScheduleOptions(page);
|
await service.updateScheduleOptions(page);
|
||||||
});
|
});
|
||||||
|
|
||||||
test.fixme(`Service specific tests`, async () => {
|
if (
|
||||||
await service.runAdditionalTests(test);
|
[POSTGRES.serviceType, REDSHIFT.serviceType].includes(service.serviceType)
|
||||||
});
|
) {
|
||||||
|
test(`Service specific tests`, async ({ page }) => {
|
||||||
|
await service.runAdditionalTests(page, test);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
test(`Delete ${service.serviceType} service`, async ({ page }) => {
|
test(`Delete ${service.serviceType} service`, async ({ page }) => {
|
||||||
await service.deleteService(page);
|
await service.deleteService(page);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test.describe('Service form', () => {
|
||||||
|
test('name field should throw error for invalid name', async ({ page }) => {
|
||||||
|
await redirectToHomePage(page);
|
||||||
|
await settingClick(page, GlobalSettingOptions.DATABASES);
|
||||||
|
await page.click('[data-testid="add-service-button"]');
|
||||||
|
await page.click('[data-testid="Mysql"]');
|
||||||
|
await page.click('[data-testid="next-button"]');
|
||||||
|
|
||||||
|
await page.waitForSelector('[data-testid="service-name"]');
|
||||||
|
await page.click('[data-testid="next-button"]');
|
||||||
|
|
||||||
|
await expect(page.locator('#name_help')).toBeVisible();
|
||||||
|
await expect(page.locator('#name_help')).toHaveText('Name is required');
|
||||||
|
|
||||||
|
await page.fill(
|
||||||
|
'[data-testid="service-name"]',
|
||||||
|
INVALID_NAMES.WITH_SPECIAL_CHARS
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(page.locator('#name_help')).toBeVisible();
|
||||||
|
await expect(page.locator('#name_help')).toHaveText(
|
||||||
|
'Name must contain only letters, numbers, underscores, hyphens, periods, parenthesis, and ampersands.'
|
||||||
|
);
|
||||||
|
|
||||||
|
await page.fill('[data-testid="service-name"]', 'test-service');
|
||||||
|
|
||||||
|
await page.click('[data-testid="next-button"]');
|
||||||
|
|
||||||
|
await expect(page.getByTestId('step-icon-3')).toHaveClass(/active/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
import { test as setup } from '@playwright/test';
|
import { test as setup } from '@playwright/test';
|
||||||
import { JWT_EXPIRY_TIME_MAP } from '../constant/login';
|
import { JWT_EXPIRY_TIME_MAP } from '../constant/login';
|
||||||
import { AdminClass } from '../support/user/AdminClass';
|
import { AdminClass } from '../support/user/AdminClass';
|
||||||
import { getApiContext, getToken } from '../utils/common';
|
import { getApiContext } from '../utils/common';
|
||||||
import { updateJWTTokenExpiryTime } from '../utils/login';
|
import { updateJWTTokenExpiryTime } from '../utils/login';
|
||||||
const adminFile = 'playwright/.auth/admin.json';
|
const adminFile = 'playwright/.auth/admin.json';
|
||||||
|
|
||||||
@ -31,10 +31,6 @@ setup('authenticate as admin', async ({ page }) => {
|
|||||||
await admin.login(page);
|
await admin.login(page);
|
||||||
await page.waitForURL('**/my-data');
|
await page.waitForURL('**/my-data');
|
||||||
|
|
||||||
const token = await getToken(page);
|
|
||||||
// eslint-disable-next-line no-console
|
|
||||||
console.log(token);
|
|
||||||
|
|
||||||
// End of authentication steps.
|
// End of authentication steps.
|
||||||
await page.context().storageState({ path: adminFile });
|
await page.context().storageState({ path: adminFile });
|
||||||
});
|
});
|
||||||
|
@ -20,7 +20,7 @@ class MetabaseIngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.Pipeline,
|
Services.Pipeline,
|
||||||
`pw-airflow-${uuid()}`,
|
`pw-airflow-with-%-${uuid()}`,
|
||||||
'Airflow',
|
'Airflow',
|
||||||
'sample_lineage'
|
'sample_lineage'
|
||||||
);
|
);
|
||||||
|
@ -25,7 +25,12 @@ class BigQueryIngestionClass extends ServiceBaseClass {
|
|||||||
filterPattern: string;
|
filterPattern: string;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super(Services.Database, `pw-bigquery-${uuid()}`, 'BigQuery', 'testtable');
|
super(
|
||||||
|
Services.Database,
|
||||||
|
`pw-bigquery-with-%-${uuid()}`,
|
||||||
|
'BigQuery',
|
||||||
|
'testtable'
|
||||||
|
);
|
||||||
|
|
||||||
this.filterPattern = 'testschema';
|
this.filterPattern = 'testschema';
|
||||||
}
|
}
|
||||||
|
@ -24,7 +24,7 @@ class KafkaIngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.Messaging,
|
Services.Messaging,
|
||||||
`pw-kafka-${uuid()}`,
|
`pw-kafka-with-%-${uuid()}`,
|
||||||
'Kafka',
|
'Kafka',
|
||||||
'__transaction_state'
|
'__transaction_state'
|
||||||
);
|
);
|
||||||
|
@ -26,7 +26,7 @@ class MetabaseIngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.Dashboard,
|
Services.Dashboard,
|
||||||
`pw-Metabase-${uuid()}`,
|
`pw-Metabase-with-%-${uuid()}`,
|
||||||
'Metabase',
|
'Metabase',
|
||||||
'jaffle_shop dashboard'
|
'jaffle_shop dashboard'
|
||||||
);
|
);
|
||||||
|
@ -23,7 +23,7 @@ class MlFlowIngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.MLModels,
|
Services.MLModels,
|
||||||
`pw-Ml-Model-${uuid()}`,
|
`pw-Ml-Model-with-%-${uuid()}`,
|
||||||
'Mlflow',
|
'Mlflow',
|
||||||
'ElasticnetWineModel',
|
'ElasticnetWineModel',
|
||||||
false,
|
false,
|
||||||
|
@ -23,7 +23,12 @@ class MysqlIngestionClass extends ServiceBaseClass {
|
|||||||
name: string;
|
name: string;
|
||||||
tableFilter: string[];
|
tableFilter: string[];
|
||||||
constructor() {
|
constructor() {
|
||||||
super(Services.Database, `pw-mysql-${uuid()}`, 'Mysql', 'bot_entity');
|
super(
|
||||||
|
Services.Database,
|
||||||
|
`pw-mysql-with-%-${uuid()}`,
|
||||||
|
'Mysql',
|
||||||
|
'bot_entity'
|
||||||
|
);
|
||||||
this.tableFilter = ['bot_entity', 'alert_entity', 'chart_entity'];
|
this.tableFilter = ['bot_entity', 'alert_entity', 'chart_entity'];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -11,9 +11,18 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Page } from '@playwright/test';
|
import {
|
||||||
|
Page,
|
||||||
|
PlaywrightTestArgs,
|
||||||
|
PlaywrightWorkerArgs,
|
||||||
|
TestType,
|
||||||
|
} from '@playwright/test';
|
||||||
import { POSTGRES } from '../../../constant/service';
|
import { POSTGRES } from '../../../constant/service';
|
||||||
import { redirectToHomePage } from '../../../utils/common';
|
import {
|
||||||
|
getApiContext,
|
||||||
|
redirectToHomePage,
|
||||||
|
toastNotification,
|
||||||
|
} from '../../../utils/common';
|
||||||
import { visitEntityPage } from '../../../utils/entity';
|
import { visitEntityPage } from '../../../utils/entity';
|
||||||
import { visitServiceDetailsPage } from '../../../utils/service';
|
import { visitServiceDetailsPage } from '../../../utils/service';
|
||||||
import {
|
import {
|
||||||
@ -72,9 +81,13 @@ class PostgresIngestionClass extends ServiceBaseClass {
|
|||||||
await page.locator('#root\\/schemaFilterPattern\\/includes').press('Enter');
|
await page.locator('#root\\/schemaFilterPattern\\/includes').press('Enter');
|
||||||
}
|
}
|
||||||
|
|
||||||
async runAdditionalTests(test) {
|
async runAdditionalTests(
|
||||||
|
page: Page,
|
||||||
|
test: TestType<PlaywrightTestArgs, PlaywrightWorkerArgs>
|
||||||
|
) {
|
||||||
if (process.env.PLAYWRIGHT_IS_OSS) {
|
if (process.env.PLAYWRIGHT_IS_OSS) {
|
||||||
test('Add Usage ingestion', async ({ page }) => {
|
await test.step('Add Usage ingestion', async () => {
|
||||||
|
const { apiContext } = await getApiContext(page);
|
||||||
await redirectToHomePage(page);
|
await redirectToHomePage(page);
|
||||||
await visitServiceDetailsPage(
|
await visitServiceDetailsPage(
|
||||||
page,
|
page,
|
||||||
@ -95,25 +108,44 @@ class PostgresIngestionClass extends ServiceBaseClass {
|
|||||||
await page.click('[data-menu-id*="usage"]');
|
await page.click('[data-menu-id*="usage"]');
|
||||||
await page.fill('#root\\/queryLogFilePath', this.queryLogFilePath);
|
await page.fill('#root\\/queryLogFilePath', this.queryLogFilePath);
|
||||||
|
|
||||||
const deployResponse = page.waitForResponse(
|
|
||||||
'/api/v1/services/ingestionPipelines/deploy/*'
|
|
||||||
);
|
|
||||||
|
|
||||||
await page.click('[data-testid="submit-btn"]');
|
await page.click('[data-testid="submit-btn"]');
|
||||||
await page.click('[data-testid="deploy-button"]');
|
// Make sure we create ingestion with None schedule to avoid conflict between Airflow and Argo behavior
|
||||||
|
await this.scheduleIngestion(page);
|
||||||
await deployResponse;
|
|
||||||
|
|
||||||
await page.click('[data-testid="view-service-button"]');
|
await page.click('[data-testid="view-service-button"]');
|
||||||
|
|
||||||
await page.waitForResponse(
|
// Header available once page loads
|
||||||
'**/api/v1/services/ingestionPipelines/status'
|
await page.waitForSelector('[data-testid="data-assets-header"]');
|
||||||
|
await page.getByTestId('loader').waitFor({ state: 'detached' });
|
||||||
|
await page.getByTestId('ingestions').click();
|
||||||
|
await page
|
||||||
|
.getByLabel('Ingestions')
|
||||||
|
.getByTestId('loader')
|
||||||
|
.waitFor({ state: 'detached' });
|
||||||
|
|
||||||
|
const response = await apiContext
|
||||||
|
.get(
|
||||||
|
`/api/v1/services/ingestionPipelines?service=${encodeURIComponent(
|
||||||
|
this.serviceName
|
||||||
|
)}&pipelineType=usage&serviceType=databaseService&limit=1`
|
||||||
|
)
|
||||||
|
.then((res) => res.json());
|
||||||
|
|
||||||
|
await page.click(
|
||||||
|
`[data-row-key*="${response.data[0].name}"] [data-testid="more-actions"]`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
await page.getByTestId('run-button').click();
|
||||||
|
|
||||||
|
await toastNotification(page, `Pipeline triggered successfully!`);
|
||||||
|
|
||||||
await this.handleIngestionRetry('usage', page);
|
await this.handleIngestionRetry('usage', page);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Verify if usage is ingested properly', async ({ page }) => {
|
await test.step('Verify if usage is ingested properly', async () => {
|
||||||
|
await page.waitForSelector('[data-testid="loader"]', {
|
||||||
|
state: 'hidden',
|
||||||
|
});
|
||||||
const entityResponse = page.waitForResponse(
|
const entityResponse = page.waitForResponse(
|
||||||
`/api/v1/tables/name/*.order_items?**`
|
`/api/v1/tables/name/*.order_items?**`
|
||||||
);
|
);
|
||||||
@ -128,9 +160,10 @@ class PostgresIngestionClass extends ServiceBaseClass {
|
|||||||
|
|
||||||
await page.getByRole('tab', { name: 'Queries' }).click();
|
await page.getByRole('tab', { name: 'Queries' }).click();
|
||||||
|
|
||||||
await page.waitForSelector(
|
// Need to connect to postgres db to get the query log
|
||||||
'[data-testid="queries-container"] >> text=selectQuery'
|
// await page.waitForSelector(
|
||||||
);
|
// '[data-testid="queries-container"] >> text=selectQuery'
|
||||||
|
// );
|
||||||
|
|
||||||
await page.click('[data-testid="schema"]');
|
await page.click('[data-testid="schema"]');
|
||||||
await page.waitForSelector('[data-testid="related-tables-data"]');
|
await page.waitForSelector('[data-testid="related-tables-data"]');
|
||||||
|
@ -11,10 +11,20 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { expect, Page } from '@playwright/test';
|
import {
|
||||||
|
expect,
|
||||||
|
Page,
|
||||||
|
PlaywrightTestArgs,
|
||||||
|
PlaywrightWorkerArgs,
|
||||||
|
TestType,
|
||||||
|
} from '@playwright/test';
|
||||||
import { DBT, HTTP_CONFIG_SOURCE, REDSHIFT } from '../../../constant/service';
|
import { DBT, HTTP_CONFIG_SOURCE, REDSHIFT } from '../../../constant/service';
|
||||||
import { SidebarItem } from '../../../constant/sidebar';
|
import { SidebarItem } from '../../../constant/sidebar';
|
||||||
import { redirectToHomePage } from '../../../utils/common';
|
import {
|
||||||
|
getApiContext,
|
||||||
|
redirectToHomePage,
|
||||||
|
toastNotification,
|
||||||
|
} from '../../../utils/common';
|
||||||
import { visitEntityPage } from '../../../utils/entity';
|
import { visitEntityPage } from '../../../utils/entity';
|
||||||
import { visitServiceDetailsPage } from '../../../utils/service';
|
import { visitServiceDetailsPage } from '../../../utils/service';
|
||||||
import {
|
import {
|
||||||
@ -75,12 +85,14 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass {
|
|||||||
.fill(this.schemaFilterPattern);
|
.fill(this.schemaFilterPattern);
|
||||||
|
|
||||||
await page.locator('#root\\/schemaFilterPattern\\/includes').press('Enter');
|
await page.locator('#root\\/schemaFilterPattern\\/includes').press('Enter');
|
||||||
|
|
||||||
await page.click('#root\\/includeViews');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async runAdditionalTests(test) {
|
async runAdditionalTests(
|
||||||
test('Add DBT ingestion', async ({ page }) => {
|
page: Page,
|
||||||
|
test: TestType<PlaywrightTestArgs, PlaywrightWorkerArgs>
|
||||||
|
) {
|
||||||
|
await test.step('Add DBT ingestion', async () => {
|
||||||
|
const { apiContext } = await getApiContext(page);
|
||||||
await redirectToHomePage(page);
|
await redirectToHomePage(page);
|
||||||
await visitServiceDetailsPage(
|
await visitServiceDetailsPage(
|
||||||
page,
|
page,
|
||||||
@ -94,6 +106,7 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass {
|
|||||||
|
|
||||||
await page.click('[data-testid="ingestions"]');
|
await page.click('[data-testid="ingestions"]');
|
||||||
await page.waitForSelector('[data-testid="ingestion-details-container"]');
|
await page.waitForSelector('[data-testid="ingestion-details-container"]');
|
||||||
|
await page.waitForTimeout(1000);
|
||||||
await page.click('[data-testid="add-new-ingestion-button"]');
|
await page.click('[data-testid="add-new-ingestion-button"]');
|
||||||
await page.waitForTimeout(1000);
|
await page.waitForTimeout(1000);
|
||||||
await page.click('[data-menu-id*="dbt"]');
|
await page.click('[data-menu-id*="dbt"]');
|
||||||
@ -115,23 +128,41 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass {
|
|||||||
'#root\\/dbtConfigSource\\/dbtRunResultsHttpPath',
|
'#root\\/dbtConfigSource\\/dbtRunResultsHttpPath',
|
||||||
HTTP_CONFIG_SOURCE.DBT_RUN_RESULTS_FILE_PATH
|
HTTP_CONFIG_SOURCE.DBT_RUN_RESULTS_FILE_PATH
|
||||||
);
|
);
|
||||||
const deployResponse = page.waitForResponse(
|
|
||||||
'/api/v1/services/ingestionPipelines/deploy/*'
|
|
||||||
);
|
|
||||||
|
|
||||||
await page.click('[data-testid="submit-btn"]');
|
await page.click('[data-testid="submit-btn"]');
|
||||||
await page.click('[data-testid="deploy-button"]');
|
// Make sure we create ingestion with None schedule to avoid conflict between Airflow and Argo behavior
|
||||||
|
await this.scheduleIngestion(page);
|
||||||
await deployResponse;
|
|
||||||
|
|
||||||
await page.click('[data-testid="view-service-button"]');
|
await page.click('[data-testid="view-service-button"]');
|
||||||
|
|
||||||
await page.waitForResponse(
|
// Header available once page loads
|
||||||
'**/api/v1/services/ingestionPipelines/status'
|
await page.waitForSelector('[data-testid="data-assets-header"]');
|
||||||
|
await page.getByTestId('loader').waitFor({ state: 'detached' });
|
||||||
|
await page.getByTestId('ingestions').click();
|
||||||
|
await page
|
||||||
|
.getByLabel('Ingestions')
|
||||||
|
.getByTestId('loader')
|
||||||
|
.waitFor({ state: 'detached' });
|
||||||
|
|
||||||
|
const response = await apiContext
|
||||||
|
.get(
|
||||||
|
`/api/v1/services/ingestionPipelines?service=${encodeURIComponent(
|
||||||
|
this.serviceName
|
||||||
|
)}&pipelineType=dbt&serviceType=databaseService&limit=1`
|
||||||
|
)
|
||||||
|
.then((res) => res.json());
|
||||||
|
|
||||||
|
await page.click(
|
||||||
|
`[data-row-key*="${response.data[0].name}"] [data-testid="more-actions"]`
|
||||||
);
|
);
|
||||||
|
await page.getByTestId('run-button').click();
|
||||||
|
|
||||||
|
await toastNotification(page, `Pipeline triggered successfully!`);
|
||||||
|
|
||||||
|
await this.handleIngestionRetry('dbt', page);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Validate DBT is ingested properly', async ({ page }) => {
|
await test.step('Validate DBT is ingested properly', async () => {
|
||||||
await sidebarClick(page, SidebarItem.TAGS);
|
await sidebarClick(page, SidebarItem.TAGS);
|
||||||
|
|
||||||
await page.waitForSelector('[data-testid="data-summary-container"]');
|
await page.waitForSelector('[data-testid="data-summary-container"]');
|
||||||
@ -153,16 +184,18 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass {
|
|||||||
await visitEntityPage({
|
await visitEntityPage({
|
||||||
page,
|
page,
|
||||||
searchTerm: REDSHIFT.DBTTable,
|
searchTerm: REDSHIFT.DBTTable,
|
||||||
dataTestId: `${REDSHIFT.serviceName}.${REDSHIFT.DBTTable}`,
|
dataTestId: `${REDSHIFT.serviceName}-${REDSHIFT.DBTTable}`,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Verify tags
|
// Verify tags
|
||||||
await page.waitForSelector('[data-testid="entity-tags"]');
|
await page.waitForSelector('[data-testid="entity-tags"]');
|
||||||
const entityTagsText = await page.textContent(
|
|
||||||
'[data-testid="entity-tags"]'
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(entityTagsText).toContain(DBT.tagName);
|
await expect(
|
||||||
|
page
|
||||||
|
.getByTestId('entity-right-panel')
|
||||||
|
.getByTestId('tags-container')
|
||||||
|
.getByTestId('entity-tags')
|
||||||
|
).toContainText(DBT.tagName);
|
||||||
|
|
||||||
// Verify DBT tab is present
|
// Verify DBT tab is present
|
||||||
await page.click('[data-testid="dbt"]');
|
await page.click('[data-testid="dbt"]');
|
||||||
@ -186,25 +219,15 @@ class RedshiftWithDBTIngestionClass extends ServiceBaseClass {
|
|||||||
await page.click('[data-testid="profiler"]');
|
await page.click('[data-testid="profiler"]');
|
||||||
|
|
||||||
await page.waitForSelector('[data-testid="profiler-tab-left-panel"]');
|
await page.waitForSelector('[data-testid="profiler-tab-left-panel"]');
|
||||||
const profilerTabLeftPanelText = await page.textContent(
|
await page.getByRole('menuitem', { name: 'Data Quality' }).click();
|
||||||
'[data-testid="profiler-tab-left-panel"]'
|
|
||||||
|
await expect(page.getByTestId(DBT.dataQualityTest1)).toHaveText(
|
||||||
|
DBT.dataQualityTest1
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(profilerTabLeftPanelText).toContain('Data Quality');
|
await expect(page.getByTestId(DBT.dataQualityTest1)).toHaveText(
|
||||||
|
DBT.dataQualityTest1
|
||||||
await page.waitForSelector(`[data-testid=${DBT.dataQualityTest1}]`);
|
|
||||||
const dataQualityTest1Text = await page.textContent(
|
|
||||||
`[data-testid=${DBT.dataQualityTest1}]`
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(dataQualityTest1Text).toContain(DBT.dataQualityTest1);
|
|
||||||
|
|
||||||
await page.waitForSelector(`[data-testid=${DBT.dataQualityTest2}]`);
|
|
||||||
const dataQualityTest2Text = await page.textContent(
|
|
||||||
`[data-testid=${DBT.dataQualityTest2}]`
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(dataQualityTest2Text).toContain(DBT.dataQualityTest2);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ class S3IngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.Storage,
|
Services.Storage,
|
||||||
`pw-s3-storage-${uuid()}`,
|
`pw-s3-storage-with-%-${uuid()}`,
|
||||||
'S3',
|
'S3',
|
||||||
'awsathena-database'
|
'awsathena-database'
|
||||||
);
|
);
|
||||||
|
@ -199,7 +199,7 @@ class ServiceBaseClass {
|
|||||||
await page.waitForSelector('[data-testid="cron-type"]');
|
await page.waitForSelector('[data-testid="cron-type"]');
|
||||||
await page.click('[data-testid="cron-type"]');
|
await page.click('[data-testid="cron-type"]');
|
||||||
await page.waitForSelector('.ant-select-item-option-content');
|
await page.waitForSelector('.ant-select-item-option-content');
|
||||||
await page.click('.ant-select-item-option-content:has-text("Hour")');
|
await page.click('.ant-select-item-option-content:has-text("None")');
|
||||||
|
|
||||||
const deployPipelinePromise = page.waitForRequest(
|
const deployPipelinePromise = page.waitForRequest(
|
||||||
`/api/v1/services/ingestionPipelines/deploy/**`
|
`/api/v1/services/ingestionPipelines/deploy/**`
|
||||||
@ -221,20 +221,34 @@ class ServiceBaseClass {
|
|||||||
// Queued status are not stored in DB. cc: @ulixius9
|
// Queued status are not stored in DB. cc: @ulixius9
|
||||||
await page.waitForTimeout(2000);
|
await page.waitForTimeout(2000);
|
||||||
|
|
||||||
|
const response = await apiContext
|
||||||
|
.get(
|
||||||
|
`/api/v1/services/ingestionPipelines?fields=pipelineStatuses&service=${
|
||||||
|
this.serviceName
|
||||||
|
}&pipelineType=${ingestionType}&serviceType=${getServiceCategoryFromService(
|
||||||
|
this.category
|
||||||
|
)}`
|
||||||
|
)
|
||||||
|
.then((res) => res.json());
|
||||||
|
|
||||||
|
const workflowData = response.data.filter(
|
||||||
|
(d) => d.pipelineType === ingestionType
|
||||||
|
)[0];
|
||||||
|
|
||||||
|
const oneHourBefore = Date.now() - 86400000;
|
||||||
|
|
||||||
await expect
|
await expect
|
||||||
.poll(
|
.poll(
|
||||||
async () => {
|
async () => {
|
||||||
const response = await apiContext
|
const response = await apiContext
|
||||||
.get(
|
.get(
|
||||||
`/api/v1/services/ingestionPipelines?fields=pipelineStatuses&service=${
|
`/api/v1/services/ingestionPipelines/${encodeURIComponent(
|
||||||
this.serviceName
|
workflowData.fullyQualifiedName
|
||||||
}&pipelineType=${ingestionType}&serviceType=${getServiceCategoryFromService(
|
)}/pipelineStatus?startTs=${oneHourBefore}&endTs=${Date.now()}`
|
||||||
this.category
|
|
||||||
)}`
|
|
||||||
)
|
)
|
||||||
.then((res) => res.json());
|
.then((res) => res.json());
|
||||||
|
|
||||||
return response.data[0]?.pipelineStatuses?.pipelineState;
|
return response.data[0]?.pipelineState;
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// Custom expect message for reporting, optional.
|
// Custom expect message for reporting, optional.
|
||||||
@ -243,7 +257,8 @@ class ServiceBaseClass {
|
|||||||
intervals: [30_000, 15_000, 5_000],
|
intervals: [30_000, 15_000, 5_000],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
.toBe('success');
|
// To allow partial success
|
||||||
|
.toContain('success');
|
||||||
|
|
||||||
const pipelinePromise = page.waitForRequest(
|
const pipelinePromise = page.waitForRequest(
|
||||||
`/api/v1/services/ingestionPipelines?**`
|
`/api/v1/services/ingestionPipelines?**`
|
||||||
@ -264,9 +279,12 @@ class ServiceBaseClass {
|
|||||||
await page.click('[data-testid="ingestions"]');
|
await page.click('[data-testid="ingestions"]');
|
||||||
await page.waitForSelector(`td:has-text("${ingestionType}")`);
|
await page.waitForSelector(`td:has-text("${ingestionType}")`);
|
||||||
|
|
||||||
await expect(page.getByTestId('pipeline-status').last()).toContainText(
|
await expect(
|
||||||
'SUCCESS'
|
page
|
||||||
);
|
.locator(`[data-row-key*="${workflowData.name}"]`)
|
||||||
|
.getByTestId('pipeline-status')
|
||||||
|
.last()
|
||||||
|
).toContainText('SUCCESS');
|
||||||
};
|
};
|
||||||
|
|
||||||
async updateService(page: Page) {
|
async updateService(page: Page) {
|
||||||
@ -442,6 +460,7 @@ class ServiceBaseClass {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async runAdditionalTests(
|
async runAdditionalTests(
|
||||||
|
_page: Page,
|
||||||
_test: TestType<PlaywrightTestArgs, PlaywrightWorkerArgs>
|
_test: TestType<PlaywrightTestArgs, PlaywrightWorkerArgs>
|
||||||
) {
|
) {
|
||||||
// Write service specific tests
|
// Write service specific tests
|
||||||
|
@ -22,7 +22,12 @@ import ServiceBaseClass from './ServiceBaseClass';
|
|||||||
class SnowflakeIngestionClass extends ServiceBaseClass {
|
class SnowflakeIngestionClass extends ServiceBaseClass {
|
||||||
schema: string;
|
schema: string;
|
||||||
constructor() {
|
constructor() {
|
||||||
super(Services.Database, `pw-snowflake-${uuid()}`, 'Snowflake', 'CUSTOMER');
|
super(
|
||||||
|
Services.Database,
|
||||||
|
`pw-snowflake-with-%-${uuid()}`,
|
||||||
|
'Snowflake',
|
||||||
|
'CUSTOMER'
|
||||||
|
);
|
||||||
this.schema = 'TPCH_SF1000';
|
this.schema = 'TPCH_SF1000';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ class SupersetIngestionClass extends ServiceBaseClass {
|
|||||||
constructor() {
|
constructor() {
|
||||||
super(
|
super(
|
||||||
Services.Dashboard,
|
Services.Dashboard,
|
||||||
`pw-Superset-${uuid()}`,
|
`pw-Superset-with-%-${uuid()}`,
|
||||||
'Superset',
|
'Superset',
|
||||||
"World Bank's Data"
|
"World Bank's Data"
|
||||||
);
|
);
|
||||||
|
@ -118,7 +118,7 @@ export const testConnection = async (page: Page) => {
|
|||||||
|
|
||||||
await page.waitForSelector('[data-testid="success-badge"]', {
|
await page.waitForSelector('[data-testid="success-badge"]', {
|
||||||
state: 'attached',
|
state: 'attached',
|
||||||
timeout: 2 * 60 * 1000,
|
timeout: 2.5 * 60 * 1000,
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(page.getByTestId('messag-text')).toContainText(
|
await expect(page.getByTestId('messag-text')).toContainText(
|
||||||
|
@ -17,6 +17,10 @@ import {
|
|||||||
} from '../constant/settings';
|
} from '../constant/settings';
|
||||||
import { SidebarItem, SIDEBAR_LIST_ITEMS } from '../constant/sidebar';
|
import { SidebarItem, SIDEBAR_LIST_ITEMS } from '../constant/sidebar';
|
||||||
|
|
||||||
|
export type SettingOptionsType =
|
||||||
|
| keyof typeof SETTINGS_OPTIONS_PATH
|
||||||
|
| keyof typeof SETTING_CUSTOM_PROPERTIES_PATH;
|
||||||
|
|
||||||
export const clickOnLogo = async (page: Page) => {
|
export const clickOnLogo = async (page: Page) => {
|
||||||
await page.click('#openmetadata_logo > [data-testid="image"]');
|
await page.click('#openmetadata_logo > [data-testid="image"]');
|
||||||
await page.mouse.move(1280, 0); // Move mouse to top right corner
|
await page.mouse.move(1280, 0); // Move mouse to top right corner
|
||||||
@ -38,7 +42,7 @@ export const sidebarClick = async (page: Page, id: string) => {
|
|||||||
|
|
||||||
export const settingClick = async (
|
export const settingClick = async (
|
||||||
page: Page,
|
page: Page,
|
||||||
dataTestId: string,
|
dataTestId: SettingOptionsType,
|
||||||
isCustomProperty?: boolean
|
isCustomProperty?: boolean
|
||||||
) => {
|
) => {
|
||||||
let paths = SETTINGS_OPTIONS_PATH[dataTestId];
|
let paths = SETTINGS_OPTIONS_PATH[dataTestId];
|
||||||
|
Loading…
x
Reference in New Issue
Block a user