diff --git a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql index 9cc8f339d82..bd3f4757de9 100644 --- a/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql @@ -206,7 +206,13 @@ UPDATE dashboard_service_entity SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions') WHERE serviceType = 'Superset'; +-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery +UPDATE dbservice_entity +SET json = JSON_REMOVE(json, '$.connection.config.partitionQueryDuration', '$.connection.config.partitionQuery', '$.connection.config.partitionField') +WHERE serviceType = 'BigQuery'; + -- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce UPDATE dbservice_entity SET json = JSON_REMOVE(json, '$.connection.config.scheme', '$.connection.config.hostPort', '$.connection.config.supportsProfiler', '$.connection.config.supportsQueryComment') -WHERE serviceType = 'Salesforce'; \ No newline at end of file +WHERE serviceType = 'Salesforce'; + diff --git a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql index 6413769060d..d1aa4f573b4 100644 --- a/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql +++ b/bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql @@ -206,7 +206,13 @@ UPDATE dashboard_service_entity SET json = json::jsonb #- '{connection,config,connectionOptions}' WHERE serviceType = 'Superset'; +-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery +UPDATE dbservice_entity +SET json = json::jsonb #- '{connection,config,partitionQueryDuration}' #- '{connection,config,partitionQuery}' #- '{connection,config,partitionField}' +WHERE serviceType = 'BigQuery'; + -- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce UPDATE dbservice_entity SET json = json::jsonb #- '{connection,config,supportsQueryComment}' #- '{connection,config,scheme}' #- '{connection,config,hostPort}' #- '{connection,config,supportsProfiler}' -WHERE serviceType = 'Salesforce'; \ No newline at end of file +WHERE serviceType = 'Salesforce'; + diff --git a/openmetadata-docs-v1/content/v0.13.2/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md b/openmetadata-docs-v1/content/v0.13.2/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md index 7df2b60e3d9..dd3a8dca25a 100644 --- a/openmetadata-docs-v1/content/v0.13.2/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md +++ b/openmetadata-docs-v1/content/v0.13.2/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md @@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`. - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*. - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`. -- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`. -- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`. -- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`. - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`. - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`. - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*. diff --git a/openmetadata-docs-v1/content/v1.0.0/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md b/openmetadata-docs-v1/content/v1.0.0/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md index 7df2b60e3d9..dd3a8dca25a 100644 --- a/openmetadata-docs-v1/content/v1.0.0/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md +++ b/openmetadata-docs-v1/content/v1.0.0/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md @@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`. - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*. - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`. -- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`. -- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`. -- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`. - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`. - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`. - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*. diff --git a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md index 7df2b60e3d9..dd3a8dca25a 100644 --- a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md +++ b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md @@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`. - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*. - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`. -- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`. -- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`. -- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`. - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`. - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`. - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*. diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json index 4ed20309190..24391aea1fb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json @@ -47,24 +47,6 @@ "description": "GCS Credentials", "$ref": "../../../../security/credentials/gcsCredentials.json" }, - "partitionQueryDuration": { - "title": "Partition Query Duration", - "description": "Duration for partitioning BigQuery tables.", - "type": "integer", - "default": 1 - }, - "partitionQuery": { - "title": "Partition Query", - "description": "Partitioning query for BigQuery tables.", - "type": "string", - "default": "select * from {}.{} WHERE {} = \"{}\" LIMIT 1000" - }, - "partitionField": { - "title": "Partition Field", - "description": "Column name on which the BigQuery table will be partitioned.", - "type": "string", - "default": "_PARTITIONTIME" - }, "taxonomyProjectID": { "title": "Taxonomy Project IDs", "description": "Project IDs used to fetch policy tags", diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md index 7ad56e31cee..f82c6e82b9d 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md @@ -132,27 +132,6 @@ $$section This is the URL of the certificate that verifies the authenticity of the service account. To fetch this key, look for the value associated with the `client_x509_cert_url` key in the service account key file. $$ -$$section -### Partition Query Duration $(id="partitionQueryDuration") - -Duration for partitioning BigQuery tables. - -$$ - -$$section -### Partition Query $(id="partitionQuery") - -Partitioning query for BigQuery tables. - -$$ - -$$section -### Partition Field $(id="partitionField") - -Column name on which the BigQuery table will be partitioned. - -$$ - $$section ### Taxonomy Project ID $(id="taxonomyProjectID") diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts index e056733794b..78fa2d8be49 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts @@ -40,13 +40,10 @@ export const mockIngestionWorkFlow = { type: 'BigQuery', scheme: 'bigquery', hostPort: 'bigquery.googleapis.com', - partitionField: '_PARTITIONTIME', - partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000', tagCategoryName: 'BigqueryPolicyTags', connectionOptions: {}, connectionArguments: {}, enablePolicyTagImport: true, - partitionQueryDuration: 1, supportsUsageExtraction: true, supportsMetadataExtraction: true, }, @@ -112,13 +109,10 @@ export const mockService = { type: 'BigQuery', scheme: 'bigquery', hostPort: 'bigquery.googleapis.com', - partitionField: '_PARTITIONTIME', - partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000', tagCategoryName: 'BigqueryPolicyTags', connectionOptions: {}, connectionArguments: {}, enablePolicyTagImport: true, - partitionQueryDuration: 1, supportsUsageExtraction: true, supportsMetadataExtraction: true, },