Remove partition fields from Bigquery (#11195)

Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>

bootstrap/sql/com.mysql.cj.jdbc.Driver/v009__create_db_connection_info.sql

@@ -206,7 +206,13 @@ UPDATE dashboard_service_entity
 SET json = JSON_REMOVE(json, '$.connection.config.connectionOptions')
 WHERE serviceType = 'Superset';
 
+-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
+UPDATE dbservice_entity 
+SET json = JSON_REMOVE(json, '$.connection.config.partitionQueryDuration', '$.connection.config.partitionQuery', '$.connection.config.partitionField')
+WHERE serviceType = 'BigQuery'; 
+
 -- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
 UPDATE dbservice_entity 
 SET json = JSON_REMOVE(json, '$.connection.config.scheme', '$.connection.config.hostPort', '$.connection.config.supportsProfiler', '$.connection.config.supportsQueryComment')
-WHERE serviceType = 'Salesforce';
+WHERE serviceType = 'Salesforce';
+

bootstrap/sql/org.postgresql.Driver/v009__create_db_connection_info.sql

@@ -206,7 +206,13 @@ UPDATE dashboard_service_entity
 SET json = json::jsonb #- '{connection,config,connectionOptions}'
 WHERE serviceType = 'Superset';
 
+-- Delete partitionQueryDuration, partitionQuery, partitionField from bigquery
+UPDATE dbservice_entity
+SET json = json::jsonb #- '{connection,config,partitionQueryDuration}' #- '{connection,config,partitionQuery}' #- '{connection,config,partitionField}'
+WHERE serviceType = 'BigQuery';
+
 -- Delete supportsQueryComment, scheme, hostPort, supportsProfiler from salesforce
 UPDATE dbservice_entity
 SET json = json::jsonb #- '{connection,config,supportsQueryComment}' #- '{connection,config,scheme}' #- '{connection,config,hostPort}' #- '{connection,config,supportsProfiler}'
-WHERE serviceType = 'Salesforce';
+WHERE serviceType = 'Salesforce';
+

openmetadata-docs-v1/content/v0.13.2/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md

@@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab
 - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`.
 - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*.
 - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`.
-- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`.
-- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`.
-- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`.
 - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`.
 - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`.
 - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*.

openmetadata-docs-v1/content/v1.0.0/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md

@@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab
 - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`.
 - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*.
 - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`.
-- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`.
-- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`.
-- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`.
 - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`.
 - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`.
 - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*.

openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/services/connections/database/bigQueryConnection.md

@@ -14,9 +14,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/services/connections/datab
 - **`hostPort`** *(string)*: BigQuery APIs URL. Default: `bigquery.googleapis.com`.
 - **`credentials`**: GCS Credentials. Refer to *../../../../security/credentials/gcsCredentials.json*.
 - **`tagCategoryName`** *(string)*: Custom OpenMetadata Tag category name for BigQuery policy tags. Default: `BigqueryPolicyTags`.
-- **`partitionQueryDuration`** *(integer)*: Duration for partitioning BigQuery tables. Default: `1`.
-- **`partitionQuery`** *(string)*: Partitioning query for BigQuery tables. Default: `select * from {}.{} WHERE {} = "{}" LIMIT 1000`.
-- **`partitionField`** *(string)*: Column name on which the BigQuery table will be partitioned. Default: `_PARTITIONTIME`.
 - **`taxonomyLocation`** *(string)*: Taxonomy location used to fetch policy tags. Default: `us`.
 - **`usageLocation`** *(string)*: Location used to query INFORMATION_SCHEMA.JOBS_BY_PROJECT to fetch usage data. You can pass multi-regions, such as `us` or `eu`, or you specific region. Australia and Asia multi-regions are not yet in GA. Default: `us`.
 - **`connectionOptions`**: Refer to *../connectionBasicType.json#/definitions/connectionOptions*.

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json

@@ -47,24 +47,6 @@
       "description": "GCS Credentials",
       "$ref": "../../../../security/credentials/gcsCredentials.json"
     },
-    "partitionQueryDuration": {
-      "title": "Partition Query Duration",
-      "description": "Duration for partitioning BigQuery tables.",
-      "type": "integer",
-      "default": 1
-    },
-    "partitionQuery": {
-      "title": "Partition Query",
-      "description": "Partitioning query for BigQuery tables.",
-      "type": "string",
-      "default": "select * from {}.{} WHERE {} = \"{}\" LIMIT 1000"
-    },
-    "partitionField": {
-      "title": "Partition Field",
-      "description": "Column name on which the BigQuery table will be partitioned.",
-      "type": "string",
-      "default": "_PARTITIONTIME"
-    },
     "taxonomyProjectID": {
       "title": "Taxonomy Project IDs",
       "description": "Project IDs used to fetch policy tags",

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md

@@ -132,27 +132,6 @@ $$section
 This is the URL of the certificate that verifies the authenticity of the service account. To fetch this key, look for the value associated with the `client_x509_cert_url` key in the service account key  file.
 $$
 
-$$section
-### Partition Query Duration $(id="partitionQueryDuration")
-
-Duration for partitioning BigQuery tables.
-<!-- partitionQueryDuration to be updated -->
-$$
-
-$$section
-### Partition Query $(id="partitionQuery")
-
-Partitioning query for BigQuery tables.
-<!-- partitionQuery to be updated -->
-$$
-
-$$section
-### Partition Field $(id="partitionField")
-
-Column name on which the BigQuery table will be partitioned.
-<!-- partitionField to be updated -->
-$$
-
 $$section
 ### Taxonomy Project ID $(id="taxonomyProjectID")
 

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.mock.ts

@@ -40,13 +40,10 @@ export const mockIngestionWorkFlow = {
               type: 'BigQuery',
               scheme: 'bigquery',
               hostPort: 'bigquery.googleapis.com',
-              partitionField: '_PARTITIONTIME',
-              partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000',
               tagCategoryName: 'BigqueryPolicyTags',
               connectionOptions: {},
               connectionArguments: {},
               enablePolicyTagImport: true,
-              partitionQueryDuration: 1,
               supportsUsageExtraction: true,
               supportsMetadataExtraction: true,
             },
@@ -112,13 +109,10 @@ export const mockService = {
       type: 'BigQuery',
       scheme: 'bigquery',
       hostPort: 'bigquery.googleapis.com',
-      partitionField: '_PARTITIONTIME',
-      partitionQuery: 'select * from {}.{} WHERE {} = "{}" LIMIT 1000',
       tagCategoryName: 'BigqueryPolicyTags',
       connectionOptions: {},
       connectionArguments: {},
       enablePolicyTagImport: true,
-      partitionQueryDuration: 1,
       supportsUsageExtraction: true,
       supportsMetadataExtraction: true,
     },