From 5754b1424ada7a0c9848e778b95222f6541f985c Mon Sep 17 00:00:00 2001
From: Mayur Singal <39544459+ulixius9@users.noreply.github.com>
Date: Thu, 4 Aug 2022 13:59:45 +0530
Subject: [PATCH] Bigquery Docs Min Requirements (#6561)

---
 ingestion/src/metadata/utils/sql_queries.py   |  4 ++--
 .../connectors/database/bigquery/airflow.md   | 18 ++++++++++++++++++
 .../connectors/database/bigquery/cli.md       | 19 +++++++++++++++++++
 .../connectors/database/bigquery/index.md     | 18 ++++++++++++++++++
 .../usage/usage-workflow-query-logs.md        |  3 +--
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/ingestion/src/metadata/utils/sql_queries.py b/ingestion/src/metadata/utils/sql_queries.py
index 0a17c7476ab..6d608999ef6 100644
--- a/ingestion/src/metadata/utils/sql_queries.py
+++ b/ingestion/src/metadata/utils/sql_queries.py
@@ -400,8 +400,8 @@ WHERE creation_time BETWEEN "{start_time}" AND "{end_time}"
   AND job_type = "QUERY"
   AND state = "DONE"
   AND IFNULL(statement_type, "NO") not in ("NO", "DROP_TABLE", "CREATE_TABLE")
-  AND text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
-  AND text NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+  AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+  AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%'
 """
 
 
diff --git a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/airflow.md b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/airflow.md
index 3a153382ef1..9696b4a0f51 100644
--- a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/airflow.md
+++ b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/airflow.md
@@ -7,6 +7,24 @@ slug: /openmetadata/connectors/database/bigquery/airflow
 
 <Requirements />
 
+<h4>GCP Permissions</h4>
+
+<p> To execute metadata extraction and usage workflow successfully the user or the service account should have enough access to fetch required data. Following table describes the minimum required permissions </p>
+
+| # | GCP Permission | GCP Role | Required For |
+| :---------- | :---------- | :---------- | :---------- |
+| 1 | bigquery.datasets.get | BigQuery Data Viewer | Metadata Ingestion |
+| 2 | bigquery.tables.get | BigQuery Data Viewer | Metadata Ingestion |
+| 3 | bigquery.tables.getData | BigQuery Data Viewer | Metadata Ingestion |
+| 4 | bigquery.tables.list | BigQuery Data Viewer | Metadata Ingestion |
+| 5 | resourcemanager.projects.get | BigQuery Data Viewer | Metadata Ingestion |
+| 6 | bigquery.jobs.create | BigQuery Job User | Metadata Ingestion |
+| 7 | bigquery.jobs.listAll | BigQuery Job User | Metadata Ingestion |
+| 8 | datacatalog.taxonomies.get | BigQuery Policy Admin | Fetch Policy Tags |
+| 9 | datacatalog.taxonomies.list | BigQuery Policy Admin | Fetch Policy Tags |
+| 10 | bigquery.readsessions.create | BigQuery Admin | Bigquery Usage Workflow |
+| 11 | bigquery.readsessions.getData | BigQuery Admin | Bigquery Usage Workflow |
+
 <MetadataIngestionServiceDev service="database" connector="BigQuery" goal="Airflow"/>
 
 <h4>Source Configuration - Service Connection</h4>
diff --git a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/cli.md b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/cli.md
index ae51607f1aa..6ceb6608c08 100644
--- a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/cli.md
+++ b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/cli.md
@@ -7,6 +7,25 @@ slug: /openmetadata/connectors/database/bigquery/cli
 
 <Requirements />
 
+<h4>GCP Permissions</h4>
+
+<p> To execute metadata extraction and usage workflow successfully the user or the service account should have enough access to fetch required data. Following table describes the minimum required permissions </p>
+
+| # | GCP Permission | GCP Role | Required For |
+| :---------- | :---------- | :---------- | :---------- |
+| 1 | bigquery.datasets.get | BigQuery Data Viewer | Metadata Ingestion |
+| 2 | bigquery.tables.get | BigQuery Data Viewer | Metadata Ingestion |
+| 3 | bigquery.tables.getData | BigQuery Data Viewer | Metadata Ingestion |
+| 4 | bigquery.tables.list | BigQuery Data Viewer | Metadata Ingestion |
+| 5 | resourcemanager.projects.get | BigQuery Data Viewer | Metadata Ingestion |
+| 6 | bigquery.jobs.create | BigQuery Job User | Metadata Ingestion |
+| 7 | bigquery.jobs.listAll | BigQuery Job User | Metadata Ingestion |
+| 8 | datacatalog.taxonomies.get | BigQuery Policy Admin | Fetch Policy Tags |
+| 9 | datacatalog.taxonomies.list | BigQuery Policy Admin | Fetch Policy Tags |
+| 10 | bigquery.readsessions.create | BigQuery Admin | Bigquery Usage Workflow |
+| 11 | bigquery.readsessions.getData | BigQuery Admin | Bigquery Usage Workflow |
+
+
 <MetadataIngestionServiceDev service="database" connector="BigQuery" goal="CLI"/>
 
 <h4>Source Configuration - Service Connection</h4>
diff --git a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/index.md b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/index.md
index dcdc9d586be..a868042bd02 100644
--- a/openmetadata-docs/content/openmetadata/connectors/database/bigquery/index.md
+++ b/openmetadata-docs/content/openmetadata/connectors/database/bigquery/index.md
@@ -7,6 +7,24 @@ slug: /openmetadata/connectors/database/bigquery
 
 <Requirements />
 
+<h4>GCP Permissions</h4>
+
+<p> To execute metadata extraction and usage workflow successfully the user or the service account should have enough access to fetch required data. Following table describes the minimum required permissions </p>
+
+| # | GCP Permission | GCP Role | Required For |
+| :---------- | :---------- | :---------- | :---------- |
+| 1 | bigquery.datasets.get | BigQuery Data Viewer | Metadata Ingestion |
+| 2 | bigquery.tables.get | BigQuery Data Viewer | Metadata Ingestion |
+| 3 | bigquery.tables.getData | BigQuery Data Viewer | Metadata Ingestion |
+| 4 | bigquery.tables.list | BigQuery Data Viewer | Metadata Ingestion |
+| 5 | resourcemanager.projects.get | BigQuery Data Viewer | Metadata Ingestion |
+| 6 | bigquery.jobs.create | BigQuery Job User | Metadata Ingestion |
+| 7 | bigquery.jobs.listAll | BigQuery Job User | Metadata Ingestion |
+| 8 | datacatalog.taxonomies.get | BigQuery Policy Admin | Fetch Policy Tags |
+| 9 | datacatalog.taxonomies.list | BigQuery Policy Admin | Fetch Policy Tags |
+| 10 | bigquery.readsessions.create | BigQuery Admin | Bigquery Usage Workflow |
+| 11 | bigquery.readsessions.getData | BigQuery Admin | Bigquery Usage Workflow |
+
 <MetadataIngestionService connector="BigQuery"/>
 
 <h4>Connection Options</h4>
diff --git a/openmetadata-docs/content/openmetadata/ingestion/workflows/usage/usage-workflow-query-logs.md b/openmetadata-docs/content/openmetadata/ingestion/workflows/usage/usage-workflow-query-logs.md
index 11e067ce8a2..3a6e7009a82 100644
--- a/openmetadata-docs/content/openmetadata/ingestion/workflows/usage/usage-workflow-query-logs.md
+++ b/openmetadata-docs/content/openmetadata/ingestion/workflows/usage/usage-workflow-query-logs.md
@@ -50,8 +50,7 @@ source:
       queryLogFilePath: <path to query log file>
 processor:
   type: query-parser
-  config:
-    filter: ''
+  config: {}
 stage:
   type: table-usage
   config: