From ad35ab94d2102c6ff897f3ea75e723f6de56f83b Mon Sep 17 00:00:00 2001
From: Ayush Shah <ayush.shah@deuexsolutions.com>
Date: Mon, 18 Oct 2021 17:39:46 +0000
Subject: [PATCH] GitBook: [#199] Added Bigquery Usage Documentation

---
 docs/SUMMARY.md                               |   1 +
 .../connectors/database-services/README.md    |   4 +
 .../database-services/bigquery-usage.md       | 125 ++++++++++++++++++
 3 files changed, 130 insertions(+)
 create mode 100644 docs/install/metadata-ingestion/connectors/database-services/bigquery-usage.md

diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md
index 3ed90cbbf7e..90436a0b1de 100644
--- a/docs/SUMMARY.md
+++ b/docs/SUMMARY.md
@@ -75,6 +75,7 @@
     * [Database Services](install/metadata-ingestion/connectors/database-services/README.md)
       * [Athena](install/metadata-ingestion/connectors/database-services/athena.md)
       * [BigQuery](install/metadata-ingestion/connectors/database-services/bigquery.md)
+      * [BigQuery Usage](install/metadata-ingestion/connectors/database-services/bigquery-usage.md)
       * [ElasticSearch](install/metadata-ingestion/connectors/database-services/elastic-search.md)
       * [MsSQL](install/metadata-ingestion/connectors/database-services/mssql.md)
       * [MySQL](install/metadata-ingestion/connectors/database-services/mysql.md)
diff --git a/docs/install/metadata-ingestion/connectors/database-services/README.md b/docs/install/metadata-ingestion/connectors/database-services/README.md
index 9fef2ec989a..a6d47b48c65 100644
--- a/docs/install/metadata-ingestion/connectors/database-services/README.md
+++ b/docs/install/metadata-ingestion/connectors/database-services/README.md
@@ -10,6 +10,10 @@ OpenMetadata supports connectors to some popular data sources. We will continue
 [bigquery.md](bigquery.md)
 {% endcontent-ref %}
 
+{% content-ref url="bigquery-usage.md" %}
+[bigquery-usage.md](bigquery-usage.md)
+{% endcontent-ref %}
+
 {% content-ref url="elastic-search.md" %}
 [elastic-search.md](elastic-search.md)
 {% endcontent-ref %}
diff --git a/docs/install/metadata-ingestion/connectors/database-services/bigquery-usage.md b/docs/install/metadata-ingestion/connectors/database-services/bigquery-usage.md
new file mode 100644
index 00000000000..0df45f420e0
--- /dev/null
+++ b/docs/install/metadata-ingestion/connectors/database-services/bigquery-usage.md
@@ -0,0 +1,125 @@
+---
+description: This guide will help install BigQuery Usage connector and run manually
+---
+
+# BigQuery Usage
+
+{% hint style="info" %}
+**Prerequisites**
+
+OpenMetadata is built using Java, DropWizard, Jetty, and MySQL.
+
+1. Python 3.7 or above
+{% endhint %}
+
+### Install from PyPI
+
+{% tabs %}
+{% tab title="Install Using PyPI" %}
+```bash
+pip install 'openmetadata-ingestion[bigquery-usage]'
+```
+{% endtab %}
+{% endtabs %}
+
+## Run Manually
+
+```bash
+metadata ingest -c ./examples/workflows/bigquery_usage.json
+```
+
+### Configuration
+
+{% code title="bigquery-creds.json (boilerplate)" %}
+```javascript
+{
+  "type": "service_account",
+  "project_id": "project_id",
+  "private_key_id": "private_key_id",
+  "private_key": "",
+  "client_email": "gcpuser@project_id.iam.gserviceaccount.com",
+  "client_id": "",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": ""
+}
+```
+{% endcode %}
+
+{% code title="bigquery_usage.json" %}
+```javascript
+{
+  "source": {
+    "type": "bigquery-usage",
+    "config": {
+      "project_id": "project_id",
+      "host_port": "https://bigquery.googleapis.com",
+      "username": "gcpuser@project_id.iam.gserviceaccount.com",
+      "service_name": "gcp_bigquery",
+      "duration": 2,
+      "options": {
+        "credentials_path": "examples/creds/bigquery-cred.json"
+      }
+    }
+  },
+```
+{% endcode %}
+
+1. **username** - pass the Bigquery username.
+2. **password** - the password for the Bigquery username.
+3. **service_name** - Service Name for this Bigquery cluster. If you added the Bigquery cluster through OpenMetadata UI, make sure the service name matches the same.
+4. **filter_pattern** - It contains includes, excludes options to choose which pattern of datasets you want to ingest into OpenMetadata.
+5. **database -** Database name from where data is to be fetched.
+
+### Publish to OpenMetadata
+
+Below is the configuration to publish Bigquery data into the OpenMetadata service.
+
+Add optionally`query-parser` processor, `table-usage` stage and `metadata-usage` bulk_sink along with `metadata-server` config
+
+{% code title="bigquery_usage.json" %}
+```javascript
+{
+  "source": {
+    "type": "bigquery-usage",
+    "config": {
+      "project_id": "project_id",
+      "host_port": "https://bigquery.googleapis.com",
+      "username": "gcpuser@project_id.iam.gserviceaccount.com",
+      "service_name": "gcp_bigquery",
+      "duration": 2,
+      "options": {
+        "credentials_path": "examples/creds/bigquery-cred.json"
+      }
+    }
+  },
+  "processor": {
+    "type": "query-parser",
+    "config": {
+      "filter": ""
+    }
+  },
+  "stage": {
+    "type": "table-usage",
+    "config": {
+      "filename": "/tmp/bigquery_usage"
+    }
+  },
+  "bulk_sink": {
+    "type": "metadata-usage",
+    "config": {
+      "filename": "/tmp/bigquery_usage"
+    }
+  },
+  "metadata_server": {
+    "type": "metadata-server",
+    "config": {
+      "api_endpoint": "http://localhost:8585/api",
+      "auth_provider_type": "no-auth"
+    }
+  }
+}
+
+```
+{% endcode %}