mirror of
https://github.com/datahub-project/datahub.git
synced 2025-09-03 06:13:14 +00:00
1 line
11 KiB
JavaScript
1 line
11 KiB
JavaScript
"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[24442],{15680:(e,t,r)=>{r.d(t,{xA:()=>l,yg:()=>d});var o=r(96540);function i(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function a(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?n(Object(r),!0).forEach((function(t){i(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):n(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function c(e,t){if(null==e)return{};var r,o,i=function(e,t){if(null==e)return{};var r,o,i={},n=Object.keys(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var s=o.createContext({}),u=function(e){var t=o.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},l=function(e){var t=u(e.components);return o.createElement(s.Provider,{value:t},e.children)},g="mdxType",p={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},y=o.forwardRef((function(e,t){var r=e.components,i=e.mdxType,n=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),g=u(r),y=i,d=g["".concat(s,".").concat(y)]||g[y]||p[y]||n;return r?o.createElement(d,a(a({ref:t},l),{},{components:r})):o.createElement(d,a({ref:t},l))}));function d(e,t){var r=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var n=r.length,a=new Array(n);a[0]=y;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c[g]="string"==typeof e?e:i,a[1]=c;for(var u=2;u<n;u++)a[u]=r[u];return o.createElement.apply(null,a)}return o.createElement.apply(null,r)}y.displayName="MDXCreateElement"},33525:(e,t,r)=>{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>c,metadata:()=>u,toc:()=>g});r(96540);var o=r(15680);function i(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){return t=null!=t?t:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):function(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}(Object(t)).forEach((function(r){Object.defineProperty(e,r,Object.getOwnPropertyDescriptor(t,r))})),e}function a(e,t){if(null==e)return{};var r,o,i=function(e,t){if(null==e)return{};var r,o,i={},n=Object.keys(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}const c={title:"Setup",sidebar_label:"Setup",slug:"/quick-ingestion-guides/bigquery/setup",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/setup.md"},s="BigQuery Ingestion Guide: Setup & Prerequisites",u={unversionedId:"docs/quick-ingestion-guides/bigquery/setup",id:"version-1.1.0/docs/quick-ingestion-guides/bigquery/setup",title:"Setup",description:"To configure ingestion from BigQuery, you'll need a Service Account configured with the proper permission sets and an associated Service Account Key.",source:"@site/versioned_docs/version-1.1.0/docs/quick-ingestion-guides/bigquery/setup.md",sourceDirName:"docs/quick-ingestion-guides/bigquery",slug:"/quick-ingestion-guides/bigquery/setup",permalink:"/docs/1.1.0/quick-ingestion-guides/bigquery/setup",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/setup.md",tags:[],version:"1.1.0",frontMatter:{title:"Setup",sidebar_label:"Setup",slug:"/quick-ingestion-guides/bigquery/setup",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/setup.md"},sidebar:"overviewSidebar",previous:{title:"Overview",permalink:"/docs/1.1.0/quick-ingestion-guides/bigquery/overview"},next:{title:"Configuration",permalink:"/docs/1.1.0/quick-ingestion-guides/bigquery/configuration"}},l={},g=[{value:"BigQuery Prerequisites",id:"bigquery-prerequisites",level:2},{value:"BigQuery Setup",id:"bigquery-setup",level:2},{value:"Next Steps",id:"next-steps",level:2}],p={toc:g},y="wrapper";function d(e){var{components:t}=e,r=a(e,["components"]);return(0,o.yg)(y,n(function(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{},o=Object.keys(r);"function"==typeof Object.getOwnPropertySymbols&&(o=o.concat(Object.getOwnPropertySymbols(r).filter((function(e){return Object.getOwnPropertyDescriptor(r,e).enumerable})))),o.forEach((function(t){i(e,t,r[t])}))}return e}({},p,r),{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h1",{id:"bigquery-ingestion-guide-setup--prerequisites"},"BigQuery Ingestion Guide: Setup & Prerequisites"),(0,o.yg)("p",null,"To configure ingestion from BigQuery, you'll need a ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/creating-managing-service-accounts"},"Service Account")," configured with the proper permission sets and an associated ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/creating-managing-service-account-keys"},"Service Account Key"),"."),(0,o.yg)("p",null,"This setup guide will walk you through the steps you'll need to take via your Google Cloud Console."),(0,o.yg)("h2",{id:"bigquery-prerequisites"},"BigQuery Prerequisites"),(0,o.yg)("p",null,"If you do not have an existing Service Account and Service Account Key, please work with your BigQuery Admin to ensure you have the appropriate permissions and/or roles to continue with this setup guide."),(0,o.yg)("p",null,"When creating and managing new Service Accounts and Service Account Keys, we have found the following permissions and roles to be required:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Create a Service Account: ",(0,o.yg)("inlineCode",{parentName:"li"},"iam.serviceAccounts.create")," permission"),(0,o.yg)("li",{parentName:"ul"},"Assign roles to a Service Account: ",(0,o.yg)("inlineCode",{parentName:"li"},"serviceusage.services.enable")," permission"),(0,o.yg)("li",{parentName:"ul"},"Set permission policy to the project: ",(0,o.yg)("inlineCode",{parentName:"li"},"resourcemanager.projects.setIamPolicy")," permission"),(0,o.yg)("li",{parentName:"ul"},"Generate Key for Service Account: Service Account Key Admin (",(0,o.yg)("inlineCode",{parentName:"li"},"roles/iam.serviceAccountKeyAdmin"),") IAM role")),(0,o.yg)("admonition",{type:"note"},(0,o.yg)("p",{parentName:"admonition"},"Please refer to the BigQuery ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/permissions-reference"},"Permissions")," and ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/understanding-roles"},"IAM Roles")," references for details")),(0,o.yg)("h2",{id:"bigquery-setup"},"BigQuery Setup"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"To set up a new Service Account follow ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/creating-managing-service-accounts"},"this guide"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"When you are creating a Service Account, assign the following predefined Roles:"),(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/bigquery/docs/access-control#bigquery.jobUser"},"BigQuery Job User")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer"},"BigQuery Metadata Viewer")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer"},"BigQuery Resource Viewer")," -> This role is for Table-Level Lineage and Usage extraction"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/logging/docs/access-control#logging.viewAccessor"},"Logs View Accessor")," -> This role is for Table-Level Lineage and Usage extraction"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer"},"BigQuery Data Viewer")," -> This role is for Profiling"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://cloud.google.com/bigquery/docs/access-control#bigquery.readSessionUser"},"BigQuery Read Session User")," -> This role is for Profiling")))),(0,o.yg)("admonition",{type:"note"},(0,o.yg)("p",{parentName:"admonition"},"You can always add/remove roles to Service Accounts later on. Please refer to the BigQuery ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/granting-changing-revoking-access"},"Manage access to projects, folders, and organizations")," guide for more details.")),(0,o.yg)("ol",{start:3},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"To filter projects based on the ",(0,o.yg)("inlineCode",{parentName:"p"},"project_labels")," configuration, first visit ",(0,o.yg)("a",{parentName:"p",href:"https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/overview"},"cloudresourcemanager.googleapis.com")," and enable the ",(0,o.yg)("inlineCode",{parentName:"p"},"Cloud Resource Manager API"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Create and download a ",(0,o.yg)("a",{parentName:"p",href:"https://cloud.google.com/iam/docs/creating-managing-service-account-keys"},"Service Account Key"),". We will use this to set up authentication within DataHub."))),(0,o.yg)("p",null,"The key file looks like this:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-json"},'{\n "type": "service_account",\n "project_id": "project-id-1234567",\n "private_key_id": "d0121d0000882411234e11166c6aaa23ed5d74e0",\n "private_key": "-----BEGIN PRIVATE KEY-----\\nMIIyourkey\\n-----END PRIVATE KEY-----",\n "client_email": "test@suppproject-id-1234567.iam.gserviceaccount.com",\n "client_id": "113545814931671546333",\n "auth_uri": "https://accounts.google.com/o/oauth2/auth",\n "token_uri": "https://oauth2.googleapis.com/token",\n "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%suppproject-id-1234567.iam.gserviceaccount.com"\n}\n')),(0,o.yg)("h2",{id:"next-steps"},"Next Steps"),(0,o.yg)("p",null,"Once you've confirmed all of the above in BigQuery, it's time to ",(0,o.yg)("a",{parentName:"p",href:"/docs/1.1.0/quick-ingestion-guides/bigquery/configuration"},"move on")," to configure the actual ingestion source within the DataHub UI."))}d.isMDXComponent=!0}}]); |