2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								---
							 
						 
					
						
							
								
									
										
										
										
											2025-08-06 19:22:30 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								title: GCS Storage | `brandName`  Cloud Storage Integration
							 
						 
					
						
							
								
									
										
										
										
											2025-07-03 17:39:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								description: Connect OpenMetadata to Google Cloud Storage with our comprehensive GCS connector guide. Setup instructions, configuration options, and best practices included.
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								slug: /connectors/storage/gcs
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								---
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% connectorDetailsHeader
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								name="GCS"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								stage="PROD"
							 
						 
					
						
							
								
									
										
										
										
											2024-08-01 17:02:33 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								platform="OpenMetadata"
							 
						 
					
						
							
								
									
										
										
										
											2024-07-31 00:05:58 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								availableFeatures=["Metadata", "Structured Containers"]
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								unavailableFeatures=["Unstructured Containers"]
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								/ %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								This page contains the setup guide and reference information for the GCS connector.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Configure and schedule GCS metadata workflows from the OpenMetadata UI:
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  [Requirements ](#requirements )
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  [Metadata Ingestion ](#metadata-ingestion )
							 
						 
					
						
							
								
									
										
										
										
											2025-04-03 15:54:47 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								-  [Troubleshooting ](/connectors/storage/gcs/troubleshooting )
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								{% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/storage/gcs/yaml"} /%}
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								## Requirements
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								We need the following permissions in GCP:
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								### GCS Permissions
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								For all the buckets that we want to ingest, we need to provide the following:
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  `storage.buckets.get` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  `storage.buckets.list` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  `storage.objects.get` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								-  `storage.objects.list` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								### OpenMetadata Manifest
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								In any other connector, extracting metadata happens automatically. In this case, we will be able to extract high-level
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								metadata from buckets, but in order to understand their internal structure we need users to provide an `openmetadata.json` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								file at the bucket root.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								`Supported File Formats: [ "csv",  "tsv", "avro", "parquet", "json", "json.gz", "json.zip" ]` 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								You can learn more about this [here ](/connectors/storage ). Keep reading for an example on the shape of the manifest file.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								{% partial file="/v1.7/connectors/storage/manifest.md" /%}
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								## Metadata Ingestion
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepsContainer %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% step srNumber=1 %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepDescription title="1. Visit the Services Page" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								The first step is ingesting the metadata from your sources. Under
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Settings, you will find a Services link an external source system to
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								OpenMetadata. Once a service is created, it can be used to configure
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								metadata, usage, and profiler workflows.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								To visit the Services page, select Services from the Settings menu.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepDescription %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% image
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								src="/images/v1.7/connectors/visit-services-page.png"
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								alt="Visit Services Page"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								caption="Find Dashboard option on left panel of the settings page" /%}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /step %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% step srNumber=2 %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepDescription title="2. Create a New Service" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Click on the 'Add New Service' button to start the Service creation.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepDescription %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% image
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								src="/images/v1.7/connectors/create-new-service.png"
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								alt="Create a new service"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								caption="Add a new Service from the Storage Services page" /%}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /step %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% step srNumber=3 %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepDescription title="3. Select the Service Type" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Select GCS as the service type and click Next.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepDescription %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% image
							 
						 
					
						
							
								
									
										
										
										
											2025-02-24 13:15:27 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								  src="/images/v1.7/connectors/gcs/select-service.png"
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								  alt="Select Service"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  caption="Select your service from the list" /%}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /step %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% step srNumber=4 %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepDescription title="4. Name and Describe your Service" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								Provide a name and description for your service.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								#### Service Name
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								OpenMetadata uniquely identifies services by their Service Name. Provide
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								a name that distinguishes your deployment from other services, including
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								the other Storage services that you might be ingesting metadata
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								from.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepDescription %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% image
							 
						 
					
						
							
								
									
										
										
										
											2025-02-24 13:15:27 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								  src="/images/v1.7/connectors/gcs/add-new-service.png"
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								  alt="Add New Service"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  caption="Provide a Name and description for your Service" /%}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /step %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% step srNumber=5 %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepDescription title="5. Configure the Service Connection" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								In this step, we will configure the connection settings required for
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								this connector. Please follow the instructions below to ensure that
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								you've configured the connector to read from your GCS service as
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								desired.
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepDescription %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% image
							 
						 
					
						
							
								
									
										
										
										
											2025-02-24 13:15:27 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								  src="/images/v1.7/connectors/gcs/service-connection.png"
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								  alt="Configure service connection"
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								  caption="Configure the service connection by filling the form" /%}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepVisualInfo %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /step %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% extraContent parentTagName="stepsContainer" %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								#### Connection Details
 
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /extraContent %}
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								{% partial file="/v1.7/connectors/test-connection.md" /%}
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								{% partial file="/v1.7/connectors/storage/configure-ingestion.md" /%}
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
									
										
										
										
											2024-12-12 11:34:09 +05:30 
										
									 
								 
							 
							
								
									
										 
									 
								
							 
							
								 
							 
							
							
								{% partial file="/v1.7/connectors/ingestion-schedule-and-deploy.md" /%}
							 
						 
					
						
							
								
									
										
										
										
											2024-06-18 15:53:06 +02:00 
										
									 
								 
							 
							
								
							 
							
								 
							 
							
							
								
							 
						 
					
						
							
								
							 
							
								
							 
							
								 
							 
							
							
								{% /stepsContainer %}