yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-27 09:55:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

table-filter-added (#2770)

Browse Source 
Co-authored-by: Ayush Shah <ayush@getcollate.io>
This commit is contained in:
codingwithabhi 2022-02-16 19:26:40 +05:30 committed by GitHub
parent 983d534da0
commit 9011b27bf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 25 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
View File

@ -26,6 +26,7 @@
"Druid", "Druid",
"Db2", "Db2",
"ClickHouse", "ClickHouse",
"DynamoDB",
"SingleStore" "SingleStore"
], ],
"javaEnums": [ "javaEnums": [
@ -80,6 +81,9 @@
{ {
"name": "ClickHouse" "name": "ClickHouse"
}, },
{
"name": "DynamoDB"
},
{ {
"name": "SingleStore" "name": "SingleStore"
} }
@ -178,6 +182,12 @@
"default": false "default": false
} }
}, },
"required": ["id", "name", "serviceType", "href", "databaseConnection"], "required": [
"id",
"name",
"serviceType",
"href",
"databaseConnection"
],
"additionalProperties": false "additionalProperties": false
} }

7
ingestion/examples/workflows/dynamodb.json
View File

@ -6,8 +6,11 @@
"aws_secret_access_key": "aws_secret_access_key", "aws_secret_access_key": "aws_secret_access_key",
"service_name": "DynamoDB", "service_name": "DynamoDB",
"region_name": "us-east-2", "region_name": "us-east-2",
"endpoint_url": "dynamodb.us-east-2.amazonaws.com", "endpoint_url": "https://dynamodb.us-east-2.amazonaws.com",
"db_name":"custom_database_name" "db_name":"custom_database_name",
"table_filter_pattern":{
"excludes": [""]
}
} }
}, },
"sink": { "sink": {

11
ingestion/src/metadata/ingestion/source/dynamodb.py
View File

@ -9,7 +9,7 @@ from metadata.generated.schema.entity.services.databaseService import (
DatabaseServiceType, DatabaseServiceType,
) )
from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.common import Entity from metadata.ingestion.api.common import Entity, IncludeFilterPattern
from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.api.source import Source, SourceStatus
from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
@ -23,11 +23,12 @@ logger: logging.Logger = logging.getLogger(__name__)
class DynamoDBSourceConfig(AWSClientConfigModel): class DynamoDBSourceConfig(AWSClientConfigModel):
service_type = "DynamoDB" service_type = DatabaseServiceType.DynamoDB.value
service_name: str service_name: str
endpoint_url: str endpoint_url: str
host_port: str = "" host_port: str = ""
db_name = "DynamoDB" db_name = "DynamoDB"
table_filter_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all()
def get_service_type(self) -> DatabaseServiceType: def get_service_type(self) -> DatabaseServiceType:
return DatabaseServiceType[self.service_type] return DatabaseServiceType[self.service_type]
@ -74,6 +75,12 @@ class DynamodbSource(Source[Entity]):
try: try:
tables = list(self.dynamodb.tables.all()) tables = list(self.dynamodb.tables.all())
for table in tables: for table in tables:
if not self.config.table_filter_pattern.included(table.name):
self.status.filter(
"{}".format(table.name),
"Table pattern not allowed",
)
continue
database_entity = Database( database_entity = Database(
name=self.config.db_name, name=self.config.db_name,
service=EntityReference(id=self.service.id, type="databaseService"), service=EntityReference(id=self.service.id, type="databaseService"),