From 9011b27bf682f84f757515d514a737c9bd76b9a1 Mon Sep 17 00:00:00 2001
From: codingwithabhi <63392662+codingwithabhi@users.noreply.github.com>
Date: Wed, 16 Feb 2022 19:26:40 +0530
Subject: [PATCH] table-filter-added (#2770)

Co-authored-by: Ayush Shah <ayush@getcollate.io>
---
 .../json/schema/entity/services/databaseService.json | 12 +++++++++++-
 ingestion/examples/workflows/dynamodb.json           |  7 +++++--
 ingestion/src/metadata/ingestion/source/dynamodb.py  | 11 +++++++++--
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
index 2e39db25e7f..1e2a630ad58 100644
--- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
+++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json
@@ -26,6 +26,7 @@
         "Druid",
         "Db2",
         "ClickHouse",
+        "DynamoDB",
         "SingleStore"
       ],
       "javaEnums": [
@@ -80,6 +81,9 @@
         {
           "name": "ClickHouse"
         },
+        {
+          "name": "DynamoDB"
+        },
         {
           "name": "SingleStore"
         }
@@ -178,6 +182,12 @@
       "default": false
     }
   },
-  "required": ["id", "name", "serviceType", "href", "databaseConnection"],
+  "required": [
+    "id",
+    "name",
+    "serviceType",
+    "href",
+    "databaseConnection"
+  ],
   "additionalProperties": false
 }
diff --git a/ingestion/examples/workflows/dynamodb.json b/ingestion/examples/workflows/dynamodb.json
index 712b9292430..46cff4d6d04 100644
--- a/ingestion/examples/workflows/dynamodb.json
+++ b/ingestion/examples/workflows/dynamodb.json
@@ -6,8 +6,11 @@
         "aws_secret_access_key": "aws_secret_access_key",
         "service_name": "DynamoDB",
         "region_name": "us-east-2",
-        "endpoint_url": "dynamodb.us-east-2.amazonaws.com",
-        "db_name":"custom_database_name"
+        "endpoint_url": "https://dynamodb.us-east-2.amazonaws.com",
+        "db_name":"custom_database_name",
+        "table_filter_pattern":{
+          "excludes": [""]
+        }
       }
     },
     "sink": {
diff --git a/ingestion/src/metadata/ingestion/source/dynamodb.py b/ingestion/src/metadata/ingestion/source/dynamodb.py
index 6c8fb8125e1..fb8a471d895 100644
--- a/ingestion/src/metadata/ingestion/source/dynamodb.py
+++ b/ingestion/src/metadata/ingestion/source/dynamodb.py
@@ -9,7 +9,7 @@ from metadata.generated.schema.entity.services.databaseService import (
     DatabaseServiceType,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.common import Entity
+from metadata.ingestion.api.common import Entity, IncludeFilterPattern
 from metadata.ingestion.api.source import Source, SourceStatus
 from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -23,11 +23,12 @@ logger: logging.Logger = logging.getLogger(__name__)
 
 
 class DynamoDBSourceConfig(AWSClientConfigModel):
-    service_type = "DynamoDB"
+    service_type = DatabaseServiceType.DynamoDB.value
     service_name: str
     endpoint_url: str
     host_port: str = ""
     db_name = "DynamoDB"
+    table_filter_pattern: IncludeFilterPattern = IncludeFilterPattern.allow_all()
 
     def get_service_type(self) -> DatabaseServiceType:
         return DatabaseServiceType[self.service_type]
@@ -74,6 +75,12 @@ class DynamodbSource(Source[Entity]):
         try:
             tables = list(self.dynamodb.tables.all())
             for table in tables:
+                if not self.config.table_filter_pattern.included(table.name):
+                    self.status.filter(
+                        "{}".format(table.name),
+                        "Table pattern not allowed",
+                    )
+                    continue
                 database_entity = Database(
                     name=self.config.db_name,
                     service=EntityReference(id=self.service.id, type="databaseService"),