From 9699e39cb50a8d6f8c17a0ecff27eba2e298ff80 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 19 Aug 2021 15:36:43 -0700 Subject: [PATCH] ISSUE-239: Add provision to ingest view definition --- .../catalog/jdbi3/TableRepository.java | 2 ++ .../resources/databases/DatabaseUtil.java | 9 +++++ .../resources/databases/TableResource.java | 8 +++-- .../json/schema/api/data/createTable.json | 5 +++ .../json/schema/entity/data/table.json | 4 +++ .../resources/json/schema/type/basic.json | 4 +++ .../databases/TableResourceTest.java | 34 +++++++++++++++++++ .../metadata/generated/data/tags/__init__.py | 2 +- .../generated/data/tags/personalDataTags.py | 2 +- .../metadata/generated/data/tags/piiTags.py | 2 +- .../metadata/generated/data/tags/tierTags.py | 2 +- .../metadata/generated/data/tags/userTags.py | 2 +- .../metadata/generated/schema/api/__init__.py | 2 +- .../generated/schema/api/catalogVersion.py | 2 +- .../generated/schema/api/data/__init__.py | 2 +- .../schema/api/data/createDatabase.py | 2 +- .../generated/schema/api/data/createTable.py | 5 ++- .../generated/schema/api/feed/__init__.py | 2 +- .../generated/schema/api/feed/createThread.py | 2 +- .../generated/schema/api/services/__init__.py | 2 +- .../api/services/createDatabaseService.py | 2 +- .../api/services/updateDatabaseService.py | 2 +- .../metadata/generated/schema/api/setOwner.py | 2 +- .../generated/schema/api/tags/__init__.py | 2 +- .../generated/schema/api/tags/createTag.py | 2 +- .../schema/api/tags/createTagCategory.py | 2 +- .../generated/schema/api/teams/__init__.py | 2 +- .../generated/schema/api/teams/createTeam.py | 2 +- .../generated/schema/api/teams/createUser.py | 2 +- .../generated/schema/entity/__init__.py | 2 +- .../metadata/generated/schema/entity/bots.py | 2 +- .../generated/schema/entity/data/__init__.py | 2 +- .../generated/schema/entity/data/dashboard.py | 2 +- .../generated/schema/entity/data/database.py | 2 +- .../generated/schema/entity/data/metrics.py | 2 +- .../generated/schema/entity/data/pipeline.py | 2 +- .../generated/schema/entity/data/report.py | 2 +- .../generated/schema/entity/data/table.py | 5 ++- .../generated/schema/entity/feed/__init__.py | 2 +- .../generated/schema/entity/feed/thread.py | 2 +- .../schema/entity/services/__init__.py | 2 +- .../schema/entity/services/databaseService.py | 2 +- .../generated/schema/entity/tags/__init__.py | 2 +- .../schema/entity/tags/tagCategory.py | 2 +- .../generated/schema/entity/teams/__init__.py | 2 +- .../generated/schema/entity/teams/team.py | 2 +- .../generated/schema/entity/teams/user.py | 2 +- .../generated/schema/type/__init__.py | 2 +- .../generated/schema/type/auditLog.py | 2 +- .../metadata/generated/schema/type/basic.py | 8 ++++- .../schema/type/collectionDescriptor.py | 2 +- .../generated/schema/type/dailyCount.py | 2 +- .../generated/schema/type/entityReference.py | 2 +- .../generated/schema/type/entityUsage.py | 2 +- .../generated/schema/type/jdbcConnection.py | 2 +- .../metadata/generated/schema/type/profile.py | 2 +- .../generated/schema/type/schedule.py | 2 +- .../generated/schema/type/tagLabel.py | 2 +- .../generated/schema/type/usageDetails.py | 2 +- .../src/metadata/ingestion/ometa/client.py | 2 +- .../ingestion/sink/metadata_rest_tables.py | 12 +++++-- .../metadata/ingestion/source/sql_source.py | 10 +++++- 62 files changed, 146 insertions(+), 60 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index ee89c94091e..9c58ac0330b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -417,6 +417,7 @@ public abstract class TableRepository { getColumnTags(fields.contains("tags"), table); table.setJoins(fields.contains("joins") ? getJoins(table) : null); table.setSampleData(fields.contains("sampleData") ? getSampleData(table) : null); + table.setViewDefinition(fields.contains("viewDefinition") ? table.getViewDefinition() : null); return table; } @@ -629,6 +630,7 @@ public abstract class TableRepository { TableData.class); } + public interface TableDAO { @SqlUpdate("INSERT INTO table_entity (json) VALUES (:json)") void insert(@Bind("json") String json); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseUtil.java index a39f9120715..caf3e5848eb 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseUtil.java @@ -19,6 +19,7 @@ package org.openmetadata.catalog.resources.databases; import org.openmetadata.catalog.type.Column; import org.openmetadata.catalog.type.ColumnConstraint; import org.openmetadata.catalog.type.TableConstraint; +import org.openmetadata.catalog.type.TableType; import java.util.ArrayList; import java.util.List; @@ -65,4 +66,12 @@ public final class DatabaseUtil { } } } + + public static void validateViewDefinition(TableType tableType, String viewDefinition) { + if ( (tableType == null || tableType.equals(TableType.Regular) || tableType.equals(TableType.External)) + && viewDefinition != null && !viewDefinition.isEmpty()) { + throw new IllegalArgumentException("ViewDefinition can only be set on TableType View, " + + "SecureView or MaterializedView"); + } + } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index e47647f8f3d..2a5f03cce19 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -115,7 +115,8 @@ public class TableResource { } } - static final String FIELDS = "columns,tableConstraints,usageSummary,owner,database,tags,followers,joins,sampleData"; + static final String FIELDS = "columns,tableConstraints,usageSummary,owner," + + "database,tags,followers,joins,sampleData,viewDefinition"; public static final List FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "") .split(",")); @@ -236,7 +237,7 @@ public class TableResource { Table table = new Table().withId(UUID.randomUUID()).withName(create.getName()) .withColumns(create.getColumns()).withDescription(create.getDescription()) .withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType()) - .withTags(create.getTags()); + .withTags(create.getTags()).withViewDefinition(create.getViewDefinition()); table = addHref(uriInfo, dao.create(validateNewTable(table), create.getOwner(), create.getDatabase())); return Response.created(table.getHref()).entity(table).build(); } @@ -256,7 +257,7 @@ public class TableResource { Table table = new Table().withId(UUID.randomUUID()).withName(create.getName()) .withColumns(create.getColumns()).withDescription(create.getDescription()) .withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType()) - .withTags(create.getTags()); + .withTags(create.getTags()).withViewDefinition(create.getViewDefinition()); SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, dao.getOwnerReference(table)); PutResponse response = dao.createOrUpdate(validateNewTable(table), create.getOwner(), create.getDatabase()); table = addHref(uriInfo, response.getEntity()); @@ -384,6 +385,7 @@ public class TableResource { public static Table validateNewTable(Table table) { table.setId(UUID.randomUUID()); DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints()); + DatabaseUtil.validateViewDefinition(table.getTableType(), table.getViewDefinition()); return table; } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/data/createTable.json b/catalog-rest-service/src/main/resources/json/schema/api/data/createTable.json index e89a02402c7..4ce5217f1fe 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/data/createTable.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/data/createTable.json @@ -48,6 +48,11 @@ "$ref": "../../type/tagLabel.json" }, "default": null + }, + "viewDefinition": { + "description": "View Definition in SQL. Applies to TableType.View only", + "$ref": "../../type/basic.json#/definitions/sqlQuery", + "default" : null } }, "required": ["name","columns","database"] diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index f0a8a6d990e..b4e9aefccd1 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -300,6 +300,10 @@ "description": "Reference to Database that contains this table.", "$ref": "../../type/entityReference.json" }, + "viewDefinition": { + "description": "View Definition in SQL. Applies to TableType.View only", + "$ref": "../../type/basic.json#/definitions/sqlQuery" + }, "tags": { "description": "Tags for this table.", "type": "array", diff --git a/catalog-rest-service/src/main/resources/json/schema/type/basic.json b/catalog-rest-service/src/main/resources/json/schema/type/basic.json index fbd2c640bee..65c63a71d1d 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/basic.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/basic.json @@ -59,6 +59,10 @@ "description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.", "type": "string", "format": "date-Time" + }, + "sqlQuery": { + "description": "SQL query statement. Example - 'select * from orders'.", + "type": "string" } } } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java index 3ebcd4201e5..974da7ca2c0 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java @@ -575,6 +575,40 @@ public class TableResourceTest extends CatalogApplicationTest { "as 2 sample values"); } + @Test + public void put_viewDefinition_200(TestInfo test) throws HttpResponseException { + CreateTable createTable = create(test); + createTable.setTableType(TableType.View); + String query = "sales_vw\n" + + "create view sales_vw as\n" + + "select * from public.sales\n" + + "union all\n" + + "select * from spectrum.sales\n" + + "with no schema binding;\n"; + createTable.setViewDefinition(query); + Table table = createAndCheckTable(createTable, adminAuthHeaders()); + table = getTable(table.getId(), "viewDefinition", adminAuthHeaders()); + LOG.info("table view definition {}", table.getViewDefinition()); + assertEquals(table.getViewDefinition(), query); + } + + @Test + public void put_viewDefinition_invalid_table_4xx(TestInfo test) throws HttpResponseException { + CreateTable createTable = create(test); + createTable.setTableType(TableType.Regular); + String query = "sales_vw\n" + + "create view sales_vw as\n" + + "select * from public.sales\n" + + "union all\n" + + "select * from spectrum.sales\n" + + "with no schema binding;\n"; + createTable.setViewDefinition(query); + HttpResponseException exception = assertThrows(HttpResponseException.class, () + -> createAndCheckTable(createTable, adminAuthHeaders())); + TestUtils.assertResponseContains(exception, BAD_REQUEST, "ViewDefinition can only be set on " + + "TableType View, SecureView or MaterializedView"); + } + @Test public void get_nonExistentTable_404_notFound() { HttpResponseException exception = assertThrows(HttpResponseException.class, () -> diff --git a/ingestion/src/metadata/generated/data/tags/__init__.py b/ingestion/src/metadata/generated/data/tags/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/data/tags/__init__.py +++ b/ingestion/src/metadata/generated/data/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/data/tags/personalDataTags.py b/ingestion/src/metadata/generated/data/tags/personalDataTags.py index 975678a34c2..750940f1531 100644 --- a/ingestion/src/metadata/generated/data/tags/personalDataTags.py +++ b/ingestion/src/metadata/generated/data/tags/personalDataTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/personalDataTags.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/piiTags.py b/ingestion/src/metadata/generated/data/tags/piiTags.py index 14c64ea4916..739cf94efba 100644 --- a/ingestion/src/metadata/generated/data/tags/piiTags.py +++ b/ingestion/src/metadata/generated/data/tags/piiTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/piiTags.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/tierTags.py b/ingestion/src/metadata/generated/data/tags/tierTags.py index 68752a5508c..37f0c31940f 100644 --- a/ingestion/src/metadata/generated/data/tags/tierTags.py +++ b/ingestion/src/metadata/generated/data/tags/tierTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/tierTags.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/userTags.py b/ingestion/src/metadata/generated/data/tags/userTags.py index 83f05942832..0ee44d7bee0 100644 --- a/ingestion/src/metadata/generated/data/tags/userTags.py +++ b/ingestion/src/metadata/generated/data/tags/userTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/userTags.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/__init__.py b/ingestion/src/metadata/generated/schema/api/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/catalogVersion.py b/ingestion/src/metadata/generated/schema/api/catalogVersion.py index 5b4888ed8be..5ba4ec6420d 100644 --- a/ingestion/src/metadata/generated/schema/api/catalogVersion.py +++ b/ingestion/src/metadata/generated/schema/api/catalogVersion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/catalogVersion.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/__init__.py b/ingestion/src/metadata/generated/schema/api/data/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py index 98e29f295d4..7b8d32bd706 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDatabase.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTable.py b/ingestion/src/metadata/generated/schema/api/data/createTable.py index f48c33e5904..cc76b463f16 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTable.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTable.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTable.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations @@ -34,3 +34,6 @@ class CreateTableEntityRequest(BaseModel): tags: Optional[List[tagLabel.TagLabel]] = Field( None, description='Tags for this table' ) + viewDefinition: Optional[basic.SqlQuery] = Field( + None, description='View Definition in SQL. Applies to TableType.View only' + ) diff --git a/ingestion/src/metadata/generated/schema/api/feed/__init__.py b/ingestion/src/metadata/generated/schema/api/feed/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/feed/createThread.py b/ingestion/src/metadata/generated/schema/api/feed/createThread.py index a4a8c41e450..3ab770ad269 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/createThread.py +++ b/ingestion/src/metadata/generated/schema/api/feed/createThread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/feed/createThread.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/__init__.py b/ingestion/src/metadata/generated/schema/api/services/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py index bf6b0c59702..471f66317d8 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDatabaseService.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py index 1203cd2924d..e8938d3137a 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDatabaseService.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/setOwner.py b/ingestion/src/metadata/generated/schema/api/setOwner.py index 4886b1514cf..33771fef297 100644 --- a/ingestion/src/metadata/generated/schema/api/setOwner.py +++ b/ingestion/src/metadata/generated/schema/api/setOwner.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/setOwner.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/__init__.py b/ingestion/src/metadata/generated/schema/api/tags/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTag.py b/ingestion/src/metadata/generated/schema/api/tags/createTag.py index 390035ef37d..8461e5312f6 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTag.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTag.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTag.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py index 76ba3e0ceb3..1ca1206e79c 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTagCategory.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/__init__.py b/ingestion/src/metadata/generated/schema/api/teams/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py index 4ec7acd8421..0381d9f96eb 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createTeam.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/createUser.py b/ingestion/src/metadata/generated/schema/api/teams/createUser.py index 34a55595852..5c8635cd64e 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createUser.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createUser.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createUser.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/__init__.py b/ingestion/src/metadata/generated/schema/entity/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/bots.py b/ingestion/src/metadata/generated/schema/entity/bots.py index 1f315afd1a7..b5ff15afbd1 100644 --- a/ingestion/src/metadata/generated/schema/entity/bots.py +++ b/ingestion/src/metadata/generated/schema/entity/bots.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/bots.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/__init__.py b/ingestion/src/metadata/generated/schema/entity/data/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py index e37761c7aa1..1b9466e8a44 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py +++ b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/dashboard.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/database.py b/ingestion/src/metadata/generated/schema/entity/data/database.py index f81496bf6ae..d2440ae77a6 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/database.py +++ b/ingestion/src/metadata/generated/schema/entity/data/database.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/database.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/metrics.py b/ingestion/src/metadata/generated/schema/entity/data/metrics.py index 79fc17bedbc..73a908364e1 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/metrics.py +++ b/ingestion/src/metadata/generated/schema/entity/data/metrics.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/metrics.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py index abcdab15d3a..abb84fe5a54 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py +++ b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/pipeline.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/report.py b/ingestion/src/metadata/generated/schema/entity/data/report.py index e679247d1c5..a889f45a74b 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/report.py +++ b/ingestion/src/metadata/generated/schema/entity/data/report.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/report.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/table.py b/ingestion/src/metadata/generated/schema/entity/data/table.py index e9bb2705695..9438620b321 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/table.py +++ b/ingestion/src/metadata/generated/schema/entity/data/table.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/table.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations @@ -181,6 +181,9 @@ class Table(BaseModel): database: Optional[entityReference.EntityReference] = Field( None, description='Reference to Database that contains this table.' ) + viewDefinition: Optional[basic.SqlQuery] = Field( + None, description='View Definition in SQL. Applies to TableType.View only' + ) tags: Optional[List[tagLabel.TagLabel]] = Field( None, description='Tags for this table.' ) diff --git a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/feed/thread.py b/ingestion/src/metadata/generated/schema/entity/feed/thread.py index 06064914ffe..beff1911ee2 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/thread.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/thread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/feed/thread.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/__init__.py b/ingestion/src/metadata/generated/schema/entity/services/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index 40caa08baf9..c3b003b45f9 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/databaseService.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py index 8edc3f7683c..6536f08142c 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/tags/tagCategory.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/teams/team.py b/ingestion/src/metadata/generated/schema/entity/teams/team.py index 28273da2eb5..c89fb5a2336 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/team.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/team.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/team.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/user.py b/ingestion/src/metadata/generated/schema/entity/teams/user.py index d3ed808f0b2..6b7f6f10b56 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/user.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/user.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/user.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/__init__.py b/ingestion/src/metadata/generated/schema/type/__init__.py index b7b6907b02b..1d5e88a9eaf 100644 --- a/ingestion/src/metadata/generated/schema/type/__init__.py +++ b/ingestion/src/metadata/generated/schema/type/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 diff --git a/ingestion/src/metadata/generated/schema/type/auditLog.py b/ingestion/src/metadata/generated/schema/type/auditLog.py index 180709eea12..c7c76055135 100644 --- a/ingestion/src/metadata/generated/schema/type/auditLog.py +++ b/ingestion/src/metadata/generated/schema/type/auditLog.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/auditLog.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/basic.py b/ingestion/src/metadata/generated/schema/type/basic.py index bc4c8b1bae8..9b862b4bd84 100644 --- a/ingestion/src/metadata/generated/schema/type/basic.py +++ b/ingestion/src/metadata/generated/schema/type/basic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/basic.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations @@ -66,3 +66,9 @@ class DateTime(BaseModel): ..., description="Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.", ) + + +class SqlQuery(BaseModel): + __root__: str = Field( + ..., description="SQL query statement. Example - 'select * from orders'." + ) diff --git a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py index d65b6d9c09c..61a259918b5 100644 --- a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py +++ b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/collectionDescriptor.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/dailyCount.py b/ingestion/src/metadata/generated/schema/type/dailyCount.py index c00192b09a7..a9763be8a20 100644 --- a/ingestion/src/metadata/generated/schema/type/dailyCount.py +++ b/ingestion/src/metadata/generated/schema/type/dailyCount.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/dailyCount.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityReference.py b/ingestion/src/metadata/generated/schema/type/entityReference.py index 2f07489e0d1..160b2471315 100644 --- a/ingestion/src/metadata/generated/schema/type/entityReference.py +++ b/ingestion/src/metadata/generated/schema/type/entityReference.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityReference.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityUsage.py b/ingestion/src/metadata/generated/schema/type/entityUsage.py index 719175a2b7c..02521ecb1e7 100644 --- a/ingestion/src/metadata/generated/schema/type/entityUsage.py +++ b/ingestion/src/metadata/generated/schema/type/entityUsage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityUsage.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py index 5a5b1d7774d..ee4c476868d 100644 --- a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py +++ b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/jdbcConnection.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/profile.py b/ingestion/src/metadata/generated/schema/type/profile.py index 35f31a98650..4be9820d5b6 100644 --- a/ingestion/src/metadata/generated/schema/type/profile.py +++ b/ingestion/src/metadata/generated/schema/type/profile.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/profile.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/schedule.py b/ingestion/src/metadata/generated/schema/type/schedule.py index 9fd7c9890fd..1a9f9155be0 100644 --- a/ingestion/src/metadata/generated/schema/type/schedule.py +++ b/ingestion/src/metadata/generated/schema/type/schedule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/schedule.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/tagLabel.py b/ingestion/src/metadata/generated/schema/type/tagLabel.py index 77aa832583e..a81bb0410c8 100644 --- a/ingestion/src/metadata/generated/schema/type/tagLabel.py +++ b/ingestion/src/metadata/generated/schema/type/tagLabel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/tagLabel.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/usageDetails.py b/ingestion/src/metadata/generated/schema/type/usageDetails.py index f7799c07625..ac1180c06e0 100644 --- a/ingestion/src/metadata/generated/schema/type/usageDetails.py +++ b/ingestion/src/metadata/generated/schema/type/usageDetails.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/usageDetails.json -# timestamp: 2021-08-17T03:53:57+00:00 +# timestamp: 2021-08-19T22:32:24+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/ingestion/ometa/client.py b/ingestion/src/metadata/ingestion/ometa/client.py index 5cf90ae7dd6..141036f36e4 100644 --- a/ingestion/src/metadata/ingestion/ometa/client.py +++ b/ingestion/src/metadata/ingestion/ometa/client.py @@ -271,7 +271,7 @@ class REST(object): return [Table(**t) for t in resp['data']] def ingest_sample_data(self, id, sample_data): - resp = self.put('/tables/{}/sampleData'.format(id.__root__),data=sample_data.json()) + resp = self.put('/tables/{}/sampleData'.format(id.__root__), data=sample_data.json()) return TableData(**resp['sampleData']) def get_table_by_id(self, table_id: str, fields: [] = ['columns']) -> Table: diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest_tables.py b/ingestion/src/metadata/ingestion/sink/metadata_rest_tables.py index 4104000844f..f367dc68a23 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest_tables.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest_tables.py @@ -59,12 +59,18 @@ class MetadataRestTablesSink(Sink): type="databaseService")) db = self.rest.create_database(db_request) table_request = CreateTableEntityRequest(name=table_and_db.table.name, + tableType=table_and_db.table.tableType, columns=table_and_db.table.columns, description=table_and_db.table.description, database=db.id) + + if table_and_db.table.viewDefinition is not None and table_and_db.table.viewDefinition != "": + table_request.viewDefinition = table_and_db.table.viewDefinition.__root__ + created_table = self.rest.create_or_update_table(table_request) - if hasattr(table_and_db.table,'sampleData'): - self.rest.ingest_sample_data(id=created_table.id,sample_data=table_and_db.table.sampleData) + if table_and_db.table.sampleData is not None: + self.rest.ingest_sample_data(id=created_table.id, sample_data=table_and_db.table.sampleData) + logger.info( 'Successfully ingested {}.{}'.format(table_and_db.database.name.__root__, created_table.name.__root__)) self.status.records_written( @@ -73,7 +79,7 @@ class MetadataRestTablesSink(Sink): logger.error( "Failed to ingest table {} in database {} ".format(table_and_db.table.name, table_and_db.database.name)) logger.error(err) - self.status.failures(table_and_db.table.name) + self.status.failure(table_and_db.table.name) def get_status(self): return self.status diff --git a/ingestion/src/metadata/ingestion/source/sql_source.py b/ingestion/src/metadata/ingestion/source/sql_source.py index bcde41887d8..9cc74e04197 100644 --- a/ingestion/src/metadata/ingestion/source/sql_source.py +++ b/ingestion/src/metadata/ingestion/source/sql_source.py @@ -226,13 +226,21 @@ class SQLSource(Source): self.status.filtered('{}.{}'.format(self.config.get_service_name(), view), "Table pattern not allowed") continue + + try: + view_definition = inspector.get_view_definition(view, schema) + view_definition = "" if view_definition is None else str(view_definition) + except NotImplementedError: + view_definition = "" + description = self._get_table_description(schema, view, inspector) table_columns = self._get_columns(schema, view, inspector) table = Table(id=uuid.uuid4(), name=view, tableType='View', description=description if description is not None else ' ', - columns=table_columns) + columns=table_columns, + viewDefinition=view_definition) table_and_db = OMetaDatabaseAndTable(table=table, database=self._get_database(schema)) yield table_and_db except ValidationError as err: