yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-27 00:31:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISSUE-239: Add provision to ingest view definition

Browse Source
This commit is contained in:
Suresh Srinivas 2021-08-19 15:36:43 -07:00
parent dc43efa81c
commit 9699e39cb5
62 changed files with 146 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java
View File

@ -417,6 +417,7 @@ public abstract class TableRepository {
getColumnTags(fields.contains("tags"), table);
table.setJoins(fields.contains("joins") ? getJoins(table) : null);
table.setSampleData(fields.contains("sampleData") ? getSampleData(table) : null);
table.setViewDefinition(fields.contains("viewDefinition") ? table.getViewDefinition() : null);
return table;
}
@ -629,6 +630,7 @@ public abstract class TableRepository {
TableData.class);
}
public interface TableDAO {
@SqlUpdate("INSERT INTO table_entity (json) VALUES (:json)")
void insert(@Bind("json") String json);

9
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/DatabaseUtil.java
View File

@ -19,6 +19,7 @@ package org.openmetadata.catalog.resources.databases;
import org.openmetadata.catalog.type.Column;
import org.openmetadata.catalog.type.ColumnConstraint;
import org.openmetadata.catalog.type.TableConstraint;
import org.openmetadata.catalog.type.TableType;
import java.util.ArrayList;
import java.util.List;
@ -65,4 +66,12 @@ public final class DatabaseUtil {
}
}
}
public static void validateViewDefinition(TableType tableType, String viewDefinition) {
if ( (tableType == null || tableType.equals(TableType.Regular) || tableType.equals(TableType.External))
&& viewDefinition != null && !viewDefinition.isEmpty()) {
throw new IllegalArgumentException("ViewDefinition can only be set on TableType View, " +
"SecureView or MaterializedView");
}
}
}

8
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java
View File

@ -115,7 +115,8 @@ public class TableResource {
}
}
static final String FIELDS = "columns,tableConstraints,usageSummary,owner,database,tags,followers,joins,sampleData";
static final String FIELDS = "columns,tableConstraints,usageSummary,owner," +
"database,tags,followers,joins,sampleData,viewDefinition";
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "")
.split(","));
@ -236,7 +237,7 @@ public class TableResource {
Table table = new Table().withId(UUID.randomUUID()).withName(create.getName())
.withColumns(create.getColumns()).withDescription(create.getDescription())
.withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType())
.withTags(create.getTags());
.withTags(create.getTags()).withViewDefinition(create.getViewDefinition());
table = addHref(uriInfo, dao.create(validateNewTable(table), create.getOwner(), create.getDatabase()));
return Response.created(table.getHref()).entity(table).build();
}
@ -256,7 +257,7 @@ public class TableResource {
Table table = new Table().withId(UUID.randomUUID()).withName(create.getName())
.withColumns(create.getColumns()).withDescription(create.getDescription())
.withTableConstraints(create.getTableConstraints()).withTableType(create.getTableType())
.withTags(create.getTags());
.withTags(create.getTags()).withViewDefinition(create.getViewDefinition());
SecurityUtil.checkAdminRoleOrPermissions(authorizer, securityContext, dao.getOwnerReference(table));
PutResponse<Table> response = dao.createOrUpdate(validateNewTable(table), create.getOwner(), create.getDatabase());
table = addHref(uriInfo, response.getEntity());
@ -384,6 +385,7 @@ public class TableResource {
public static Table validateNewTable(Table table) {
table.setId(UUID.randomUUID());
DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints());
DatabaseUtil.validateViewDefinition(table.getTableType(), table.getViewDefinition());
return table;
}
}

5
catalog-rest-service/src/main/resources/json/schema/api/data/createTable.json
View File

@ -48,6 +48,11 @@
"$ref": "../../type/tagLabel.json"
},
"default": null
},
"viewDefinition": {
"description": "View Definition in SQL. Applies to TableType.View only",
"$ref": "../../type/basic.json#/definitions/sqlQuery",
"default" : null
}
},
"required": ["name","columns","database"]

4
catalog-rest-service/src/main/resources/json/schema/entity/data/table.json
View File

@ -300,6 +300,10 @@
"description": "Reference to Database that contains this table.",
"$ref": "../../type/entityReference.json"
},
"viewDefinition": {
"description": "View Definition in SQL. Applies to TableType.View only",
"$ref": "../../type/basic.json#/definitions/sqlQuery"
},
"tags": {
"description": "Tags for this table.",
"type": "array",

4
catalog-rest-service/src/main/resources/json/schema/type/basic.json
View File

@ -59,6 +59,10 @@
"description": "Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.",
"type": "string",
"format": "date-Time"
},
"sqlQuery": {
"description": "SQL query statement. Example - 'select * from orders'.",
"type": "string"
}
}
}

34
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/TableResourceTest.java
View File

@ -575,6 +575,40 @@ public class TableResourceTest extends CatalogApplicationTest {
"as 2 sample values");
}
@Test
public void put_viewDefinition_200(TestInfo test) throws HttpResponseException {
CreateTable createTable = create(test);
createTable.setTableType(TableType.View);
String query = "sales_vw\n" +
"create view sales_vw as\n" +
"select * from public.sales\n" +
"union all\n" +
"select * from spectrum.sales\n" +
"with no schema binding;\n";
createTable.setViewDefinition(query);
Table table = createAndCheckTable(createTable, adminAuthHeaders());
table = getTable(table.getId(), "viewDefinition", adminAuthHeaders());
LOG.info("table view definition {}", table.getViewDefinition());
assertEquals(table.getViewDefinition(), query);
}
@Test
public void put_viewDefinition_invalid_table_4xx(TestInfo test) throws HttpResponseException {
CreateTable createTable = create(test);
createTable.setTableType(TableType.Regular);
String query = "sales_vw\n" +
"create view sales_vw as\n" +
"select * from public.sales\n" +
"union all\n" +
"select * from spectrum.sales\n" +
"with no schema binding;\n";
createTable.setViewDefinition(query);
HttpResponseException exception = assertThrows(HttpResponseException.class, ()
-> createAndCheckTable(createTable, adminAuthHeaders()));
TestUtils.assertResponseContains(exception, BAD_REQUEST, "ViewDefinition can only be set on " +
"TableType View, SecureView or MaterializedView");
}
@Test
public void get_nonExistentTable_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->

2
ingestion/src/metadata/generated/data/tags/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/data/tags/personalDataTags.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: data/tags/personalDataTags.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/data/tags/piiTags.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: data/tags/piiTags.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/data/tags/tierTags.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: data/tags/tierTags.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/data/tags/userTags.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: data/tags/userTags.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/catalogVersion.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/catalogVersion.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/data/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/data/createDatabase.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/data/createDatabase.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

5
ingestion/src/metadata/generated/schema/api/data/createTable.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/data/createTable.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations
@ -34,3 +34,6 @@ class CreateTableEntityRequest(BaseModel):
tags: Optional[List[tagLabel.TagLabel]] = Field(
None, description='Tags for this table'
)
viewDefinition: Optional[basic.SqlQuery] = Field(
None, description='View Definition in SQL. Applies to TableType.View only'
)

2
ingestion/src/metadata/generated/schema/api/feed/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/feed/createThread.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/feed/createThread.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/services/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/services/createDatabaseService.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/services/updateDatabaseService.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/setOwner.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/setOwner.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/tags/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/tags/createTag.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/tags/createTag.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/tags/createTagCategory.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/teams/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/api/teams/createTeam.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/teams/createTeam.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/api/teams/createUser.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/api/teams/createUser.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/bots.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/bots.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/data/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/data/dashboard.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/dashboard.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/data/database.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/database.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/data/metrics.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/metrics.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/data/pipeline.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/pipeline.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/data/report.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/report.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

5
ingestion/src/metadata/generated/schema/entity/data/table.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/data/table.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations
@ -181,6 +181,9 @@ class Table(BaseModel):
database: Optional[entityReference.EntityReference] = Field(
None, description='Reference to Database that contains this table.'
)
viewDefinition: Optional[basic.SqlQuery] = Field(
None, description='View Definition in SQL. Applies to TableType.View only'
)
tags: Optional[List[tagLabel.TagLabel]] = Field(
None, description='Tags for this table.'
)

2
ingestion/src/metadata/generated/schema/entity/feed/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/feed/thread.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/feed/thread.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/services/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/services/databaseService.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/services/databaseService.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/tags/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/tags/tagCategory.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/teams/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/entity/teams/team.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/teams/team.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/entity/teams/user.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/entity/teams/user.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/__init__.py
View File

@ -1,3 +1,3 @@
# generated by datamodel-codegen:
# filename: json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00

2
ingestion/src/metadata/generated/schema/type/auditLog.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/auditLog.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

8
ingestion/src/metadata/generated/schema/type/basic.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/basic.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations
@ -66,3 +66,9 @@ class DateTime(BaseModel):
...,
description="Date and time in ISO 8601 format. Example - '2018-11-13T20:20:39+00:00'.",
)
class SqlQuery(BaseModel):
__root__: str = Field(
..., description="SQL query statement. Example - 'select * from orders'."
)

2
ingestion/src/metadata/generated/schema/type/collectionDescriptor.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/collectionDescriptor.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/dailyCount.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/dailyCount.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/entityReference.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/entityReference.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/entityUsage.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/entityUsage.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/jdbcConnection.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/jdbcConnection.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/profile.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/profile.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/schedule.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/schedule.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/tagLabel.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/tagLabel.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/generated/schema/type/usageDetails.py
View File

@ -1,6 +1,6 @@
# generated by datamodel-codegen:
# filename: schema/type/usageDetails.json
# timestamp: 2021-08-17T03:53:57+00:00
# timestamp: 2021-08-19T22:32:24+00:00
from __future__ import annotations

2
ingestion/src/metadata/ingestion/ometa/client.py
View File

@ -271,7 +271,7 @@ class REST(object):
return [Table(**t) for t in resp['data']]
def ingest_sample_data(self, id, sample_data):
resp = self.put('/tables/{}/sampleData'.format(id.__root__),data=sample_data.json())
resp = self.put('/tables/{}/sampleData'.format(id.__root__), data=sample_data.json())
return TableData(**resp['sampleData'])
def get_table_by_id(self, table_id: str, fields: [] = ['columns']) -> Table:

12
ingestion/src/metadata/ingestion/sink/metadata_rest_tables.py
View File

@ -59,12 +59,18 @@ class MetadataRestTablesSink(Sink):
type="databaseService"))
db = self.rest.create_database(db_request)
table_request = CreateTableEntityRequest(name=table_and_db.table.name,
tableType=table_and_db.table.tableType,
columns=table_and_db.table.columns,
description=table_and_db.table.description,
database=db.id)
if table_and_db.table.viewDefinition is not None and table_and_db.table.viewDefinition != "":
table_request.viewDefinition = table_and_db.table.viewDefinition.__root__
created_table = self.rest.create_or_update_table(table_request)
if hasattr(table_and_db.table,'sampleData'):
self.rest.ingest_sample_data(id=created_table.id,sample_data=table_and_db.table.sampleData)
if table_and_db.table.sampleData is not None:
self.rest.ingest_sample_data(id=created_table.id, sample_data=table_and_db.table.sampleData)
logger.info(
'Successfully ingested {}.{}'.format(table_and_db.database.name.__root__, created_table.name.__root__))
self.status.records_written(
@ -73,7 +79,7 @@ class MetadataRestTablesSink(Sink):
logger.error(
"Failed to ingest table {} in database {} ".format(table_and_db.table.name, table_and_db.database.name))
logger.error(err)
self.status.failures(table_and_db.table.name)
self.status.failure(table_and_db.table.name)
def get_status(self):
return self.status

10
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -226,13 +226,21 @@ class SQLSource(Source):
self.status.filtered('{}.{}'.format(self.config.get_service_name(), view),
"Table pattern not allowed")
continue
try:
view_definition = inspector.get_view_definition(view, schema)
view_definition = "" if view_definition is None else str(view_definition)
except NotImplementedError:
view_definition = ""
description = self._get_table_description(schema, view, inspector)
table_columns = self._get_columns(schema, view, inspector)
table = Table(id=uuid.uuid4(),
name=view,
tableType='View',
description=description if description is not None else ' ',
columns=table_columns)
columns=table_columns,
viewDefinition=view_definition)
table_and_db = OMetaDatabaseAndTable(table=table, database=self._get_database(schema))
yield table_and_db
except ValidationError as err: