diff --git a/ingestion/setup.py b/ingestion/setup.py index ae885a47980..549a750f82c 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -59,7 +59,6 @@ base_requirements = { "PyYAML", } pii_requirements = { - "en_core_web_sm@https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web", "spacy==3.0.5", } @@ -78,7 +77,7 @@ base_plugins = { } plugins: Dict[str, Set[str]] = { "athena": {"PyAthena[SQLAlchemy]"}, - "bigquery": {"openmetadata-sqlalchemy-bigquery==0.2.0"}, + "bigquery": {"openmetadata-sqlalchemy-bigquery==0.2.2"}, "bigquery-usage": {"google-cloud-logging", "cachetools"}, "docker": {"docker==5.0.3"}, "dbt": {}, @@ -126,7 +125,7 @@ plugins: Dict[str, Set[str]] = { build_options = {"includes": ["_cffi_backend"]} setup( name="openmetadata-ingestion", - version="0.4.4", + version="0.4.8", url="https://open-metadata.org/", author="OpenMetadata Committers", license="Apache License 2.0", diff --git a/ingestion/src/metadata/generated/data/tags/__init__.py b/ingestion/src/metadata/generated/data/tags/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/data/tags/__init__.py +++ b/ingestion/src/metadata/generated/data/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/data/tags/personalDataTags.py b/ingestion/src/metadata/generated/data/tags/personalDataTags.py index a0170079c17..4f2e274725d 100644 --- a/ingestion/src/metadata/generated/data/tags/personalDataTags.py +++ b/ingestion/src/metadata/generated/data/tags/personalDataTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/personalDataTags.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/piiTags.py b/ingestion/src/metadata/generated/data/tags/piiTags.py index e79afc0172b..fc9a7b392d5 100644 --- a/ingestion/src/metadata/generated/data/tags/piiTags.py +++ b/ingestion/src/metadata/generated/data/tags/piiTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/piiTags.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/tierTags.py b/ingestion/src/metadata/generated/data/tags/tierTags.py index d25912f12ad..7bdb68a60be 100644 --- a/ingestion/src/metadata/generated/data/tags/tierTags.py +++ b/ingestion/src/metadata/generated/data/tags/tierTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/tierTags.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/data/tags/userTags.py b/ingestion/src/metadata/generated/data/tags/userTags.py index 9af33049ed5..0e791a57118 100644 --- a/ingestion/src/metadata/generated/data/tags/userTags.py +++ b/ingestion/src/metadata/generated/data/tags/userTags.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: data/tags/userTags.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/__init__.py b/ingestion/src/metadata/generated/schema/api/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/catalogVersion.py b/ingestion/src/metadata/generated/schema/api/catalogVersion.py index 2b1b71acc27..544616007dd 100644 --- a/ingestion/src/metadata/generated/schema/api/catalogVersion.py +++ b/ingestion/src/metadata/generated/schema/api/catalogVersion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/catalogVersion.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/__init__.py b/ingestion/src/metadata/generated/schema/api/data/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/data/createChart.py b/ingestion/src/metadata/generated/schema/api/data/createChart.py index c2d03e4ddff..7cb72b5633c 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createChart.py +++ b/ingestion/src/metadata/generated/schema/api/data/createChart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createChart.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py index d9870e98d2d..c21e1c7dded 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDashboard.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDashboard.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py index 3f20a6474e7..162f98ac293 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createDatabase.py +++ b/ingestion/src/metadata/generated/schema/api/data/createDatabase.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createDatabase.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createDbtModel.py b/ingestion/src/metadata/generated/schema/api/data/createDbtModel.py new file mode 100644 index 00000000000..23077b43e96 --- /dev/null +++ b/ingestion/src/metadata/generated/schema/api/data/createDbtModel.py @@ -0,0 +1,36 @@ +# generated by datamodel-codegen: +# filename: schema/api/data/createDbtModel.json +# timestamp: 2021-12-02T02:28:42+00:00 + +from __future__ import annotations + +from typing import List, Optional + +from pydantic import BaseModel, Field + +from ...entity.data import dbtmodel, table +from ...type import basic, entityReference, tagLabel + + +class CreateDbtModelEntityRequest(BaseModel): + name: dbtmodel.DbtModelName = Field( + ..., + description='Name that identifies the this entity instance uniquely. Same as id if when name is not unique', + ) + description: Optional[str] = Field( + None, description='Description of DBTModel instance.' + ) + dbtNodeType: Optional[dbtmodel.DbtNodeType] = None + columns: List[table.Column] = Field(..., description='Schema of the Model') + owner: Optional[entityReference.EntityReference] = Field( + None, description='Owner of this entity' + ) + database: Optional[basic.Uuid] = Field( + None, description='Database corresponding to this table' + ) + tags: Optional[List[tagLabel.TagLabel]] = Field( + None, description='Tags for this model' + ) + viewDefinition: Optional[basic.SqlQuery] = Field( + None, description='View Definition in SQL.' + ) diff --git a/ingestion/src/metadata/generated/schema/api/data/createLocation.py b/ingestion/src/metadata/generated/schema/api/data/createLocation.py index 9fe158b09c6..ee014057214 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createLocation.py +++ b/ingestion/src/metadata/generated/schema/api/data/createLocation.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createLocation.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createMlModel.py b/ingestion/src/metadata/generated/schema/api/data/createMlModel.py index 78ee438469b..71ccfb3d908 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createMlModel.py +++ b/ingestion/src/metadata/generated/schema/api/data/createMlModel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: schema/api/data/createMlModel.json -# timestamp: 2021-11-20T15:09:34+00:00 +# filename: schema/api/data/createMLModel.json +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createModel.py b/ingestion/src/metadata/generated/schema/api/data/createModel.py index 94f9bb3498e..c8cf35835ce 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createModel.py +++ b/ingestion/src/metadata/generated/schema/api/data/createModel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createModel.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py index 2e1641a3499..8ec0c186168 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createPipeline.py +++ b/ingestion/src/metadata/generated/schema/api/data/createPipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createPipeline.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTable.py b/ingestion/src/metadata/generated/schema/api/data/createTable.py index 8867963b426..aa117fc439e 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTable.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTable.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTable.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/data/createTopic.py b/ingestion/src/metadata/generated/schema/api/data/createTopic.py index c72eb43bcd8..c417f894747 100644 --- a/ingestion/src/metadata/generated/schema/api/data/createTopic.py +++ b/ingestion/src/metadata/generated/schema/api/data/createTopic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/data/createTopic.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/feed/__init__.py b/ingestion/src/metadata/generated/schema/api/feed/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/feed/createThread.py b/ingestion/src/metadata/generated/schema/api/feed/createThread.py index e269d603728..fb321943960 100644 --- a/ingestion/src/metadata/generated/schema/api/feed/createThread.py +++ b/ingestion/src/metadata/generated/schema/api/feed/createThread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/feed/createThread.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py index e83886280fb..07738cb68e5 100644 --- a/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py +++ b/ingestion/src/metadata/generated/schema/api/lineage/addLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/lineage/addLineage.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/operations/workflows/__init__.py b/ingestion/src/metadata/generated/schema/api/operations/workflows/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/operations/workflows/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/operations/workflows/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/operations/workflows/createIngestion.py b/ingestion/src/metadata/generated/schema/api/operations/workflows/createIngestion.py index e29c88e5719..54c4bae1a0e 100644 --- a/ingestion/src/metadata/generated/schema/api/operations/workflows/createIngestion.py +++ b/ingestion/src/metadata/generated/schema/api/operations/workflows/createIngestion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/operations/workflows/createIngestion.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/policies/__init__.py b/ingestion/src/metadata/generated/schema/api/policies/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/policies/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/policies/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/policies/createPolicy.py b/ingestion/src/metadata/generated/schema/api/policies/createPolicy.py index a67b0fc75e2..6110def09fe 100644 --- a/ingestion/src/metadata/generated/schema/api/policies/createPolicy.py +++ b/ingestion/src/metadata/generated/schema/api/policies/createPolicy.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/policies/createPolicy.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/__init__.py b/ingestion/src/metadata/generated/schema/api/services/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py index 8ca532795c4..8502a84b9b3 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDashboardService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py index d0066da21ba..e59647f44d8 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createDatabaseService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py index 5b2e03fc1c7..5dccba11925 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createMessagingService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py index df1244b3b1e..90a08691c79 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createPipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createPipelineService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/createStorageService.py b/ingestion/src/metadata/generated/schema/api/services/createStorageService.py index 6a60958b77d..0869d3de985 100644 --- a/ingestion/src/metadata/generated/schema/api/services/createStorageService.py +++ b/ingestion/src/metadata/generated/schema/api/services/createStorageService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/createStorageService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py index 17efee904da..f49c10746e4 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDashboardService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py index 5ea16c54d8d..e56f0af9767 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateDatabaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateDatabaseService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py index 05048bb7d88..3f5f70ef4f8 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateMessagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateMessagingService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py index 01bb6a0c206..0c3b3529a6c 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updatePipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updatePipelineService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/services/updateStorageService.py b/ingestion/src/metadata/generated/schema/api/services/updateStorageService.py index 977d19230a4..b3dd8fde120 100644 --- a/ingestion/src/metadata/generated/schema/api/services/updateStorageService.py +++ b/ingestion/src/metadata/generated/schema/api/services/updateStorageService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/services/updateStorageService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/setOwner.py b/ingestion/src/metadata/generated/schema/api/setOwner.py index 63b957915b4..dcb8b7638f8 100644 --- a/ingestion/src/metadata/generated/schema/api/setOwner.py +++ b/ingestion/src/metadata/generated/schema/api/setOwner.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/setOwner.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/__init__.py b/ingestion/src/metadata/generated/schema/api/tags/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTag.py b/ingestion/src/metadata/generated/schema/api/tags/createTag.py index 2885a993eb5..c8d57bcc754 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTag.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTag.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTag.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py index 1b662b32c60..64580015a4c 100644 --- a/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py +++ b/ingestion/src/metadata/generated/schema/api/tags/createTagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/tags/createTagCategory.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/__init__.py b/ingestion/src/metadata/generated/schema/api/teams/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/api/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py index 770c58ba277..f6030436885 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createTeam.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createTeam.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createTeam.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/api/teams/createUser.py b/ingestion/src/metadata/generated/schema/api/teams/createUser.py index 7027d07ea42..a0edb075472 100644 --- a/ingestion/src/metadata/generated/schema/api/teams/createUser.py +++ b/ingestion/src/metadata/generated/schema/api/teams/createUser.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/api/teams/createUser.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/__init__.py b/ingestion/src/metadata/generated/schema/entity/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/bots.py b/ingestion/src/metadata/generated/schema/entity/bots.py index 8d29d76a4ca..ac606f008f0 100644 --- a/ingestion/src/metadata/generated/schema/entity/bots.py +++ b/ingestion/src/metadata/generated/schema/entity/bots.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/bots.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/__init__.py b/ingestion/src/metadata/generated/schema/entity/data/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/data/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/data/chart.py b/ingestion/src/metadata/generated/schema/entity/data/chart.py index 55e409a59e6..4d690041c95 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/chart.py +++ b/ingestion/src/metadata/generated/schema/entity/data/chart.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/chart.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py index 5241b9e244f..d98771c3848 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/dashboard.py +++ b/ingestion/src/metadata/generated/schema/entity/data/dashboard.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/dashboard.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/database.py b/ingestion/src/metadata/generated/schema/entity/data/database.py index f1f8631cdaa..fa4e55e8056 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/database.py +++ b/ingestion/src/metadata/generated/schema/entity/data/database.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/database.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/dbtmodel.py b/ingestion/src/metadata/generated/schema/entity/data/dbtmodel.py new file mode 100644 index 00000000000..a0ab91e8ce1 --- /dev/null +++ b/ingestion/src/metadata/generated/schema/entity/data/dbtmodel.py @@ -0,0 +1,90 @@ +# generated by datamodel-codegen: +# filename: schema/entity/data/dbtmodel.json +# timestamp: 2021-12-02T02:28:42+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, Extra, Field, constr + +from ...type import basic, entityHistory, entityReference, tagLabel +from . import table + + +class DbtNodeType(Enum): + Seed = 'Seed' + Model = 'Model' + + +class DbtCatalogType(Enum): + BaseTable = 'BaseTable' + + +class DbtMaterializationType(Enum): + Table = 'Table' + Seed = 'Seed' + + +class DbtModelName(BaseModel): + __root__: constr(regex=r'^[^.]*$', min_length=1, max_length=128) = Field( + ..., description='Local name (not fully qualified name) of a table.' + ) + + +class FullyQualifiedColumnName(BaseModel): + __root__: constr(min_length=1, max_length=256) = Field( + ..., + description='Fully qualified name of the column that includes `serviceName.databaseName.tableName.columnName[.nestedColumnName]`. When columnName is null for dataType struct fields, `field_#` where `#` is field index is used. For map dataType, for key the field name `key` is used and for the value field `value` is used.', + ) + + +class DbtModel(BaseModel): + class Config: + extra = Extra.forbid + + id: basic.Uuid = Field(..., description='Unique identifier of this model instance.') + name: DbtModelName = Field( + ..., description='Name of a model. Expected to be unique within a database.' + ) + displayName: Optional[str] = Field( + None, + description='Display Name that identifies this model. It could be title or label from the source services.', + ) + fullyQualifiedName: Optional[str] = Field( + None, + description='Fully qualified name of a model in the form `serviceName.databaseName.dbtModelName`.', + ) + description: Optional[str] = Field(None, description='Description of a DBT Model.') + version: Optional[entityHistory.EntityVersion] = Field( + None, description='Metadata version of the entity.' + ) + updatedAt: Optional[basic.DateTime] = Field( + None, + description='Last update time corresponding to the new version of the entity.', + ) + updatedBy: Optional[str] = Field(None, description='User who made the update.') + href: Optional[basic.Href] = Field(None, description='Link to this table resource.') + dbtNodeType: Optional[DbtNodeType] = None + dbtCatalogType: Optional[DbtCatalogType] = None + dbtMaterializationType: Optional[DbtMaterializationType] = None + columns: List[table.Column] = Field(..., description='Columns in this DBT Model.') + owner: Optional[entityReference.EntityReference] = Field( + None, description='Owner of this DBT Model.' + ) + database: Optional[entityReference.EntityReference] = Field( + None, description='Reference to Database that uses this DBT Model.' + ) + viewDefinition: Optional[basic.SqlQuery] = Field( + None, description='View Definition in SQL.' + ) + tags: Optional[List[tagLabel.TagLabel]] = Field( + None, description='Tags for this DBT.' + ) + followers: Optional[entityReference.EntityReferenceList] = Field( + None, description='Followers of this table.' + ) + changeDescription: Optional[entityHistory.ChangeDescription] = Field( + None, description='Change that lead to this version of the entity.' + ) diff --git a/ingestion/src/metadata/generated/schema/entity/data/location.py b/ingestion/src/metadata/generated/schema/entity/data/location.py index 9a34765cc16..a071a433ace 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/location.py +++ b/ingestion/src/metadata/generated/schema/entity/data/location.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/location.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/metrics.py b/ingestion/src/metadata/generated/schema/entity/data/metrics.py index 50927a05053..fba03ac0cf8 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/metrics.py +++ b/ingestion/src/metadata/generated/schema/entity/data/metrics.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/metrics.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/mlmodel.py b/ingestion/src/metadata/generated/schema/entity/data/mlmodel.py index a133f8eacef..a37cdd8bc12 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/mlmodel.py +++ b/ingestion/src/metadata/generated/schema/entity/data/mlmodel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/mlmodel.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/model.py b/ingestion/src/metadata/generated/schema/entity/data/model.py index ca5c050f1bf..2eee04067bb 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/model.py +++ b/ingestion/src/metadata/generated/schema/entity/data/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/model.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py index 627a618c6b2..5c802a1250a 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/pipeline.py +++ b/ingestion/src/metadata/generated/schema/entity/data/pipeline.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/pipeline.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/report.py b/ingestion/src/metadata/generated/schema/entity/data/report.py index 42480958151..6048845d2c4 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/report.py +++ b/ingestion/src/metadata/generated/schema/entity/data/report.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/report.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/table.py b/ingestion/src/metadata/generated/schema/entity/data/table.py index 6f24fa521bf..6a99fbc4f99 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/table.py +++ b/ingestion/src/metadata/generated/schema/entity/data/table.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/table.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/data/topic.py b/ingestion/src/metadata/generated/schema/entity/data/topic.py index 7789f3ad981..bd68e9e13d1 100644 --- a/ingestion/src/metadata/generated/schema/entity/data/topic.py +++ b/ingestion/src/metadata/generated/schema/entity/data/topic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/data/topic.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/feed/thread.py b/ingestion/src/metadata/generated/schema/entity/feed/thread.py index 0f4abb714a2..27d87ffb553 100644 --- a/ingestion/src/metadata/generated/schema/entity/feed/thread.py +++ b/ingestion/src/metadata/generated/schema/entity/feed/thread.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/feed/thread.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/__init__.py b/ingestion/src/metadata/generated/schema/entity/policies/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/__init__.py b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/rule.py b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/rule.py index ed4c9b65033..262ea910588 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/rule.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/rule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/accessControl/rule.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/tagBased.py b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/tagBased.py index c2179d730d1..62189fa2230 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/accessControl/tagBased.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/accessControl/tagBased.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/accessControl/tagBased.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/filters.py b/ingestion/src/metadata/generated/schema/entity/policies/filters.py index 3056f7a3e46..936a6c168d9 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/filters.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/filters.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/filters.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/__init__.py b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/deleteAction.py b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/deleteAction.py index dcaa0e8eb58..ddc16843233 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/deleteAction.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/deleteAction.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/lifecycle/deleteAction.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/moveAction.py b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/moveAction.py index cccb3e49b65..0db55359c8e 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/moveAction.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/moveAction.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/lifecycle/moveAction.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/rule.py b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/rule.py index c4e195b00a1..36cf3c9bc21 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/rule.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/lifecycle/rule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/lifecycle/rule.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/policies/policy.py b/ingestion/src/metadata/generated/schema/entity/policies/policy.py index 95a0f1391ff..3b6f5d8f1c5 100644 --- a/ingestion/src/metadata/generated/schema/entity/policies/policy.py +++ b/ingestion/src/metadata/generated/schema/entity/policies/policy.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/policies/policy.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/__init__.py b/ingestion/src/metadata/generated/schema/entity/services/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/services/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py index 1fd7e4cc85a..57bf5498f0a 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/dashboardService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/dashboardService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py index c92d43bb425..dc715bcfd15 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/databaseService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/databaseService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/databaseService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py index 33b6b4a029f..1d7b60e7ed8 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/messagingService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/messagingService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/messagingService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py index 90685cd4b23..b5eba081767 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/pipelineService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/pipelineService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/services/storageService.py b/ingestion/src/metadata/generated/schema/entity/services/storageService.py index 34305de96ca..6edb509eea2 100644 --- a/ingestion/src/metadata/generated/schema/entity/services/storageService.py +++ b/ingestion/src/metadata/generated/schema/entity/services/storageService.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/services/storageService.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py index 0b7efef0679..a06dd11f471 100644 --- a/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py +++ b/ingestion/src/metadata/generated/schema/entity/tags/tagCategory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/tags/tagCategory.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/__init__.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/entity/teams/team.py b/ingestion/src/metadata/generated/schema/entity/teams/team.py index be821a22d65..c54e5337333 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/team.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/team.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/team.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/entity/teams/user.py b/ingestion/src/metadata/generated/schema/entity/teams/user.py index 8d5cfeb32f6..9ead45c1c26 100644 --- a/ingestion/src/metadata/generated/schema/entity/teams/user.py +++ b/ingestion/src/metadata/generated/schema/entity/teams/user.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/entity/teams/user.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/operations/workflows/__init__.py b/ingestion/src/metadata/generated/schema/operations/workflows/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/operations/workflows/__init__.py +++ b/ingestion/src/metadata/generated/schema/operations/workflows/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/operations/workflows/ingestion.py b/ingestion/src/metadata/generated/schema/operations/workflows/ingestion.py index 6c1ed51ed97..ca3fbaf0903 100644 --- a/ingestion/src/metadata/generated/schema/operations/workflows/ingestion.py +++ b/ingestion/src/metadata/generated/schema/operations/workflows/ingestion.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/operations/workflows/ingestion.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/__init__.py b/ingestion/src/metadata/generated/schema/type/__init__.py index f041b45f278..e9cab79e1e4 100644 --- a/ingestion/src/metadata/generated/schema/type/__init__.py +++ b/ingestion/src/metadata/generated/schema/type/__init__.py @@ -1,3 +1,3 @@ # generated by datamodel-codegen: # filename: json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 diff --git a/ingestion/src/metadata/generated/schema/type/auditLog.py b/ingestion/src/metadata/generated/schema/type/auditLog.py index 9abe4924c41..c9454db9467 100644 --- a/ingestion/src/metadata/generated/schema/type/auditLog.py +++ b/ingestion/src/metadata/generated/schema/type/auditLog.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/auditLog.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/basic.py b/ingestion/src/metadata/generated/schema/type/basic.py index 225dee1fa64..65e0a0e2509 100644 --- a/ingestion/src/metadata/generated/schema/type/basic.py +++ b/ingestion/src/metadata/generated/schema/type/basic.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/basic.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/changeEvent.py b/ingestion/src/metadata/generated/schema/type/changeEvent.py index 566f9abfa9d..e95717d6e4c 100644 --- a/ingestion/src/metadata/generated/schema/type/changeEvent.py +++ b/ingestion/src/metadata/generated/schema/type/changeEvent.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/changeEvent.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py index de9ac0eebbc..e574adcc7d9 100644 --- a/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py +++ b/ingestion/src/metadata/generated/schema/type/collectionDescriptor.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/collectionDescriptor.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/dailyCount.py b/ingestion/src/metadata/generated/schema/type/dailyCount.py index d5525039d07..4ea49a15e17 100644 --- a/ingestion/src/metadata/generated/schema/type/dailyCount.py +++ b/ingestion/src/metadata/generated/schema/type/dailyCount.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/dailyCount.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityHistory.py b/ingestion/src/metadata/generated/schema/type/entityHistory.py index 28c6a74f36c..3e9e0f26388 100644 --- a/ingestion/src/metadata/generated/schema/type/entityHistory.py +++ b/ingestion/src/metadata/generated/schema/type/entityHistory.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityHistory.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityLineage.py b/ingestion/src/metadata/generated/schema/type/entityLineage.py index 97ab3a5a608..176dc605f3a 100644 --- a/ingestion/src/metadata/generated/schema/type/entityLineage.py +++ b/ingestion/src/metadata/generated/schema/type/entityLineage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityLineage.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityReference.py b/ingestion/src/metadata/generated/schema/type/entityReference.py index dec9ecd21cd..c6e0b0a8b5d 100644 --- a/ingestion/src/metadata/generated/schema/type/entityReference.py +++ b/ingestion/src/metadata/generated/schema/type/entityReference.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityReference.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/entityUsage.py b/ingestion/src/metadata/generated/schema/type/entityUsage.py index 7d4b6b88fcb..dc74ae65619 100644 --- a/ingestion/src/metadata/generated/schema/type/entityUsage.py +++ b/ingestion/src/metadata/generated/schema/type/entityUsage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/entityUsage.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py index ea25efa2518..3fc8a7d8b71 100644 --- a/ingestion/src/metadata/generated/schema/type/jdbcConnection.py +++ b/ingestion/src/metadata/generated/schema/type/jdbcConnection.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/jdbcConnection.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/paging.py b/ingestion/src/metadata/generated/schema/type/paging.py index fb9d8ce7718..198b1e24818 100644 --- a/ingestion/src/metadata/generated/schema/type/paging.py +++ b/ingestion/src/metadata/generated/schema/type/paging.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/paging.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/profile.py b/ingestion/src/metadata/generated/schema/type/profile.py index 2c14a594fef..ccf148753b0 100644 --- a/ingestion/src/metadata/generated/schema/type/profile.py +++ b/ingestion/src/metadata/generated/schema/type/profile.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/profile.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/schedule.py b/ingestion/src/metadata/generated/schema/type/schedule.py index d4943636364..b3546901f8f 100644 --- a/ingestion/src/metadata/generated/schema/type/schedule.py +++ b/ingestion/src/metadata/generated/schema/type/schedule.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/schedule.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/storage.py b/ingestion/src/metadata/generated/schema/type/storage.py index f3ea6da2432..16990869551 100644 --- a/ingestion/src/metadata/generated/schema/type/storage.py +++ b/ingestion/src/metadata/generated/schema/type/storage.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/storage.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/tagLabel.py b/ingestion/src/metadata/generated/schema/type/tagLabel.py index 2e5e171e77f..3bd3b791759 100644 --- a/ingestion/src/metadata/generated/schema/type/tagLabel.py +++ b/ingestion/src/metadata/generated/schema/type/tagLabel.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/tagLabel.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/generated/schema/type/usageDetails.py b/ingestion/src/metadata/generated/schema/type/usageDetails.py index 30035525d60..33ea719b05a 100644 --- a/ingestion/src/metadata/generated/schema/type/usageDetails.py +++ b/ingestion/src/metadata/generated/schema/type/usageDetails.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: schema/type/usageDetails.json -# timestamp: 2021-11-20T15:09:34+00:00 +# timestamp: 2021-12-02T02:38:07+00:00 from __future__ import annotations diff --git a/ingestion/src/metadata/ingestion/source/bigquery_usage.py b/ingestion/src/metadata/ingestion/source/bigquery_usage.py index 9626d28c984..93f4f8b4b28 100644 --- a/ingestion/src/metadata/ingestion/source/bigquery_usage.py +++ b/ingestion/src/metadata/ingestion/source/bigquery_usage.py @@ -64,46 +64,50 @@ class BigqueryUsageSource(Source): pass def next_record(self) -> Iterable[TableQuery]: - logging_client = logging.Client() - logger = logging_client.logger(self.logger_name) - print("Listing entries for logger {}:".format(logger.name)) - start, end = get_start_and_end(self.config.duration) - for entry in logger.list_entries(): - timestamp = entry.timestamp.isoformat() - timestamp = datetime.strptime(timestamp[0:10], "%Y-%m-%d") - if timestamp >= start and timestamp <= end: - if ("query" in str(entry.payload)) and type( - entry.payload - ) == collections.OrderedDict: - payload = list(entry.payload.items())[-1][1] - if "jobChange" in payload: - print(f"\nEntries: {payload}") - queryConfig = payload["jobChange"]["job"]["jobConfig"][ - "queryConfig" - ] - jobStats = payload["jobChange"]["job"]["jobStats"] - statementType = "" - if hasattr(queryConfig, "statementType"): - statementType = queryConfig["statementType"] - database = "" - if hasattr(queryConfig, "destinationTable"): - database = queryConfig["destinationTable"] - analysis_date = str( - datetime.strptime( - jobStats["startTime"][0:19], "%Y-%m-%dT%H:%M:%S" - ).strftime("%Y-%m-%d %H:%M:%S") - ) - tq = TableQuery( - query=statementType, - user_name=entry.resource.labels["project_id"], - starttime=str(jobStats["startTime"]), - endtime=str(jobStats["endTime"]), - analysis_date=analysis_date, - aborted=0, - database=str(database), - sql=queryConfig["query"], - ) - yield tq + try: + logging_client = logging.Client() + logger = logging_client.logger(self.logger_name) + print("Listing entries for logger {}:".format(logger.name)) + start, end = get_start_and_end(self.config.duration) + entries = logger.list_entries() + for entry in entries: + timestamp = entry.timestamp.isoformat() + timestamp = datetime.strptime(timestamp[0:10], "%Y-%m-%d") + if timestamp >= start and timestamp <= end: + if ("query" in str(entry.payload)) and type( + entry.payload + ) == collections.OrderedDict: + payload = list(entry.payload.items())[-1][1] + if "jobChange" in payload: + print(f"\nEntries: {payload}") + queryConfig = payload["jobChange"]["job"]["jobConfig"][ + "queryConfig" + ] + jobStats = payload["jobChange"]["job"]["jobStats"] + statementType = "" + if hasattr(queryConfig, "statementType"): + statementType = queryConfig["statementType"] + database = "" + if hasattr(queryConfig, "destinationTable"): + database = queryConfig["destinationTable"] + analysis_date = str( + datetime.strptime( + jobStats["startTime"][0:19], "%Y-%m-%dT%H:%M:%S" + ).strftime("%Y-%m-%d %H:%M:%S") + ) + tq = TableQuery( + query=statementType, + user_name=entry.resource.labels["project_id"], + starttime=str(jobStats["startTime"]), + endtime=str(jobStats["endTime"]), + analysis_date=analysis_date, + aborted=0, + database=str(database), + sql=queryConfig["query"], + ) + yield tq + except Exception as err: + logger.error(err) def close(self): pass