mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-13 08:37:03 +00:00
This commit is contained in:
Sriharsha Chintalapani
GitHub
parent
512dbd6201
commit
3d10347aec
File diff suppressed because it is too large
Load Diff
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: data/tags/personalDataTags.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: data/tags/piiTags.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: data/tags/tierTags.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: data/tags/userTags.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/catalogVersion.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createChart.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -14,7 +14,7 @@ from ...type import entityReference, tagLabel
|
||||
|
||||
class CreateChartEntityRequest(BaseModel):
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this dashboard.'
|
||||
..., description='Name that identifies this Chart.'
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
@ -22,7 +22,7 @@ class CreateChartEntityRequest(BaseModel):
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None,
|
||||
description='Description of the database instance. What it has and how to use it.',
|
||||
description='Description of the chart instance. What it has and how to use it.',
|
||||
)
|
||||
chartType: Optional[chart.ChartType] = None
|
||||
chartUrl: Optional[AnyUrl] = Field(
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createDashboard.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createDatabase.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -0,0 +1,41 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createPipeline.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...type import entityReference, tagLabel
|
||||
|
||||
|
||||
class CreatePipelineEntityRequest(BaseModel):
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this pipeline instance uniquely.'
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Pipeline. It could be title or label from the source services.',
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None,
|
||||
description='Description of the database instance. What it has and how to use it.',
|
||||
)
|
||||
pipelineUrl: Optional[AnyUrl] = Field(
|
||||
None,
|
||||
description='Pipeline URL to visit/manage. This URL points to respective pipeline service UI',
|
||||
)
|
||||
tasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are part of pipeline.'
|
||||
)
|
||||
tags: Optional[List[tagLabel.TagLabel]] = Field(
|
||||
None, description='Tags for this Pipeline.'
|
||||
)
|
||||
owner: Optional[entityReference.EntityReference] = Field(
|
||||
None, description='Owner of this database'
|
||||
)
|
||||
service: entityReference.EntityReference = Field(
|
||||
..., description='Link to the database service where this database is hosted in'
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createTable.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -0,0 +1,48 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createTask.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...entity.data import task
|
||||
from ...type import entityReference, tagLabel
|
||||
|
||||
|
||||
class CreateTaskEntityRequest(BaseModel):
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this Task.'
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Task. It could be title or label from the pipeline services',
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None,
|
||||
description='Description of the task instance. What it has and how to use it.',
|
||||
)
|
||||
taskUrl: Optional[AnyUrl] = Field(
|
||||
None,
|
||||
description='Task URL to visit/manage. This URL points to respective pipeline service UI',
|
||||
)
|
||||
upstreamTasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are upstream of this task.'
|
||||
)
|
||||
downstreamTasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are downstream of this task.'
|
||||
)
|
||||
taskConfig: Optional[task.TaskConfig] = Field(
|
||||
None, description='Task Configuration.'
|
||||
)
|
||||
tags: Optional[List[tagLabel.TagLabel]] = Field(
|
||||
None, description='Tags for this chart'
|
||||
)
|
||||
owner: Optional[entityReference.EntityReference] = Field(
|
||||
None, description='Owner of this Task'
|
||||
)
|
||||
service: entityReference.EntityReference = Field(
|
||||
..., description='Link to the pipeline service where this task is used'
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/data/createTopic.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/feed/createThread.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/createDashboardService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/createDatabaseService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/createMessagingService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/createPipelineService.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...entity.services import pipelineService
|
||||
from ...type import schedule
|
||||
|
||||
|
||||
class CreatePipelineServiceEntityRequest(BaseModel):
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies the this entity instance uniquely'
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None, description='Description of pipeline service entity.'
|
||||
)
|
||||
serviceType: pipelineService.PipelineServiceType
|
||||
pipelineUrl: AnyUrl = Field(..., description='Pipeline UI URL')
|
||||
ingestionSchedule: Optional[schedule.Schedule] = Field(
|
||||
None, description='Schedule for running pipeline ingestion jobs'
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/updateDashboardService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/updateDatabaseService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/updateMessagingService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/services/updatePipelineService.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field
|
||||
|
||||
from ...type import schedule
|
||||
|
||||
|
||||
class UpdatePipelineServiceEntityRequest(BaseModel):
|
||||
description: Optional[str] = Field(
|
||||
None, description='Description of Pipeline service entity.'
|
||||
)
|
||||
pipelineUrl: Optional[AnyUrl] = Field(None, description='Pipeline Service UI URL.')
|
||||
ingestionSchedule: Optional[schedule.Schedule] = Field(
|
||||
None, description='Schedule for running metadata ingestion jobs'
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/setOwner.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/tags/createTag.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/tags/createTagCategory.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/teams/createTeam.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/api/teams/createUser.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/bots.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/chart.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -34,7 +34,7 @@ class Chart(BaseModel):
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Chart. It could be title or label from the source services',
|
||||
description='Display Name that identifies this Chart. It could be title or label from the source services.',
|
||||
)
|
||||
fullyQualifiedName: Optional[constr(min_length=1, max_length=64)] = Field(
|
||||
None,
|
||||
@ -45,7 +45,7 @@ class Chart(BaseModel):
|
||||
)
|
||||
chartType: Optional[ChartType] = None
|
||||
chartUrl: Optional[AnyUrl] = Field(
|
||||
None, description='Chart URL, pointing to its own Service URL'
|
||||
None, description='Chart URL, pointing to its own Service URL.'
|
||||
)
|
||||
href: Optional[basic.Href] = Field(
|
||||
None, description='Link to the resource corresponding to this entity.'
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/dashboard.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -20,7 +20,7 @@ class Dashboard(BaseModel):
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Dashboard. It could be title or label from the source services',
|
||||
description='Display Name that identifies this Dashboard. It could be title or label from the source services.',
|
||||
)
|
||||
fullyQualifiedName: Optional[constr(min_length=1, max_length=64)] = Field(
|
||||
None,
|
||||
@ -29,7 +29,7 @@ class Dashboard(BaseModel):
|
||||
description: Optional[str] = Field(
|
||||
None, description='Description of the dashboard, what it is, and how to use it.'
|
||||
)
|
||||
dashboardUrl: Optional[AnyUrl] = Field(None, description='Dashboard URL')
|
||||
dashboardUrl: Optional[AnyUrl] = Field(None, description='Dashboard URL.')
|
||||
charts: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the charts included in this Dashboard.'
|
||||
)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/database.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/metrics.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/pipeline.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, constr
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...type import basic, entityReference
|
||||
from ...type import basic, entityReference, tagLabel
|
||||
|
||||
|
||||
class Pipeline(BaseModel):
|
||||
@ -18,12 +18,29 @@ class Pipeline(BaseModel):
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this pipeline instance uniquely.'
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Pipeline. It could be title or label from the source services.',
|
||||
)
|
||||
fullyQualifiedName: Optional[constr(min_length=1, max_length=64)] = Field(
|
||||
None,
|
||||
description="A unique name that identifies a pipeline in the format 'ServiceName.PipelineName'.",
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None, description='Description of this pipeline.'
|
||||
None, description='Description of this Pipeline.'
|
||||
)
|
||||
pipelineUrl: Optional[AnyUrl] = Field(
|
||||
None,
|
||||
description='Pipeline URL to visit/manage. This URL points to respective pipeline service UI',
|
||||
)
|
||||
tasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are part of pipeline.'
|
||||
)
|
||||
followers: Optional[entityReference.EntityReferenceList] = Field(
|
||||
None, description='Followers of this Pipeline.'
|
||||
)
|
||||
tags: Optional[List[tagLabel.TagLabel]] = Field(
|
||||
None, description='Tags for this Pipeline.'
|
||||
)
|
||||
href: Optional[basic.Href] = Field(
|
||||
None, description='Link to the resource corresponding to this entity.'
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/report.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/table.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -20,12 +20,13 @@ class TableType(Enum):
|
||||
MaterializedView = 'MaterializedView'
|
||||
|
||||
|
||||
class ColumnDataType(Enum):
|
||||
class DataType(Enum):
|
||||
NUMBER = 'NUMBER'
|
||||
TINYINT = 'TINYINT'
|
||||
SMALLINT = 'SMALLINT'
|
||||
INT = 'INT'
|
||||
BIGINT = 'BIGINT'
|
||||
BYTEINT = 'BYTEINT'
|
||||
FLOAT = 'FLOAT'
|
||||
DOUBLE = 'DOUBLE'
|
||||
DECIMAL = 'DECIMAL'
|
||||
@ -56,7 +57,7 @@ class ColumnDataType(Enum):
|
||||
JSON = 'JSON'
|
||||
|
||||
|
||||
class ColumnConstraint(Enum):
|
||||
class Constraint(Enum):
|
||||
NULL = 'NULL'
|
||||
NOT_NULL = 'NOT_NULL'
|
||||
UNIQUE = 'UNIQUE'
|
||||
@ -78,7 +79,8 @@ class TableConstraint(BaseModel):
|
||||
|
||||
class ColumnName(BaseModel):
|
||||
__root__: constr(regex=r'^[^.]*$', min_length=1, max_length=64) = Field(
|
||||
..., description='Local name (not fully qualified name) of the column.'
|
||||
...,
|
||||
description='Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields',
|
||||
)
|
||||
|
||||
|
||||
@ -91,7 +93,7 @@ class TableName(BaseModel):
|
||||
class FullyQualifiedColumnName(BaseModel):
|
||||
__root__: constr(min_length=1, max_length=256) = Field(
|
||||
...,
|
||||
description='Fully qualified name of the column that includes `serviceName.databaseName.tableName.columnName`.',
|
||||
description='Fully qualified name of the column that includes `serviceName.databaseName.tableName.columnName[.nestedColumnName]`. When columnName is null for dataType struct fields, `field_#` where `#` is field index is used. For map dataType, for key the field name `key` is used and for the value field `value` is used.',
|
||||
)
|
||||
|
||||
|
||||
@ -127,16 +129,16 @@ class TableData(BaseModel):
|
||||
class ColumnProfile(BaseModel):
|
||||
name: Optional[str] = Field(None, description='Column Name.')
|
||||
uniqueCount: Optional[float] = Field(
|
||||
None, description='No. of unique values in the column'
|
||||
None, description='No. of unique values in the column.'
|
||||
)
|
||||
uniqueProportion: Optional[float] = Field(
|
||||
None, description='Proportion of number of unique values in a column'
|
||||
None, description='Proportion of number of unique values in a column.'
|
||||
)
|
||||
nullCount: Optional[float] = Field(
|
||||
None, description='No.of null values in a column'
|
||||
None, description='No.of null values in a column.'
|
||||
)
|
||||
nullProportion: Optional[float] = Field(
|
||||
None, description='No.of null value proportion in columns'
|
||||
None, description='No.of null value proportion in columns.'
|
||||
)
|
||||
min: Optional[str] = Field(None, description='Minimum value in a column.')
|
||||
max: Optional[str] = Field(None, description='Maximum value in a column.')
|
||||
@ -173,21 +175,43 @@ class TableProfile(BaseModel):
|
||||
|
||||
|
||||
class Column(BaseModel):
|
||||
class Config:
|
||||
extra = Extra.forbid
|
||||
|
||||
name: ColumnName
|
||||
columnDataType: ColumnDataType = Field(
|
||||
dataType: DataType = Field(
|
||||
..., description='Data type of the column (int, date etc.).'
|
||||
)
|
||||
arrayDataType: Optional[DataType] = Field(
|
||||
None,
|
||||
description='Data type used array in dataType. For example, `array<int>` has dataType as `array` and arrayDataType as `int`.',
|
||||
)
|
||||
dataLength: Optional[int] = Field(
|
||||
None,
|
||||
description='Length of `char`, `varchar`, `binary`, `varbinary` `dataTypes`, else null. For example, `varchar(20)` has dataType as `varchar` and dataLength as `20`.',
|
||||
)
|
||||
dataTypeDisplay: Optional[str] = Field(
|
||||
None,
|
||||
description='Display name used for dataType. This is useful for complex types, such as `array<int>, map<int,string>, struct<>, and union types.',
|
||||
)
|
||||
description: Optional[str] = Field(None, description='Description of the column.')
|
||||
fullyQualifiedName: Optional[FullyQualifiedColumnName] = None
|
||||
tags: Optional[List[tagLabel.TagLabel]] = Field(
|
||||
None, description='Tags associated with the column.'
|
||||
)
|
||||
columnConstraint: Optional[ColumnConstraint] = Field(
|
||||
constraint: Optional[Constraint] = Field(
|
||||
None, description='Column level constraint.'
|
||||
)
|
||||
ordinalPosition: Optional[int] = Field(
|
||||
None, description='Ordinal position of the column.'
|
||||
)
|
||||
jsonSchema: Optional[str] = Field(
|
||||
None, description='Json schema only if the dataType is JSON else null.'
|
||||
)
|
||||
children: Optional[List[Column]] = Field(
|
||||
None,
|
||||
description='Child columns if dataType or arrayDataType is `map`, `struct`, or `union` else `null`.',
|
||||
)
|
||||
|
||||
|
||||
class Table(BaseModel):
|
||||
@ -234,3 +258,6 @@ class Table(BaseModel):
|
||||
tableProfile: Optional[List[TableProfile]] = Field(
|
||||
None, description='Data profile for a table.'
|
||||
)
|
||||
|
||||
|
||||
Column.update_forward_refs()
|
||||
|
||||
61
ingestion/src/metadata/generated/schema/entity/data/task.py
Normal file
61
ingestion/src/metadata/generated/schema/entity/data/task.py
Normal file
@ -0,0 +1,61 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/task.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...type import basic, entityReference, tagLabel
|
||||
|
||||
|
||||
class TaskConfig(BaseModel):
|
||||
codeLocation: Optional[str] = Field(None, description='Location of task file')
|
||||
startDate: Optional[basic.Date] = Field(None, description='Start Date of the task')
|
||||
concurrency: Optional[int] = Field(None, description='Concurrency of the Task')
|
||||
|
||||
|
||||
class Task(BaseModel):
|
||||
id: basic.Uuid = Field(
|
||||
..., description='Unique identifier that identifies a task instance.'
|
||||
)
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this task instance uniquely.'
|
||||
)
|
||||
displayName: Optional[str] = Field(
|
||||
None,
|
||||
description='Display Name that identifies this Task. It could be title or label from the pipeline services.',
|
||||
)
|
||||
fullyQualifiedName: Optional[constr(min_length=1, max_length=64)] = Field(
|
||||
None,
|
||||
description="A unique name that identifies a pipeline in the format 'ServiceName.PipelineName.TaskName'.",
|
||||
)
|
||||
description: Optional[str] = Field(None, description='Description of this Task.')
|
||||
taskUrl: Optional[AnyUrl] = Field(
|
||||
None,
|
||||
description='Task URL to visit/manage. This URL points to respective pipeline service UI',
|
||||
)
|
||||
upstreamTasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are upstream of this task.'
|
||||
)
|
||||
downstreamTasks: Optional[List[entityReference.EntityReference]] = Field(
|
||||
None, description='All the tasks that are downstream of this task.'
|
||||
)
|
||||
taskConfig: Optional[TaskConfig] = Field(None, description='Task Configuration.')
|
||||
followers: Optional[entityReference.EntityReferenceList] = Field(
|
||||
None, description='Followers of this Pipeline.'
|
||||
)
|
||||
tags: Optional[List[tagLabel.TagLabel]] = Field(
|
||||
None, description='Tags for this Pipeline.'
|
||||
)
|
||||
href: Optional[basic.Href] = Field(
|
||||
None, description='Link to the resource corresponding to this entity.'
|
||||
)
|
||||
owner: Optional[entityReference.EntityReference] = Field(
|
||||
None, description='Owner of this pipeline.'
|
||||
)
|
||||
service: entityReference.EntityReference = Field(
|
||||
..., description='Link to service where this pipeline is hosted in.'
|
||||
)
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/data/topic.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/feed/thread.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/services/dashboardService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@ -34,13 +34,13 @@ class DashboardService(BaseModel):
|
||||
)
|
||||
dashboardUrl: AnyUrl = Field(
|
||||
...,
|
||||
description='Dashboard Service URL. This will be used to make REST API calls to Dashboard Service',
|
||||
description='Dashboard Service URL. This will be used to make REST API calls to Dashboard Service.',
|
||||
)
|
||||
username: Optional[str] = Field(
|
||||
None, description='Username to log-into Dashboard Service'
|
||||
None, description='Username to log-into Dashboard Service.'
|
||||
)
|
||||
password: Optional[str] = Field(
|
||||
None, description='Password to log-into Dashboard Service'
|
||||
None, description='Password to log-into Dashboard Service.'
|
||||
)
|
||||
ingestionSchedule: Optional[schedule.Schedule] = Field(
|
||||
None, description='Schedule for running metadata ingestion jobs.'
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/services/databaseService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/services/messagingService.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/services/pipelineService.json
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import AnyUrl, BaseModel, Field, constr
|
||||
|
||||
from ...type import basic, schedule
|
||||
|
||||
|
||||
class PipelineServiceType(Enum):
|
||||
Airflow = 'Airflow'
|
||||
Prefect = 'Prefect'
|
||||
|
||||
|
||||
class MessagingService(BaseModel):
|
||||
id: basic.Uuid = Field(
|
||||
..., description='Unique identifier of this pipeline service instance.'
|
||||
)
|
||||
name: constr(min_length=1, max_length=64) = Field(
|
||||
..., description='Name that identifies this pipeline service.'
|
||||
)
|
||||
serviceType: Optional[PipelineServiceType] = Field(
|
||||
None, description='Type of pipeline service such as Airflow or Prefect...'
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None, description='Description of a messaging service instance.'
|
||||
)
|
||||
pipelineUrl: AnyUrl = Field(..., description='Pipeline Service Management/UI URL')
|
||||
ingestionSchedule: Optional[schedule.Schedule] = Field(
|
||||
None, description='Schedule for running metadata ingestion jobs.'
|
||||
)
|
||||
href: Optional[basic.Href] = Field(
|
||||
None,
|
||||
description='Link to the resource corresponding to this messaging service.',
|
||||
)
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/tags/tagCategory.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/teams/team.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/entity/teams/user.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/auditLog.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/basic.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/collectionDescriptor.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/dailyCount.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/entityReference.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Extra, Field
|
||||
|
||||
from . import basic
|
||||
|
||||
|
||||
class EntityReference(BaseModel):
|
||||
class Config:
|
||||
extra = Extra.forbid
|
||||
|
||||
id: basic.Uuid = Field(
|
||||
..., description='Unique identifier that identifies an entity instance.'
|
||||
)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/entityUsage.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/jdbcConnection.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/paging.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/profile.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/schedule.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/tagLabel.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field, constr
|
||||
from pydantic import BaseModel, Extra, Field, constr
|
||||
|
||||
from . import basic
|
||||
|
||||
@ -25,6 +25,9 @@ class State(Enum):
|
||||
|
||||
|
||||
class TagLabel(BaseModel):
|
||||
class Config:
|
||||
extra = Extra.forbid
|
||||
|
||||
tagFQN: Optional[constr(max_length=45)] = None
|
||||
labelType: Optional[LabelType] = Field(
|
||||
'Manual',
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# generated by datamodel-codegen:
|
||||
# filename: schema/type/usageDetails.json
|
||||
# timestamp: 2021-09-13T04:07:21+00:00
|
||||
# timestamp: 2021-09-27T15:46:37+00:00
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@ -247,13 +247,13 @@ class GenerateFakeSampleData:
|
||||
elif "country" in col_name:
|
||||
value = fake.country()
|
||||
if value is None:
|
||||
if "TIMESTAMP" in column['columnDataType'] or "date" in col_name:
|
||||
if "TIMESTAMP" in column['dataType'] or "date" in col_name:
|
||||
value = fake.unix_time()
|
||||
elif "BOOLEAN" in column['columnDataType']:
|
||||
elif "BOOLEAN" in column['dataType']:
|
||||
value = fake.pybool()
|
||||
elif "NUMERIC" in column['columnDataType']:
|
||||
elif "NUMERIC" in column['dataType']:
|
||||
value = fake.pyint()
|
||||
elif "VARCHAR" in column['columnDataType']:
|
||||
elif "VARCHAR" in column['dataType']:
|
||||
value = fake.text(max_nb_chars=20)
|
||||
else:
|
||||
value = None
|
||||
|
||||
@ -28,9 +28,8 @@ from metadata.generated.schema.type.entityReference import EntityReference
|
||||
|
||||
from metadata.generated.schema.entity.data.database import Database
|
||||
|
||||
from metadata.generated.schema.entity.data.table import Table, Column, ColumnConstraint, TableType, \
|
||||
TableData, \
|
||||
TableProfile
|
||||
from metadata.generated.schema.entity.data.table import Table, Column, TableType, Constraint, \
|
||||
TableData, TableProfile, ConstraintType, TableConstraint
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.sql import sqltypes as types
|
||||
@ -370,23 +369,31 @@ class SQLSource(Source):
|
||||
logger.error(err)
|
||||
col_constraint = None
|
||||
if column['nullable']:
|
||||
col_constraint = ColumnConstraint.NULL
|
||||
col_constraint = Constraint.NULL
|
||||
elif not column['nullable']:
|
||||
col_constraint = ColumnConstraint.NOT_NULL
|
||||
col_constraint = Constraint.NOT_NULL
|
||||
|
||||
if column['name'] in pk_columns:
|
||||
col_constraint = ColumnConstraint.PRIMARY_KEY
|
||||
col_constraint = Constraint.PRIMARY_KEY
|
||||
elif column['name'] in unique_columns:
|
||||
col_constraint = ColumnConstraint.UNIQUE
|
||||
table_columns.append(
|
||||
Column(
|
||||
col_constraint = Constraint.UNIQUE
|
||||
col_data_length = None
|
||||
if col_type in ['CHAR', 'VARCHAR', 'BINARY', 'VARBINARY']:
|
||||
col_data_length = column['type'].length
|
||||
|
||||
om_column = Column(
|
||||
name=column['name'],
|
||||
description=column.get("comment", None),
|
||||
columnDataType=col_type,
|
||||
columnConstraint=col_constraint,
|
||||
dataType=col_type,
|
||||
dataTypeDisplay=col_type,
|
||||
constraint=col_constraint,
|
||||
ordinalPosition=row_order
|
||||
)
|
||||
)
|
||||
|
||||
if col_data_length is not None:
|
||||
om_column.dataLength = col_data_length
|
||||
|
||||
table_columns.append(om_column)
|
||||
row_order = row_order + 1
|
||||
|
||||
return table_columns
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user