yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-06-27 04:22:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

MINOR: Modify Sample data (#21599)

Browse Source
This commit is contained in:
Ayush Shah 2025-06-24 17:16:13 +05:30 committed by GitHub
parent d304c161d0
commit 11ac56356b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 995 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ingestion/examples/sample_data/dashboards/dashboardDataModels.json
View File

File diff suppressed because one or more lines are too long

10
ingestion/examples/sample_data/domains/domain.json Normal file
View File

File diff suppressed because one or more lines are too long

36
ingestion/examples/sample_data/lineage/lineage.json
View File

@ -80,5 +80,41 @@
"type": "dashboard"
},
"edge_meta": { "fqn": "", "type": "" }
},
{
"from": {
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
"type": "table"
},
"to": {
"fqn": "mysql_sample.default.posts_db.Tags",
"type": "table"
},
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
},
{
"from": {
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
"type": "table"
},
"to": {
"fqn": "mysql_sample.default.posts_db.Tags",
"type": "table"
},
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
},
{
"from": {
"fqn": "sample_data.ecommerce_db.shopify.dim_address",
"type": "table"
},
"to": {
"fqn": "mysql_sample.default.posts_db.Tags",
"type": "table"
},
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
}
]

717
ingestion/examples/sample_data/mysql/tables.json
View File

@ -3,7 +3,7 @@
{
"name": "Tags",
"displayName": null,
"fullyQualifiedName": "mysql_sample.default.new_er_database.Tags",
"fullyQualifiedName": "mysql_sample.default.posts_db.Tags",
"description": null,
"tableType": "Regular",
"columns": [
@ -16,7 +16,700 @@
"ordinalPosition": 1
},
{
"name": "name",
"name": "name3094163198113948825877815619",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name34276180133870265105954391099",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2441499925237597726956298255",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2814684936150693784793040409",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1869763170455684210434861925",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name4124554307345419454558400303",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name75886882051625377936942795",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3409393152543106078206012715",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name421540831829594310801570407",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name4003031190279759257319330017",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name19012108203374193108287426115",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1770970018265244420187761576",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name476611616299383227079413588",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name70351798697260971769351375",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name350991760359413296741413974",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name4283862931579085177262724176",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3310958237961866523190661681",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1224596797714534121096587485",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1562876223209206861552514105",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name31647922762300013113413673700",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1912481498589517113047727841",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1659673305932008675691605693",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2637822360235572072870003055",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name238208342434987831873855377",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name420155849991603272613408515",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name15972886689246181101772939375",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name688709923919816581184215225",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1022463067852712999242878794",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name330760591968592581255523907",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1708392467761475191722765562",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1658376835960305241667488302",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name4033359328214956553789157088",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name369833371654718386723959022",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name465018259538438139805269650",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2715371926906221316791802831",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3050180292855408514040736054",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name21470057371546281110932736285",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3099415666579376924925889636",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name36479174507575420980893515",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1207048522823275573719023148",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name261354207560879241822950667",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name507404839708785849834908578",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name356041519142674587045794893",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name78165181278611542418646526",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2775847881247922277830021562",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1541992313919189715096513442",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name220762524955213653585049171",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2005519742774849749874303433",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name610458044848431891871396915",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name365734778477526611223173154",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name425938918137801643639056704",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name2215010653706834389772240463",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1440186534886048547958401836",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name209335940254166988617840543",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name316094430102183377084650459",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3667341401377478345026497716",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name17305634904498865857107781",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3417999330399761618045671580",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name211106106691877115087937883",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1613143231282999794497573691",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1870108582286596771842692713",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name351950139162348739508882307",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name43901162779529439302557163",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3955992500845922464654446333",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1568787278694331673878407012",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name6696584825684261105918556418",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1812534224752309820163311130",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1121161899474705257732444063",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name82060473660912990517778068",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name19803974379854754106168362272",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name4176943752701696279633522269",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name1220833521817492666534807653",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name21443710083362718103903214900",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name961635519888549358895807149",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3442749814965486110711153157",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name3896612358229893411062789826",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name301734380196122686062074610",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
"description": null,
"constraint": "NOT_NULL",
"ordinalPosition": 2
},
{
"name": "name497182842769943118714958952",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",
@ -28,10 +721,10 @@
"databaseSchema": {
"id": "5f40fbdc-7652-4bb5-8dd8-5834c382b8cf",
"type": "databaseSchema",
"name": "new_er_database",
"fullyQualifiedName": "mysql_sample.default.new_er_database",
"name": "posts_db",
"fullyQualifiedName": "mysql_sample.default.posts_db",
"description": null,
"displayName": "new_er_database",
"displayName": "posts_db",
"deleted": false,
"inherited": null
},
@ -55,12 +748,12 @@
"deleted": false,
"inherited": null
},
"sourceUrl": "https://localhost:3306/new_er_database"
"sourceUrl": "https://localhost:3306/posts_db"
},
{
"name": "Users",
"displayName": null,
"fullyQualifiedName": "mysql_sample.default.new_er_database.Users",
"fullyQualifiedName": "mysql_sample.default.posts_db.Users",
"description": null,
"tableType": "Regular",
"columns": [
@ -100,10 +793,10 @@
"databaseSchema": {
"id": "5f40fbdc-7652-4bb5-8dd8-5834c382b8cf",
"type": "databaseSchema",
"name": "new_er_database",
"fullyQualifiedName": "mysql_sample.default.new_er_database",
"name": "posts_db",
"fullyQualifiedName": "mysql_sample.default.posts_db",
"description": null,
"displayName": "new_er_database",
"displayName": "posts_db",
"deleted": false,
"inherited": null
},
@ -127,7 +820,7 @@
"deleted": false,
"inherited": null
},
"sourceUrl": "https://localhost:3306/new_er_database"
"sourceUrl": "https://localhost:3306/posts_db"
},
{
"name": "Categories",
@ -147,7 +840,7 @@
"ordinalPosition": 1
},
{
"name": "name",
"name": "name2507782527390313519868558824",
"dataType": "VARCHAR",
"dataLength": 100,
"dataTypeDisplay": "varchar(100)",

242
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -53,7 +53,11 @@ from metadata.generated.schema.api.data.createTableProfile import (
CreateTableProfileRequest,
)
from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.api.teams.createRole import CreateRoleRequest
from metadata.generated.schema.api.teams.createTeam import CreateTeamRequest
from metadata.generated.schema.api.teams.createUser import CreateUserRequest
@ -82,7 +86,9 @@ from metadata.generated.schema.entity.data.storedProcedure import (
StoredProcedureCode,
)
from metadata.generated.schema.entity.data.table import (
Column,
ColumnProfile,
DataType,
SystemProfile,
Table,
TableData,
@ -94,8 +100,15 @@ from metadata.generated.schema.entity.services.apiService import ApiService
from metadata.generated.schema.entity.services.connections.database.customDatabaseConnection import (
CustomDatabaseConnection,
)
from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
SnowflakeConnection,
)
from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService
from metadata.generated.schema.entity.services.databaseService import (
DatabaseConnection,
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.mlmodelService import MlModelService
from metadata.generated.schema.entity.services.pipelineService import PipelineService
@ -112,7 +125,11 @@ from metadata.generated.schema.tests.resolved import Resolved, TestCaseFailureRe
from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
from metadata.generated.schema.tests.testSuite import TestSuite
from metadata.generated.schema.type.basic import FullyQualifiedEntityName, Timestamp
from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
from metadata.generated.schema.type.entityLineage import (
ColumnLineage,
EntitiesEdge,
LineageDetails,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.generated.schema.type.lifeCycle import AccessDetails, LifeCycle
from metadata.generated.schema.type.schema import Topic as TopicSchema
@ -123,6 +140,7 @@ from metadata.ingestion.models.data_insight import OMetaDataInsightSample
from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
from metadata.ingestion.models.profile_data import OMetaTableProfileSampleData
from metadata.ingestion.models.table_metadata import ColumnDescription
from metadata.ingestion.models.tests_data import (
OMetaLogicalTestSuiteSample,
OMetaTestCaseResolutionStatus,
@ -150,6 +168,17 @@ COLUMN_NAME = "Column"
KEY_TYPE = "Key type"
DATA_TYPE = "Data type"
COL_DESCRIPTION = "Description"
NUM_SERVICES = 1
DATABASES_PER_SERVICE = 5
SCHEMAS_PER_DATABASE = 5
TABLES_PER_SCHEMA = 10
COLUMNS_PER_TABLE = 50
NUM_THREADS = 10
BATCH_SIZE = 10
COLUMNS = [
Column(name=f"column_{i}", dataType=DataType.STRING)
for i in range(COLUMNS_PER_TABLE)
]
TableKey = namedtuple("TableKey", ["schema", "table_name"])
@ -629,6 +658,13 @@ class SampleDataSource(
encoding=UTF_8,
)
)
self.domain = json.load(
open(
sample_data_folder + "/domains/domain.json",
"r",
encoding=UTF_8,
)
)
@classmethod
def create(
@ -647,6 +683,7 @@ class SampleDataSource(
"""Nothing to prepare"""
def _iter(self, *_, **__) -> Iterable[Entity]:
yield from self.ingest_domains()
yield from self.ingest_teams()
yield from self.ingest_users()
yield from self.ingest_tables()
@ -674,6 +711,48 @@ class SampleDataSource(
yield from self.ingest_life_cycle()
yield from self.ingest_api_service()
yield from self.ingest_ometa_api_service()
self.modify_column_descriptions()
yield from self.process_service_batch()
def ingest_domains(self):
domain_request = CreateDomainRequest(**self.domain)
yield Either(right=domain_request)
def modify_column_descriptions(self):
"""
Modify column descriptions to include the table name
"""
table: Table = self.metadata.get_by_name(
entity=Table, fqn="mysql_sample.default.posts_db.Tags"
)
col_desc_list = []
for column in table.columns:
column.description = f"{table.name} - {column.name}"
col_desc_list.append(
ColumnDescription(
column_fqn=column.fullyQualifiedName.root,
description=column.description,
)
)
self.metadata.patch_column_descriptions(
table=table,
column_descriptions=col_desc_list,
)
self.metadata.patch_column_descriptions(
table=table,
column_descriptions=[
ColumnDescription(
column_fqn=column.fullyQualifiedName.root, description=None
)
for column in table.columns
],
)
self.metadata.patch_column_descriptions(
table=table,
column_descriptions=col_desc_list,
)
def ingest_teams(self) -> Iterable[Either[CreateTeamRequest]]:
"""
@ -752,6 +831,7 @@ class SampleDataSource(
tableConstraints=table.get("tableConstraints"),
tableType=table["tableType"],
sourceUrl=table.get("sourceUrl"),
domain="TestDomain",
)
yield Either(right=table_request)
@ -1840,3 +1920,161 @@ class SampleDataSource(
for endpoint in self.ometa_api_endpoint.get("endpoints"):
endpoint_request = CreateAPIEndpointRequest(**endpoint)
yield Either(right=endpoint_request)
def create_database_service(self, service_idx: int) -> None:
"""Create a database service and its databases.
Args:
service_idx: Service index
"""
service_name = f"openmetadata-{service_idx}"
try:
# Create minimal Snowflake connection
connection = DatabaseConnection(
config=SnowflakeConnection(
username="dummy",
password="dummy", # This will be handled by the library
account="dummy",
database="dummy",
warehouse="dummy",
)
)
# Create service with minimal required fields
yield Either(
right=CreateDatabaseServiceRequest(
name=service_name,
serviceType=DatabaseServiceType.Snowflake,
connection=connection,
)
)
logger.info(f"Created database service {service_name} ({NUM_SERVICES})")
tasks = []
# Create databases sequentially
for db_idx in range(DATABASES_PER_SERVICE):
yield from self.create_database(service_name, db_idx)
except Exception as e:
logger.error(f"Failed to create database service {service_name}: {e}")
def process_service_batch(self) -> None:
"""Process a batch of services.
Args:
start_idx: Start index of service batch
end_idx: End index of service batch
"""
services_per_thread = NUM_SERVICES // NUM_THREADS
# Create tasks for each thread
for service_idx in range(NUM_SERVICES):
yield from self.create_database_service(service_idx)
# create table and column lineage from the snowflake sample data to Mysql Table `Tags`
yield from self.create_table_lineage()
def create_table_lineage(self) -> None:
"""Create table lineage from the snowflake sample data to Mysql Table `Tags`"""
source_table_list = list(
self.metadata.list_entities(
entity=Table,
limit=5,
params={"database": "openmetadata-0.openmetadata-db-0"},
).entities
)
destination_table = self.metadata.get_by_name(
Table, "mysql_sample.default.posts_db.Tags"
)
for source_table in source_table_list:
yield Either(
right=AddLineageRequest(
edge=EntitiesEdge(
fromEntity=EntityReference(id=source_table.id, type="table"),
toEntity=EntityReference(id=destination_table.id, type="table"),
lineageDetails=LineageDetails(
columnsLineage=[
ColumnLineage(
fromColumns=[
from_column.fullyQualifiedName.root
for from_column in source_table.columns
][:5],
toColumn=to_column.fullyQualifiedName.root,
)
for to_column in destination_table.columns
]
),
)
)
)
def create_database(self, service_name: str, db_idx: int) -> None:
"""Create a database.
Args:
service_name: Service name
db_idx: Database index
"""
db_name = f"openmetadata-db-{db_idx}"
try:
# Create with minimal required fields
db_request = Either(
right=CreateDatabaseRequest(name=db_name, service=service_name)
)
yield db_request
database_fqn = f"{service_name}.{db_name}"
# Create schemas sequentially to avoid overwhelming the API
for schema_idx in range(SCHEMAS_PER_DATABASE):
yield from self.create_schema(database_fqn, schema_idx)
except Exception as e:
logger.error(f"Failed to create database {db_name}: {e}")
def create_schema(self, database_fqn: str, schema_idx: int) -> None:
"""Create a schema.
Args:
database_fqn: Database FQN
schema_idx: Schema index
"""
schema_name = f"openmetadata-schema-{schema_idx}"
try:
# Create with minimal required fields
schema_request = Either(
right=CreateDatabaseSchemaRequest(
name=schema_name, database=database_fqn
)
)
yield schema_request
schema_name = f"{database_fqn}.{schema_name}"
# Create tables sequentially to avoid overwhelming the API
yield from self.create_table(schema_name)
except Exception as e:
logger.error(f"Failed to create schema {schema_name}: {e}")
def create_table(self, schema_fqn: str) -> None:
"""Create a batch of tables for a schema.
Args:
schema_fqn: Fully qualified schema name
"""
# Create table requests
for i in range(TABLES_PER_SCHEMA):
table_name = f"openmetadata-table-{i}"
# Create with minimal required fields
try:
table_request = Either(
right=CreateTableRequest(
name=table_name, databaseSchema=schema_fqn, columns=COLUMNS
)
)
yield table_request
except Exception as e:
logger.warning(f"Error creating table request: {e}")

4
ingestion/tests/integration/ometa/test_ometa_table_api.py
View File

@ -291,7 +291,9 @@ class OMetaTableTest(TestCase):
self.metadata.create_or_update(data=self.create)
res = self.metadata.list_entities(entity=Table)
res = self.metadata.list_entities(
entity=Table, params={"database": "test-service-table.test-db"}
)
# Fetch our test Database. We have already inserted it, so we should find it
data = next(