diff --git a/ingestion/examples/sample_data/ometa_api_service/ometa_api_collection.json b/ingestion/examples/sample_data/ometa_api_service/ometa_api_collection.json new file mode 100644 index 00000000000..3a4646cafc9 --- /dev/null +++ b/ingestion/examples/sample_data/ometa_api_service/ometa_api_collection.json @@ -0,0 +1,14 @@ +{ + "collections" : [ + { + "name": "tables", + "service": "ometa_api_service", + "endpointURL": "https://docs.open-metadata.org/swagger.html#tag/Tables" + }, + { + "name": "users", + "service": "ometa_api_service", + "endpointURL": "https://docs.open-metadata.org/swagger.html#tag/Users" + } + ] +} \ No newline at end of file diff --git a/ingestion/examples/sample_data/ometa_api_service/ometa_api_endpoint.json b/ingestion/examples/sample_data/ometa_api_service/ometa_api_endpoint.json new file mode 100644 index 00000000000..ae6119a2ad4 --- /dev/null +++ b/ingestion/examples/sample_data/ometa_api_service/ometa_api_endpoint.json @@ -0,0 +1,266 @@ +{ + "endpoints": [ + { + "name": "listTables", + "displayName": "List tables", + "description": "Get a list of tables", + "endpointURL": "https://docs.open-metadata.org/api/v1/tables", + "apiCollection": "ometa_api_service.tables", + "requestMethod": "GET", + "responseSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "data", + "dataType": "RECORD", + "description": "list of tables", + "children": [ + { + "name": "id", + "dataType": "STRING", + "description": "ID of table" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of table" + }, + { + "name": "columns", + "dataType": "RECORD", + "description": "columns of table", + "children": [] + } + ] + }, + { + "name": "errors", + "dataType": "RECORD", + "description": "list of errors", + "children": [ + { + "name": "entity", + "dataType": "RECORD", + "description": "entity object", + "children": [] + }, + { + "name": "message", + "dataType": "STRING", + "description": "error message" + } + ] + }, + { + "name": "paging", + "dataType": "RECORD", + "description": "paging", + "children": [ + { + "name": "total", + "dataType": "INT", + "description": "total pages" + } + ] + } + ] + } + }, + { + "name": "createTable", + "displayName": "Create a table", + "description": "Create a new table under an existing database", + "endpointURL": "https://docs.open-metadata.org/api/v1/tables", + "apiCollection": "ometa_api_service.tables", + "requestMethod": "POST", + "requestSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "databaseSchema", + "dataType": "STRING", + "description": "db schema of new table" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of table" + }, + { + "name": "columns", + "dataType": "RECORD", + "description": "list of columns", + "children": [ + { + "name": "name", + "dataType": "STRING", + "description": "name of column" + }, + { + "name": "datatype", + "dataType": "STRING", + "description": "datatype of column" + } + ] + } + ] + }, + "responseSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "id", + "dataType": "STRING", + "description": "id of created table" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of created table" + }, + { + "name": "columns", + "dataType": "RECORD", + "description": "list of columns", + "children": [ + { + "name": "name", + "dataType": "STRING", + "description": "name of column" + }, + { + "name": "datatype", + "dataType": "STRING", + "description": "datatype of column" + } + ] + } + ] + } + }, + { + "name": "deleteTable", + "apiCollection": "ometa_api_service.tables", + "displayName": "Delete table by ID", + "description": "Delete table by ID", + "endpointURL": "https://docs.open-metadata.org/api/v1/tables/{id}", + "requestMethod": "DELETE" + }, + { + "name": "listUsers", + "displayName": "List users", + "description": "Get a list of users", + "endpointURL": "https://docs.open-metadata.org/api/v1/users", + "apiCollection": "ometa_api_service.users", + "requestMethod": "GET", + "responseSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "data", + "dataType": "RECORD", + "description": "list of users", + "children": [ + { + "name": "id", + "dataType": "STRING", + "description": "ID of user" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of user" + }, + { + "name": "email", + "dataType": "STRING", + "description": "email id of user" + } + ] + }, + { + "name": "errors", + "dataType": "RECORD", + "description": "list of errors", + "children": [ + { + "name": "entity", + "dataType": "RECORD", + "description": "entity object", + "children": [] + }, + { + "name": "message", + "dataType": "STRING", + "description": "error message" + } + ] + }, + { + "name": "paging", + "dataType": "RECORD", + "description": "paging", + "children": [ + { + "name": "total", + "dataType": "INT", + "description": "total pages" + } + ] + } + ] + } + }, + { + "name": "createUser", + "displayName": "Create a user", + "description": "Create a new user", + "endpointURL": "https://docs.open-metadata.org/api/v1/users", + "apiCollection": "ometa_api_service.users", + "requestMethod": "POST", + "requestSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "email", + "dataType": "STRING", + "description": "email of new user" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of new user" + } + ] + }, + "responseSchema": { + "schemaType": "JSON", + "schemaFields": [ + { + "name": "id", + "dataType": "STRING", + "description": "ID of user" + }, + { + "name": "name", + "dataType": "STRING", + "description": "name of user" + }, + { + "name": "email", + "dataType": "STRING", + "description": "email id of user" + } + ] + } + }, + { + "name": "deleteUser", + "apiCollection": "ometa_api_service.users", + "displayName": "Delete user by ID", + "description": "Delete user by ID", + "endpointURL": "https://docs.open-metadata.org/api/v1/users/{id}", + "requestMethod": "DELETE" + } + ] +} \ No newline at end of file diff --git a/ingestion/examples/sample_data/ometa_api_service/service.json b/ingestion/examples/sample_data/ometa_api_service/service.json new file mode 100644 index 00000000000..6e3c672e960 --- /dev/null +++ b/ingestion/examples/sample_data/ometa_api_service/service.json @@ -0,0 +1,14 @@ +{ + "type": "REST", + "serviceName": "ometa_api_service", + "serviceConnection": { + "config": { + "type": "REST", + "openAPISchemaURL": "https://docs.open-metadata.org/swagger.html", + "token":"token" + } + }, + "sourceConfig": { + "config": {} + } +} \ No newline at end of file diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index 388d9235016..24b84bbf06f 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -568,6 +568,31 @@ class SampleDataSource( encoding=UTF_8, ) ) + self.ometa_api_service_json = json.load( + open( # pylint: disable=consider-using-with + sample_data_folder + "/ometa_api_service/service.json", + "r", + encoding=UTF_8, + ) + ) + self.ometa_api_service = self.metadata.get_service_or_create( + entity=ApiService, + config=WorkflowSource(**self.ometa_api_service_json), + ) + self.ometa_api_collection = json.load( + open( + sample_data_folder + "/ometa_api_service/ometa_api_collection.json", + "r", + encoding=UTF_8, + ) + ) + self.ometa_api_endpoint = json.load( + open( + sample_data_folder + "/ometa_api_service/ometa_api_endpoint.json", + "r", + encoding=UTF_8, + ) + ) @classmethod def create( @@ -611,6 +636,7 @@ class SampleDataSource( yield from self.ingest_data_insights() yield from self.ingest_life_cycle() yield from self.ingest_api_service() + yield from self.ingest_ometa_api_service() def ingest_teams(self) -> Iterable[Either[CreateTeamRequest]]: """ @@ -1698,3 +1724,14 @@ class SampleDataSource( for endpoint in self.api_endpoint.get("endpoints"): endpoint_request = CreateAPIEndpointRequest(**endpoint) yield Either(right=endpoint_request) + + def ingest_ometa_api_service(self) -> Iterable[Either[Entity]]: + """Ingest users & tables ometa API services""" + + for collection in self.ometa_api_collection.get("collections"): + collection_request = CreateAPICollectionRequest(**collection) + yield Either(right=collection_request) + + for endpoint in self.ometa_api_endpoint.get("endpoints"): + endpoint_request = CreateAPIEndpointRequest(**endpoint) + yield Either(right=endpoint_request)