diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py index 0b1f69548e7..3e625c0cc34 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py @@ -17,11 +17,7 @@ from typing import Iterable, List, Optional from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest -from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import ChartType -from metadata.generated.schema.entity.data.dashboard import ( - Dashboard as Lineage_Dashboard, -) from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin @@ -86,48 +82,12 @@ class SupersetAPISource(SupersetSourceMixin): ), ) - def yield_dashboard_lineage_details( - self, dashboard_details: dict, db_service_name: str - ) -> Optional[Iterable[AddLineageRequest]]: - """ - Get lineage between dashboard and data sources - """ - for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if chart_json: - datasource_fqn = ( - self._get_datasource_fqn( - chart_json.get("datasource_id"), db_service_name - ) - if chart_json.get("datasource_id") - else None - ) - if not datasource_fqn: - continue - from_entity = self.metadata.get_by_name( - entity=Table, - fqn=datasource_fqn, - ) - try: - dashboard_fqn = fqn.build( - self.metadata, - entity_type=Lineage_Dashboard, - service_name=self.config.serviceName, - dashboard_name=str(dashboard_details["id"]), - ) - to_entity = self.metadata.get_by_name( - entity=Lineage_Dashboard, - fqn=dashboard_fqn, - ) - if from_entity and to_entity: - yield self._get_add_lineage_request( - to_entity=to_entity, from_entity=from_entity - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.error( - f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}" - ) + def _get_datasource_fqn_for_lineage(self, chart_json, db_service_name): + return ( + self._get_datasource_fqn(chart_json.get("datasource_id"), db_service_name) + if chart_json.get("datasource_id") + else None + ) def yield_dashboard_chart( self, dashboard_details: dict diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py index 242dacdeda0..762d59213e5 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py @@ -19,11 +19,7 @@ from sqlalchemy.engine import Engine from metadata.generated.schema.api.data.createChart import CreateChartRequest from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest -from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.chart import ChartType -from metadata.generated.schema.entity.data.dashboard import ( - Dashboard as Lineage_Dashboard, -) from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, @@ -61,7 +57,7 @@ class SupersetDBSource(SupersetSourceMixin): """ charts = self.engine.execute(FETCH_ALL_CHARTS) for chart in charts: - self.all_charts[chart.id] = dict(chart) + self.all_charts[chart["id"]] = dict(chart) def get_dashboards_list(self) -> Optional[List[object]]: """ @@ -81,7 +77,7 @@ class SupersetDBSource(SupersetSourceMixin): name=dashboard_details["id"], displayName=dashboard_details["dashboard_title"], description="", - dashboardUrl=f"/superset/dashboard/{dashboard_details['id']}", + dashboardUrl=f"/superset/dashboard/{dashboard_details['id']}/", owner=self.get_owner_details(dashboard_details), charts=[ EntityReference(id=chart.id.__root__, type="chart") @@ -92,46 +88,12 @@ class SupersetDBSource(SupersetSourceMixin): ), ) - def yield_dashboard_lineage_details( - self, dashboard_details: dict, db_service_name: str - ) -> Optional[Iterable[AddLineageRequest]]: - """ - Get lineage between dashboard and data sources - """ - for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if chart_json: - datasource_fqn = ( - self._get_datasource_fqn(chart_json, db_service_name) - if chart_json.get("table_name") - else None - ) - if not datasource_fqn: - continue - from_entity = self.metadata.get_by_name( - entity=Table, - fqn=datasource_fqn, - ) - try: - dashboard_fqn = fqn.build( - self.metadata, - entity_type=Lineage_Dashboard, - service_name=self.config.serviceName, - dashboard_name=str(dashboard_details["id"]), - ) - to_entity = self.metadata.get_by_name( - entity=Lineage_Dashboard, - fqn=dashboard_fqn, - ) - if from_entity and to_entity: - yield self._get_add_lineage_request( - to_entity=to_entity, from_entity=from_entity - ) - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.error( - f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}" - ) + def _get_datasource_fqn_for_lineage(self, chart_json, db_service_name): + return ( + self._get_datasource_fqn(chart_json, db_service_name) + if chart_json.get("table_name") + else None + ) def yield_dashboard_chart( self, dashboard_details: dict @@ -180,7 +142,7 @@ class SupersetDBSource(SupersetSourceMixin): service_name=db_service_name, ) return dataset_fqn - except KeyError as err: + except Exception as err: logger.debug(traceback.format_exc()) logger.warning( f"Failed to fetch Datasource with id [{chart_json.get('table_name')}]: {err}" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index ede1abb95bb..e9c626d01d9 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -12,8 +12,14 @@ Superset mixin module """ import json -from typing import List +import traceback +from typing import Iterable, List, Optional +from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.entity.data.dashboard import ( + Dashboard as Lineage_Dashboard, +) +from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.services.connections.dashboard.supersetConnection import ( SupersetConnection, ) @@ -29,6 +35,10 @@ from metadata.generated.schema.metadataIngestion.workflow import ( from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.source import InvalidSourceException, SourceStatus from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource +from metadata.utils import fqn +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() class SupersetSourceMixin(DashboardServiceSource): @@ -102,3 +112,42 @@ class SupersetSourceMixin(DashboardServiceSource): if key.startswith("CHART-") and value.get("meta", {}).get("chartId") ] return [] + + def yield_dashboard_lineage_details( + self, dashboard_details: dict, db_service_name: str + ) -> Optional[Iterable[AddLineageRequest]]: + """ + Get lineage between dashboard and data sources + """ + for chart_id in self._get_charts_of_dashboard(dashboard_details): + chart_json = self.all_charts.get(chart_id) + if chart_json: + datasource_fqn = self._get_datasource_fqn_for_lineage( + chart_json, db_service_name + ) + if not datasource_fqn: + continue + from_entity = self.metadata.get_by_name( + entity=Table, + fqn=datasource_fqn, + ) + try: + dashboard_fqn = fqn.build( + self.metadata, + entity_type=Lineage_Dashboard, + service_name=self.config.serviceName, + dashboard_name=str(dashboard_details["id"]), + ) + to_entity = self.metadata.get_by_name( + entity=Lineage_Dashboard, + fqn=dashboard_fqn, + ) + if from_entity and to_entity: + yield self._get_add_lineage_request( + to_entity=to_entity, from_entity=from_entity + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error( + f"Error to yield dashboard lineage details for DB service name [{db_service_name}]: {exc}" + ) diff --git a/ingestion/tests/unit/resources/datasets/superset_dataset.json b/ingestion/tests/unit/resources/datasets/superset_dataset.json new file mode 100644 index 00000000000..00ef8236d35 --- /dev/null +++ b/ingestion/tests/unit/resources/datasets/superset_dataset.json @@ -0,0 +1,637 @@ +{ + "dashboard": { + "count": 3, + "description_columns": {}, + "ids": [ + 14 + ], + "label_columns": { + "certification_details": "Certification Details", + "certified_by": "Certified By", + "changed_by.first_name": "Changed By First Name", + "changed_by.id": "Changed By Id", + "changed_by.last_name": "Changed By Last Name", + "changed_by.username": "Changed By Username", + "changed_by_name": "Changed By Name", + "changed_by_url": "Changed By Url", + "changed_on_delta_humanized": "Changed On Delta Humanized", + "changed_on_utc": "Changed On Utc", + "created_by.first_name": "Created By First Name", + "created_by.id": "Created By Id", + "created_by.last_name": "Created By Last Name", + "created_on_delta_humanized": "Created On Delta Humanized", + "css": "Css", + "dashboard_title": "Dashboard Title", + "id": "Id", + "is_managed_externally": "Is Managed Externally", + "json_metadata": "Json Metadata", + "owners.email": "Owners Email", + "owners.first_name": "Owners First Name", + "owners.id": "Owners Id", + "owners.last_name": "Owners Last Name", + "owners.username": "Owners Username", + "position_json": "Position Json", + "published": "Published", + "roles.id": "Roles Id", + "roles.name": "Roles Name", + "slug": "Slug", + "status": "Status", + "thumbnail_url": "Thumbnail Url", + "url": "Url" + }, + "list_columns": [ + "id", + "published", + "status", + "slug", + "url", + "css", + "position_json", + "json_metadata", + "thumbnail_url", + "certified_by", + "certification_details", + "changed_by.first_name", + "changed_by.last_name", + "changed_by.username", + "changed_by.id", + "changed_by_name", + "changed_by_url", + "changed_on_utc", + "changed_on_delta_humanized", + "created_on_delta_humanized", + "created_by.first_name", + "created_by.id", + "created_by.last_name", + "dashboard_title", + "owners.id", + "owners.username", + "owners.first_name", + "owners.last_name", + "owners.email", + "roles.id", + "roles.name", + "is_managed_externally" + ], + "list_title": "List Dashboard", + "order_columns": [ + "changed_by.first_name", + "changed_on_delta_humanized", + "created_by.first_name", + "dashboard_title", + "published", + "changed_on" + ], + "result": [ + { + "certification_details": "", + "certified_by": "", + "changed_by": { + "first_name": "Superset", + "id": 1, + "last_name": "Admin", + "username": "admin" + }, + "changed_by_name": "Superset Admin", + "changed_by_url": "/superset/profile/admin", + "changed_on_delta_humanized": "2 days ago", + "changed_on_utc": "2023-01-30T10:53:25.572977+0000", + "created_by": { + "first_name": "Superset", + "id": 1, + "last_name": "Admin" + }, + "created_on_delta_humanized": "2 days ago", + "css": "", + "dashboard_title": "My DASH", + "id": 14, + "is_managed_externally": false, + "json_metadata": "{\"show_native_filters\": true, \"color_scheme\": \"\", \"refresh_frequency\": 0, \"shared_label_colors\": {}, \"color_scheme_domain\": [], \"expanded_slices\": {}, \"label_colors\": {}, \"timed_refresh_immune_slices\": [], \"default_filters\": \"{}\", \"chart_configuration\": {}}", + "owners": [ + { + "email": "admin@openmetadata.org", + "first_name": "Superset", + "id": 1, + "last_name": "Admin", + "username": "admin" + } + ], + "position_json": "{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "published": true, + "roles": [], + "slug": null, + "status": "published", + "thumbnail_url": "/api/v1/dashboard/14/thumbnail/0088b55d1a7c34b5aa121f11252c11d5/", + "url": "/superset/dashboard/14/" + } + ] + }, + "chart": { + "count": 106, + "description_columns": {}, + "ids": [ + 37 + ], + "label_columns": { + "cache_timeout": "Cache Timeout", + "certification_details": "Certification Details", + "certified_by": "Certified By", + "changed_by.first_name": "Changed By First Name", + "changed_by.last_name": "Changed By Last Name", + "changed_by_name": "Changed By Name", + "changed_by_url": "Changed By Url", + "changed_on_delta_humanized": "Changed On Delta Humanized", + "changed_on_utc": "Changed On Utc", + "created_by.first_name": "Created By First Name", + "created_by.id": "Created By Id", + "created_by.last_name": "Created By Last Name", + "created_on_delta_humanized": "Created On Delta Humanized", + "dashboards.dashboard_title": "Dashboards Dashboard Title", + "dashboards.id": "Dashboards Id", + "datasource_id": "Datasource Id", + "datasource_name_text": "Datasource Name Text", + "datasource_type": "Datasource Type", + "datasource_url": "Datasource Url", + "description": "Description", + "description_markeddown": "Description Markeddown", + "edit_url": "Edit Url", + "id": "Id", + "is_managed_externally": "Is Managed Externally", + "last_saved_at": "Last Saved At", + "last_saved_by.first_name": "Last Saved By First Name", + "last_saved_by.id": "Last Saved By Id", + "last_saved_by.last_name": "Last Saved By Last Name", + "owners.first_name": "Owners First Name", + "owners.id": "Owners Id", + "owners.last_name": "Owners Last Name", + "owners.username": "Owners Username", + "params": "Params", + "slice_name": "Slice Name", + "table.default_endpoint": "Table Default Endpoint", + "table.table_name": "Table Table Name", + "thumbnail_url": "Thumbnail Url", + "url": "Url", + "viz_type": "Viz Type" + }, + "list_columns": [ + "is_managed_externally", + "certified_by", + "certification_details", + "cache_timeout", + "changed_by.first_name", + "changed_by.last_name", + "changed_by_name", + "changed_by_url", + "changed_on_delta_humanized", + "changed_on_utc", + "created_by.first_name", + "created_by.id", + "created_by.last_name", + "created_on_delta_humanized", + "datasource_id", + "datasource_name_text", + "datasource_type", + "datasource_url", + "description", + "description_markeddown", + "edit_url", + "id", + "last_saved_at", + "last_saved_by.id", + "last_saved_by.first_name", + "last_saved_by.last_name", + "owners.first_name", + "owners.id", + "owners.last_name", + "owners.username", + "dashboards.id", + "dashboards.dashboard_title", + "params", + "slice_name", + "table.default_endpoint", + "table.table_name", + "thumbnail_url", + "url", + "viz_type" + ], + "list_title": "List Slice", + "order_columns": [ + "changed_by.first_name", + "changed_on_delta_humanized", + "datasource_id", + "datasource_name", + "last_saved_at", + "last_saved_by.id", + "last_saved_by.first_name", + "last_saved_by.last_name", + "slice_name", + "viz_type" + ], + "result": [ + { + "cache_timeout": null, + "certification_details": null, + "certified_by": null, + "changed_by": { + "first_name": "Superset", + "last_name": "Admin" + }, + "changed_by_name": "Superset Admin", + "changed_by_url": "/superset/profile/admin", + "changed_on_delta_humanized": "3 days ago", + "changed_on_utc": "2023-01-30T09:27:04.180422+0000", + "created_by": null, + "created_on_delta_humanized": "2 months ago", + "dashboards": [ + { + "dashboard_title": "World Bank's Data", + "id": 1 + }, + { + "dashboard_title": "My DASH", + "id": 14 + } + ], + "datasource_id": 1, + "datasource_name_text": "main.wb_health_population", + "datasource_type": "table", + "datasource_url": "/explore/?dataset_type=table&dataset_id=1", + "description": "TEST DESCRIPTION", + "description_markeddown": "

TEST DESCRIPTION

", + "edit_url": "/chart/edit/37", + "id": 37, + "is_managed_externally": false, + "last_saved_at": "2023-01-30T09:27:04.176240", + "last_saved_by": { + "first_name": "Superset", + "id": 1, + "last_name": "Admin" + }, + "owners": [ + { + "first_name": "Superset", + "id": 1, + "last_name": "Admin", + "username": "admin" + } + ], + "params": "{\n \"compare_lag\": \"10\",\n \"compare_suffix\": \"o10Y\",\n \"country_fieldtype\": \"cca3\",\n \"entity\": \"country_code\",\n \"granularity_sqla\": \"year\",\n \"groupby\": [],\n \"limit\": \"25\",\n \"markup_type\": \"markdown\",\n \"metric\": \"sum__SP_RUR_TOTL_ZS\",\n \"num_period_compare\": \"10\",\n \"row_limit\": 50000,\n \"secondary_metric\": {\n \"aggregate\": \"SUM\",\n \"column\": {\n \"column_name\": \"SP_RUR_TOTL\",\n \"optionName\": \"_col_SP_RUR_TOTL\",\n \"type\": \"DOUBLE\"\n },\n \"expressionType\": \"SIMPLE\",\n \"hasCustomLabel\": true,\n \"label\": \"Rural Population\"\n },\n \"show_bubbles\": true,\n \"since\": \"2014-01-01\",\n \"time_range\": \"2014-01-01 : 2014-01-02\",\n \"until\": \"2014-01-02\",\n \"viz_type\": \"world_map\"\n}", + "slice_name": "% Rural", + "table": { + "default_endpoint": null, + "table_name": "wb_health_population" + }, + "thumbnail_url": "/api/v1/chart/37/thumbnail/7b795de85c7dd32e3cfcd3fdb228e17e/", + "url": "/explore/?slice_id=37", + "viz_type": "world_map" + } + ] + }, + "datasource": { + "description_columns": {}, + "id": 1, + "label_columns": { + "cache_timeout": "Cache Timeout", + "changed_by.first_name": "Changed By First Name", + "changed_by.last_name": "Changed By Last Name", + "changed_on": "Changed On", + "changed_on_humanized": "Changed On Humanized", + "columns.advanced_data_type": "Columns Advanced Data Type", + "columns.changed_on": "Columns Changed On", + "columns.column_name": "Columns Column Name", + "columns.created_on": "Columns Created On", + "columns.description": "Columns Description", + "columns.expression": "Columns Expression", + "columns.extra": "Columns Extra", + "columns.filterable": "Columns Filterable", + "columns.groupby": "Columns Groupby", + "columns.id": "Columns Id", + "columns.is_active": "Columns Is Active", + "columns.is_dttm": "Columns Is Dttm", + "columns.python_date_format": "Columns Python Date Format", + "columns.type": "Columns Type", + "columns.type_generic": "Columns Type Generic", + "columns.uuid": "Columns Uuid", + "columns.verbose_name": "Columns Verbose Name", + "created_by.first_name": "Created By First Name", + "created_by.last_name": "Created By Last Name", + "created_on": "Created On", + "created_on_humanized": "Created On Humanized", + "database.backend": "Database Backend", + "database.database_name": "Database Database Name", + "database.id": "Database Id", + "datasource_type": "Datasource Type", + "default_endpoint": "Default Endpoint", + "description": "Description", + "extra": "Extra", + "fetch_values_predicate": "Fetch Values Predicate", + "filter_select_enabled": "Filter Select Enabled", + "id": "Id", + "is_managed_externally": "Is Managed Externally", + "is_sqllab_view": "Is Sqllab View", + "kind": "Kind", + "main_dttm_col": "Main Dttm Col", + "metrics": "Metrics", + "metrics.changed_on": "Metrics Changed On", + "metrics.created_on": "Metrics Created On", + "metrics.d3format": "Metrics D3Format", + "metrics.description": "Metrics Description", + "metrics.expression": "Metrics Expression", + "metrics.extra": "Metrics Extra", + "metrics.id": "Metrics Id", + "metrics.metric_name": "Metrics Metric Name", + "metrics.metric_type": "Metrics Metric Type", + "metrics.verbose_name": "Metrics Verbose Name", + "metrics.warning_text": "Metrics Warning Text", + "offset": "Offset", + "owners.first_name": "Owners First Name", + "owners.id": "Owners Id", + "owners.last_name": "Owners Last Name", + "owners.username": "Owners Username", + "schema": "Schema", + "select_star": "Select Star", + "sql": "Sql", + "table_name": "Table Name", + "template_params": "Template Params", + "url": "Url" + }, + "result": { + "cache_timeout": null, + "changed_by": null, + "changed_on": "2022-11-14T12:57:13.515690", + "changed_on_humanized": "2 months ago", + "created_by": null, + "created_on": "2022-11-14T12:56:43.115906", + "created_on_humanized": "2 months ago", + "database": { + "backend": "sqlite", + "database_name": "examples", + "id": 1 + }, + "datasource_type": "table", + "default_endpoint": null, + "description": "\nThis data was downloaded from the\n[World's Health Organization's website](https://datacatalog.worldbank.org/dataset/health-nutrition-and-population-statistics)\n\nHere's the script that was used to massage the data:\n\n DIR = \"\"\n df_country = pd.read_csv(DIR + '/HNP_Country.csv')\n df_country.columns = ['country_code'] + list(df_country.columns[1:])\n df_country = df_country[['country_code', 'Region']]\n df_country.columns = ['country_code', 'region']\n\n df = pd.read_csv(DIR + '/HNP_Data.csv')\n del df['Unnamed: 60']\n df.columns = ['country_name', 'country_code'] + list(df.columns[2:])\n ndf = df.merge(df_country, how='inner')\n\n dims = ('country_name', 'country_code', 'region')\n vv = [str(i) for i in range(1960, 2015)]\n mdf = pd.melt(ndf, id_vars=dims + ('Indicator Code',), value_vars=vv)\n mdf['year'] = mdf.variable + '-01-01'\n dims = dims + ('year',)\n\n pdf = mdf.pivot_table(values='value', columns='Indicator Code', index=dims)\n pdf = pdf.reset_index()\n pdf.to_csv(DIR + '/countries.csv')\n pdf.to_json(DIR + '/countries.json', orient='records')\n\nHere's the description of the metrics available:\n\nSeries | Code Indicator Name\n--- | ---\nNY.GNP.PCAP.CD | GNI per capita, Atlas method (current US$)\nSE.ADT.1524.LT.FM.ZS | Literacy rate, youth (ages 15-24), gender parity index (GPI)\nSE.ADT.1524.LT.MA.ZS | Literacy rate, youth male (% of males ages 15-24)\nSE.ADT.1524.LT.ZS | Literacy rate, youth total (% of people ages 15-24)\nSE.ADT.LITR.FE.ZS | Literacy rate, adult female (% of females ages 15 and above)\nSE.ADT.LITR.MA.ZS | Literacy rate, adult male (% of males ages 15 and above)\nSE.ADT.LITR.ZS | Literacy rate, adult total (% of people ages 15 and above)\nSE.ENR.ORPH | Ratio of school attendance of orphans to school attendance of non-orphans ages 10-14\nSE.PRM.CMPT.FE.ZS | Primary completion rate, female (% of relevant age group)\nSE.PRM.CMPT.MA.ZS | Primary completion rate, male (% of relevant age group)\nSE.PRM.CMPT.ZS | Primary completion rate, total (% of relevant age group)\nSE.PRM.ENRR | School enrollment, primary (% gross)\nSE.PRM.ENRR.FE | School enrollment, primary, female (% gross)\nSE.PRM.ENRR.MA | School enrollment, primary, male (% gross)\nSE.PRM.NENR | School enrollment, primary (% net)\nSE.PRM.NENR.FE | School enrollment, primary, female (% net)\nSE.PRM.NENR.MA | School enrollment, primary, male (% net)\nSE.SEC.ENRR | School enrollment, secondary (% gross)\nSE.SEC.ENRR.FE | School enrollment, secondary, female (% gross)\nSE.SEC.ENRR.MA | School enrollment, secondary, male (% gross)\nSE.SEC.NENR | School enrollment, secondary (% net)\nSE.SEC.NENR.FE | School enrollment, secondary, female (% net)\nSE.SEC.NENR.MA | School enrollment, secondary, male (% net)\nSE.TER.ENRR | School enrollment, tertiary (% gross)\nSE.TER.ENRR.FE | School enrollment, tertiary, female (% gross)\nSE.XPD.TOTL.GD.ZS | Government expenditure on education, total (% of GDP)\nSH.ANM.CHLD.ZS | Prevalence of anemia among children (% of children under 5)\nSH.ANM.NPRG.ZS | Prevalence of anemia among non-pregnant women (% of women ages 15-49)\nSH.CON.1524.FE.ZS | Condom use, population ages 15-24, female (% of females ages 15-24)\nSH.CON.1524.MA.ZS | Condom use, population ages 15-24, male (% of males ages 15-24)\nSH.CON.AIDS.FE.ZS | Condom use at last high-risk sex, adult female (% ages 15-49)\nSH.CON.AIDS.MA.ZS | Condom use at last high-risk sex, adult male (% ages 15-49)\nSH.DTH.COMM.ZS | Cause of death, by communicable diseases and maternal, prenatal and nutrition conditions (% of total)\nSH.DTH.IMRT | Number of infant deaths\nSH.DTH.INJR.ZS | Cause of death, by injury (% of total)\nSH.DTH.MORT | Number of under-five deaths\nSH.DTH.NCOM.ZS | Cause of death, by non-communicable diseases (% of total)\nSH.DTH.NMRT | Number of neonatal deaths\nSH.DYN.AIDS | Adults (ages 15+) living with HIV\nSH.DYN.AIDS.DH | AIDS estimated deaths (UNAIDS estimates)\nSH.DYN.AIDS.FE.ZS | Women's share of population ages 15+ living with HIV (%)\nSH.DYN.AIDS.ZS | Prevalence of HIV, total (% of population ages 15-49)\nSH.DYN.MORT | Mortality rate, under-5 (per 1,000 live births)\nSH.DYN.MORT.FE | Mortality rate, under-5, female (per 1,000 live births)\nSH.DYN.MORT.MA | Mortality rate, under-5, male (per 1,000 live births)\nSH.DYN.NMRT | Mortality rate, neonatal (per 1,000 live births)\nSH.FPL.SATI.ZS | Met need for contraception (% of married women ages 15-49)\nSH.H2O.SAFE.RU.ZS | Improved water source, rural (% of rural population with access)\nSH.H2O.SAFE.UR.ZS | Improved water source, urban (% of urban population with access)\nSH.H2O.SAFE.ZS | Improved water source (% of population with access)\nSH.HIV.0014 | Children (0-14) living with HIV\nSH.HIV.1524.FE.ZS | Prevalence of HIV, female (% ages 15-24)\nSH.HIV.1524.KW.FE.ZS | Comprehensive correct knowledge of HIV/AIDS, ages 15-24, female (2 prevent ways and reject 3 misconceptions)\nSH.HIV.1524.KW.MA.ZS | Comprehensive correct knowledge of HIV/AIDS, ages 15-24, male (2 prevent ways and reject 3 misconceptions)\nSH.HIV.1524.MA.ZS | Prevalence of HIV, male (% ages 15-24)\nSH.HIV.ARTC.ZS | Antiretroviral therapy coverage (% of people living with HIV)\nSH.HIV.KNOW.FE.ZS | % of females ages 15-49 having comprehensive correct knowledge about HIV (2 prevent ways and reject 3 misconceptions)\nSH.HIV.KNOW.MA.ZS | % of males ages 15-49 having comprehensive correct knowledge about HIV (2 prevent ways and reject 3 misconceptions)\nSH.HIV.ORPH | Children orphaned by HIV/AIDS\nSH.HIV.TOTL | Adults (ages 15+) and children (0-14 years) living with HIV\nSH.IMM.HEPB | Immunization, HepB3 (% of one-year-old children)\nSH.IMM.HIB3 | Immunization, Hib3 (% of children ages 12-23 months)\nSH.IMM.IBCG | Immunization, BCG (% of one-year-old children)\nSH.IMM.IDPT | Immunization, DPT (% of children ages 12-23 months)\nSH.IMM.MEAS | Immunization, measles (% of children ages 12-23 months)\nSH.IMM.POL3 | Immunization, Pol3 (% of one-year-old children)\nSH.MED.BEDS.ZS | Hospital beds (per 1,000 people)\nSH.MED.CMHW.P3 | Community health workers (per 1,000 people)\nSH.MED.NUMW.P3 | Nurses and midwives (per 1,000 people)\nSH.MED.PHYS.ZS | Physicians (per 1,000 people)\nSH.MLR.NETS.ZS | Use of insecticide-treated bed nets (% of under-5 population)\nSH.MLR.PREG.ZS | Use of any antimalarial drug (% of pregnant women)\nSH.MLR.SPF2.ZS | Use of Intermittent Preventive Treatment of malaria, 2+ doses of SP/Fansidar (% of pregnant women)\nSH.MLR.TRET.ZS | Children with fever receiving antimalarial drugs (% of children under age 5 with fever)\nSH.MMR.DTHS | Number of maternal deaths\nSH.MMR.LEVE | Number of weeks of maternity leave\nSH.MMR.RISK | Lifetime risk of maternal death (1 in: rate varies by country)\nSH.MMR.RISK.ZS | Lifetime risk of maternal death (%)\nSH.MMR.WAGE.ZS | Maternal leave benefits (% of wages paid in covered period)\nSH.PRG.ANEM | Prevalence of anemia among pregnant women (%)\nSH.PRG.ARTC.ZS | Antiretroviral therapy coverage (% of pregnant women living with HIV)\nSH.PRG.SYPH.ZS | Prevalence of syphilis (% of women attending antenatal care)\nSH.PRV.SMOK.FE | Smoking prevalence, females (% of adults)\nSH.PRV.SMOK.MA | Smoking prevalence, males (% of adults)\nSH.STA.ACSN | Improved sanitation facilities (% of population with access)\nSH.STA.ACSN.RU | Improved sanitation facilities, rural (% of rural population with access)\nSH.STA.ACSN.UR | Improved sanitation facilities, urban (% of urban population with access)\nSH.STA.ANV4.ZS | Pregnant women receiving prenatal care of at least four visits (% of pregnant women)\nSH.STA.ANVC.ZS | Pregnant women receiving prenatal care (%)\nSH.STA.ARIC.ZS | ARI treatment (% of children under 5 taken to a health provider)\nSH.STA.BFED.ZS | Exclusive breastfeeding (% of children under 6 months)\nSH.STA.BRTC.ZS | Births attended by skilled health staff (% of total)\nSH.STA.BRTW.ZS | Low-birthweight babies (% of births)\nSH.STA.DIAB.ZS | Diabetes prevalence (% of population ages 20 to 79)\nSH.STA.IYCF.ZS | Infant and young child feeding practices, all 3 IYCF (% children ages 6-23 months)\nSH.STA.MALN.FE.ZS | Prevalence of underweight, weight for age, female (% of children under 5)\nSH.STA.MALN.MA.ZS | Prevalence of underweight, weight for age, male (% of children under 5)\nSH.STA.MALN.ZS | Prevalence of underweight, weight for age (% of children under 5)\nSH.STA.MALR | Malaria cases reported\nSH.STA.MMRT | Maternal mortality ratio (modeled estimate, per 100,000 live births)\nSH.STA.MMRT.NE | Maternal mortality ratio (national estimate, per 100,000 live births)\nSH.STA.ORCF.ZS | Diarrhea treatment (% of children under 5 receiving oral rehydration and continued feeding)\nSH.STA.ORTH | Diarrhea treatment (% of children under 5 who received ORS packet)\nSH.STA.OW15.FE.ZS | Prevalence of overweight, female (% of female adults)\nSH.STA.OW15.MA.ZS | Prevalence of overweight, male (% of male adults)\nSH.STA.OW15.ZS | Prevalence of overweight (% of adults)\nSH.STA.OWGH.FE.ZS | Prevalence of overweight, weight for height, female (% of children under 5)\nSH.STA.OWGH.MA.ZS | Prevalence of overweight, weight for height, male (% of children under 5)\nSH.STA.OWGH.ZS | Prevalence of overweight, weight for height (% of children under 5)\nSH.STA.PNVC.ZS | Postnatal care coverage (% mothers)\nSH.STA.STNT.FE.ZS | Prevalence of stunting, height for age, female (% of children under 5)\nSH.STA.STNT.MA.ZS | Prevalence of stunting, height for age, male (% of children under 5)\nSH.STA.STNT.ZS | Prevalence of stunting, height for age (% of children under 5)\nSH.STA.WAST.FE.ZS | Prevalence of wasting, weight for height, female (% of children under 5)\nSH.STA.WAST.MA.ZS | Prevalence of wasting, weight for height, male (% of children under 5)\nSH.STA.WAST.ZS | Prevalence of wasting, weight for height (% of children under 5)\nSH.SVR.WAST.FE.ZS | Prevalence of severe wasting, weight for height, female (% of children under 5)\nSH.SVR.WAST.MA.ZS | Prevalence of severe wasting, weight for height, male (% of children under 5)\nSH.SVR.WAST.ZS | Prevalence of severe wasting, weight for height (% of children under 5)\nSH.TBS.CURE.ZS | Tuberculosis treatment success rate (% of new cases)\nSH.TBS.DTEC.ZS | Tuberculosis case detection rate (%, all forms)\nSH.TBS.INCD | Incidence of tuberculosis (per 100,000 people)\nSH.TBS.MORT | Tuberculosis death rate (per 100,000 people)\nSH.TBS.PREV | Prevalence of tuberculosis (per 100,000 population)\nSH.VAC.TTNS.ZS | Newborns protected against tetanus (%)\nSH.XPD.EXTR.ZS | External resources for health (% of total expenditure on health)\nSH.XPD.OOPC.TO.ZS | Out-of-pocket health expenditure (% of total expenditure on health)\nSH.XPD.OOPC.ZS | Out-of-pocket health expenditure (% of private expenditure on health)\nSH.XPD.PCAP | Health expenditure per capita (current US$)\nSH.XPD.PCAP.PP.KD | Health expenditure per capita, PPP (constant 2011 international $)\nSH.XPD.PRIV | Health expenditure, private (% of total health expenditure)\nSH.XPD.PRIV.ZS | Health expenditure, private (% of GDP)\nSH.XPD.PUBL | Health expenditure, public (% of total health expenditure)\nSH.XPD.PUBL.GX.ZS | Health expenditure, public (% of government expenditure)\nSH.XPD.PUBL.ZS | Health expenditure, public (% of GDP)\nSH.XPD.TOTL.CD | Health expenditure, total (current US$)\nSH.XPD.TOTL.ZS | Health expenditure, total (% of GDP)\nSI.POV.NAHC | Poverty headcount ratio at national poverty lines (% of population)\nSI.POV.RUHC | Rural poverty headcount ratio at national poverty lines (% of rural population)\nSI.POV.URHC | Urban poverty headcount ratio at national poverty lines (% of urban population)\nSL.EMP.INSV.FE.ZS | Share of women in wage employment in the nonagricultural sector (% of total nonagricultural employment)\nSL.TLF.TOTL.FE.ZS | Labor force, female (% of total labor force)\nSL.TLF.TOTL.IN | Labor force, total\nSL.UEM.TOTL.FE.ZS | Unemployment, female (% of female labor force) (modeled ILO estimate)\nSL.UEM.TOTL.MA.ZS | Unemployment, male (% of male labor force) (modeled ILO estimate)\nSL.UEM.TOTL.ZS | Unemployment, total (% of total labor force) (modeled ILO estimate)\nSM.POP.NETM | Net migration\nSN.ITK.DEFC | Number of people who are undernourished\nSN.ITK.DEFC.ZS | Prevalence of undernourishment (% of population)\nSN.ITK.SALT.ZS | Consumption of iodized salt (% of households)\nSN.ITK.VITA.ZS | Vitamin A supplementation coverage rate (% of children ages 6-59 months)\nSP.ADO.TFRT | Adolescent fertility rate (births per 1,000 women ages 15-19)\nSP.DYN.AMRT.FE | Mortality rate, adult, female (per 1,000 female adults)\nSP.DYN.AMRT.MA | Mortality rate, adult, male (per 1,000 male adults)\nSP.DYN.CBRT.IN | Birth rate, crude (per 1,000 people)\nSP.DYN.CDRT.IN | Death rate, crude (per 1,000 people)\nSP.DYN.CONU.ZS | Contraceptive prevalence (% of women ages 15-49)\nSP.DYN.IMRT.FE.IN | Mortality rate, infant, female (per 1,000 live births)\nSP.DYN.IMRT.IN | Mortality rate, infant (per 1,000 live births)\nSP.DYN.IMRT.MA.IN | Mortality rate, infant, male (per 1,000 live births)\nSP.DYN.LE00.FE.IN | Life expectancy at birth, female (years)\nSP.DYN.LE00.IN | Life expectancy at birth, total (years)\nSP.DYN.LE00.MA.IN | Life expectancy at birth, male (years)\nSP.DYN.SMAM.FE | Mean age at first marriage, female\nSP.DYN.SMAM.MA | Mean age at first marriage, male\nSP.DYN.TFRT.IN | Fertility rate, total (births per woman)\nSP.DYN.TO65.FE.ZS | Survival to age 65, female (% of cohort)\nSP.DYN.TO65.MA.ZS | Survival to age 65, male (% of cohort)\nSP.DYN.WFRT | Wanted fertility rate (births per woman)\nSP.HOU.FEMA.ZS | Female headed households (% of households with a female head)\nSP.MTR.1519.ZS | Teenage mothers (% of women ages 15-19 who have had children or are currently pregnant)\nSP.POP.0004.FE | Population ages 0-4, female\nSP.POP.0004.FE.5Y | Population ages 0-4, female (% of female population)\nSP.POP.0004.MA | Population ages 0-4, male\nSP.POP.0004.MA.5Y | Population ages 0-4, male (% of male population)\nSP.POP.0014.FE.ZS | Population ages 0-14, female (% of total)\nSP.POP.0014.MA.ZS | Population ages 0-14, male (% of total)\nSP.POP.0014.TO | Population ages 0-14, total\nSP.POP.0014.TO.ZS | Population ages 0-14 (% of total)\nSP.POP.0509.FE | Population ages 5-9, female\nSP.POP.0509.FE.5Y | Population ages 5-9, female (% of female population)\nSP.POP.0509.MA | Population ages 5-9, male\nSP.POP.0509.MA.5Y | Population ages 5-9, male (% of male population)\nSP.POP.1014.FE | Population ages 10-14, female\nSP.POP.1014.FE.5Y | Population ages 10-14, female (% of female population)\nSP.POP.1014.MA | Population ages 10-14, male\nSP.POP.1014.MA.5Y | Population ages 10-14, male (% of male population)\nSP.POP.1519.FE | Population ages 15-19, female\nSP.POP.1519.FE.5Y | Population ages 15-19, female (% of female population)\nSP.POP.1519.MA | Population ages 15-19, male\nSP.POP.1519.MA.5Y | Population ages 15-19, male (% of male population)\nSP.POP.1564.FE.ZS | Population ages 15-64, female (% of total)\nSP.POP.1564.MA.ZS | Population ages 15-64, male (% of total)\nSP.POP.1564.TO | Population ages 15-64, total\nSP.POP.1564.TO.ZS | Population ages 15-64 (% of total)\nSP.POP.2024.FE | Population ages 20-24, female\nSP.POP.2024.FE.5Y | Population ages 20-24, female (% of female population)\nSP.POP.2024.MA | Population ages 20-24, male\nSP.POP.2024.MA.5Y | Population ages 20-24, male (% of male population)\nSP.POP.2529.FE | Population ages 25-29, female\nSP.POP.2529.FE.5Y | Population ages 25-29, female (% of female population)\nSP.POP.2529.MA | Population ages 25-29, male\nSP.POP.2529.MA.5Y | Population ages 25-29, male (% of male population)\nSP.POP.3034.FE | Population ages 30-34, female\nSP.POP.3034.FE.5Y | Population ages 30-34, female (% of female population)\nSP.POP.3034.MA | Population ages 30-34, male\nSP.POP.3034.MA.5Y | Population ages 30-34, male (% of male population)\nSP.POP.3539.FE | Population ages 35-39, female\nSP.POP.3539.FE.5Y | Population ages 35-39, female (% of female population)\nSP.POP.3539.MA | Population ages 35-39, male\nSP.POP.3539.MA.5Y | Population ages 35-39, male (% of male population)\nSP.POP.4044.FE | Population ages 40-44, female\nSP.POP.4044.FE.5Y | Population ages 40-44, female (% of female population)\nSP.POP.4044.MA | Population ages 40-44, male\nSP.POP.4044.MA.5Y | Population ages 40-44, male (% of male population)\nSP.POP.4549.FE | Population ages 45-49, female\nSP.POP.4549.FE.5Y | Population ages 45-49, female (% of female population)\nSP.POP.4549.MA | Population ages 45-49, male\nSP.POP.4549.MA.5Y | Population ages 45-49, male (% of male population)\nSP.POP.5054.FE | Population ages 50-54, female\nSP.POP.5054.FE.5Y | Population ages 50-54, female (% of female population)\nSP.POP.5054.MA | Population ages 50-54, male\nSP.POP.5054.MA.5Y | Population ages 50-54, male (% of male population)\nSP.POP.5559.FE | Population ages 55-59, female\nSP.POP.5559.FE.5Y | Population ages 55-59, female (% of female population)\nSP.POP.5559.MA | Population ages 55-59, male\nSP.POP.5559.MA.5Y | Population ages 55-59, male (% of male population)\nSP.POP.6064.FE | Population ages 60-64, female\nSP.POP.6064.FE.5Y | Population ages 60-64, female (% of female population)\nSP.POP.6064.MA | Population ages 60-64, male\nSP.POP.6064.MA.5Y | Population ages 60-64, male (% of male population)\nSP.POP.6569.FE | Population ages 65-69, female\nSP.POP.6569.FE.5Y | Population ages 65-69, female (% of female population)\nSP.POP.6569.MA | Population ages 65-69, male\nSP.POP.6569.MA.5Y | Population ages 65-69, male (% of male population)\nSP.POP.65UP.FE.ZS | Population ages 65 and above, female (% of total)\nSP.POP.65UP.MA.ZS | Population ages 65 and above, male (% of total)\nSP.POP.65UP.TO | Population ages 65 and above, total\nSP.POP.65UP.TO.ZS | Population ages 65 and above (% of total)\nSP.POP.7074.FE | Population ages 70-74, female\nSP.POP.7074.FE.5Y | Population ages 70-74, female (% of female population)\nSP.POP.7074.MA | Population ages 70-74, male\nSP.POP.7074.MA.5Y | Population ages 70-74, male (% of male population)\nSP.POP.7579.FE | Population ages 75-79, female\nSP.POP.7579.FE.5Y | Population ages 75-79, female (% of female population)\nSP.POP.7579.MA | Population ages 75-79, male\nSP.POP.7579.MA.5Y | Population ages 75-79, male (% of male population)\nSP.POP.80UP.FE | Population ages 80 and above, female\nSP.POP.80UP.FE.5Y | Population ages 80 and above, female (% of female population)\nSP.POP.80UP.MA | Population ages 80 and above, male\nSP.POP.80UP.MA.5Y | Population ages 80 and above, male (% of male population)\nSP.POP.AG00.FE.IN | Age population, age 0, female, interpolated\nSP.POP.AG00.MA.IN | Age population, age 0, male, interpolated\nSP.POP.AG01.FE.IN | Age population, age 01, female, interpolated\nSP.POP.AG01.MA.IN | Age population, age 01, male, interpolated\nSP.POP.AG02.FE.IN | Age population, age 02, female, interpolated\nSP.POP.AG02.MA.IN | Age population, age 02, male, interpolated\nSP.POP.AG03.FE.IN | Age population, age 03, female, interpolated\nSP.POP.AG03.MA.IN | Age population, age 03, male, interpolated\nSP.POP.AG04.FE.IN | Age population, age 04, female, interpolated\nSP.POP.AG04.MA.IN | Age population, age 04, male, interpolated\nSP.POP.AG05.FE.IN | Age population, age 05, female, interpolated\nSP.POP.AG05.MA.IN | Age population, age 05, male, interpolated\nSP.POP.AG06.FE.IN | Age population, age 06, female, interpolated\nSP.POP.AG06.MA.IN | Age population, age 06, male, interpolated\nSP.POP.AG07.FE.IN | Age population, age 07, female, interpolated\nSP.POP.AG07.MA.IN | Age population, age 07, male, interpolated\nSP.POP.AG08.FE.IN | Age population, age 08, female, interpolated\nSP.POP.AG08.MA.IN | Age population, age 08, male, interpolated\nSP.POP.AG09.FE.IN | Age population, age 09, female, interpolated\nSP.POP.AG09.MA.IN | Age population, age 09, male, interpolated\nSP.POP.AG10.FE.IN | Age population, age 10, female, interpolated\nSP.POP.AG10.MA.IN | Age population, age 10, male\nSP.POP.AG11.FE.IN | Age population, age 11, female, interpolated\nSP.POP.AG11.MA.IN | Age population, age 11, male\nSP.POP.AG12.FE.IN | Age population, age 12, female, interpolated\nSP.POP.AG12.MA.IN | Age population, age 12, male\nSP.POP.AG13.FE.IN | Age population, age 13, female, interpolated\nSP.POP.AG13.MA.IN | Age population, age 13, male\nSP.POP.AG14.FE.IN | Age population, age 14, female, interpolated\nSP.POP.AG14.MA.IN | Age population, age 14, male\nSP.POP.AG15.FE.IN | Age population, age 15, female, interpolated\nSP.POP.AG15.MA.IN | Age population, age 15, male, interpolated\nSP.POP.AG16.FE.IN | Age population, age 16, female, interpolated\nSP.POP.AG16.MA.IN | Age population, age 16, male, interpolated\nSP.POP.AG17.FE.IN | Age population, age 17, female, interpolated\nSP.POP.AG17.MA.IN | Age population, age 17, male, interpolated\nSP.POP.AG18.FE.IN | Age population, age 18, female, interpolated\nSP.POP.AG18.MA.IN | Age population, age 18, male, interpolated\nSP.POP.AG19.FE.IN | Age population, age 19, female, interpolated\nSP.POP.AG19.MA.IN | Age population, age 19, male, interpolated\nSP.POP.AG20.FE.IN | Age population, age 20, female, interpolated\nSP.POP.AG20.MA.IN | Age population, age 20, male, interpolated\nSP.POP.AG21.FE.IN | Age population, age 21, female, interpolated\nSP.POP.AG21.MA.IN | Age population, age 21, male, interpolated\nSP.POP.AG22.FE.IN | Age population, age 22, female, interpolated\nSP.POP.AG22.MA.IN | Age population, age 22, male, interpolated\nSP.POP.AG23.FE.IN | Age population, age 23, female, interpolated\nSP.POP.AG23.MA.IN | Age population, age 23, male, interpolated\nSP.POP.AG24.FE.IN | Age population, age 24, female, interpolated\nSP.POP.AG24.MA.IN | Age population, age 24, male, interpolated\nSP.POP.AG25.FE.IN | Age population, age 25, female, interpolated\nSP.POP.AG25.MA.IN | Age population, age 25, male, interpolated\nSP.POP.BRTH.MF | Sex ratio at birth (male births per female births)\nSP.POP.DPND | Age dependency ratio (% of working-age population)\nSP.POP.DPND.OL | Age dependency ratio, old (% of working-age population)\nSP.POP.DPND.YG | Age dependency ratio, young (% of working-age population)\nSP.POP.GROW | Population growth (annual %)\nSP.POP.TOTL | Population, total\nSP.POP.TOTL.FE.IN | Population, female\nSP.POP.TOTL.FE.ZS | Population, female (% of total)\nSP.POP.TOTL.MA.IN | Population, male\nSP.POP.TOTL.MA.ZS | Population, male (% of total)\nSP.REG.BRTH.RU.ZS | Completeness of birth registration, rural (%)\nSP.REG.BRTH.UR.ZS | Completeness of birth registration, urban (%)\nSP.REG.BRTH.ZS | Completeness of birth registration (%)\nSP.REG.DTHS.ZS | Completeness of death registration with cause-of-death information (%)\nSP.RUR.TOTL | Rural population\nSP.RUR.TOTL.ZG | Rural population growth (annual %)\nSP.RUR.TOTL.ZS | Rural population (% of total population)\nSP.URB.GROW | Urban population growth (annual %)\nSP.URB.TOTL | Urban population\nSP.URB.TOTL.IN.ZS | Urban population (% of total)\nSP.UWT.TFRT | Unmet need for contraception (% of married women ages 15-49)\n", + "extra": null, + "fetch_values_predicate": null, + "filter_select_enabled": true, + "id": 1, + "is_managed_externally": false, + "is_sqllab_view": false, + "kind": "physical", + "main_dttm_col": "year", + "metrics": [ + { + "changed_on": "2022-11-14T12:56:43.141358", + "created_on": "2022-11-14T12:56:43.141351", + "d3format": null, + "description": null, + "expression": "sum(\"SP_POP_TOTL\")", + "extra": null, + "id": 1, + "metric_name": "sum__SP_POP_TOTL", + "metric_type": null, + "verbose_name": null, + "warning_text": null + }, + { + "changed_on": "2022-11-14T12:56:43.141851", + "created_on": "2022-11-14T12:56:43.141846", + "d3format": null, + "description": null, + "expression": "sum(\"SH_DYN_AIDS\")", + "extra": null, + "id": 2, + "metric_name": "sum__SH_DYN_AIDS", + "metric_type": null, + "verbose_name": null, + "warning_text": null + }, + { + "changed_on": "2022-11-14T12:56:43.142026", + "created_on": "2022-11-14T12:56:43.142022", + "d3format": null, + "description": null, + "expression": "sum(\"SP_RUR_TOTL_ZS\")", + "extra": null, + "id": 3, + "metric_name": "sum__SP_RUR_TOTL_ZS", + "metric_type": null, + "verbose_name": null, + "warning_text": null + }, + { + "changed_on": "2022-11-14T12:56:43.142184", + "created_on": "2022-11-14T12:56:43.142180", + "d3format": null, + "description": null, + "expression": "sum(\"SP_DYN_LE00_IN\")", + "extra": null, + "id": 4, + "metric_name": "sum__SP_DYN_LE00_IN", + "metric_type": null, + "verbose_name": null, + "warning_text": null + }, + { + "changed_on": "2022-11-14T12:56:43.142334", + "created_on": "2022-11-14T12:56:43.142330", + "d3format": null, + "description": null, + "expression": "sum(\"SP_RUR_TOTL\")", + "extra": null, + "id": 5, + "metric_name": "sum__SP_RUR_TOTL", + "metric_type": null, + "verbose_name": null, + "warning_text": null + }, + { + "changed_on": "2022-11-14T12:56:43.672030", + "created_on": "2022-11-14T12:56:43.672018", + "d3format": null, + "description": null, + "expression": "COUNT(*)", + "extra": null, + "id": 6, + "metric_name": "count", + "metric_type": "count", + "verbose_name": "COUNT(*)", + "warning_text": null + } + ], + "offset": 0, + "owners": [], + "schema": "main", + "select_star": "SELECT *\nFROM main.wb_health_population\nLIMIT 100\nOFFSET 0", + "sql": null, + "table_name": "wb_health_population", + "template_params": null, + "url": "/tablemodelview/edit/1" + }, + "show_columns": [ + "id", + "database.database_name", + "database.id", + "table_name", + "sql", + "filter_select_enabled", + "fetch_values_predicate", + "schema", + "description", + "main_dttm_col", + "offset", + "default_endpoint", + "cache_timeout", + "is_sqllab_view", + "template_params", + "select_star", + "owners.id", + "owners.username", + "owners.first_name", + "owners.last_name", + "columns.advanced_data_type", + "columns.changed_on", + "columns.column_name", + "columns.created_on", + "columns.description", + "columns.expression", + "columns.filterable", + "columns.groupby", + "columns.id", + "columns.is_active", + "columns.extra", + "columns.is_dttm", + "columns.python_date_format", + "columns.type", + "columns.uuid", + "columns.verbose_name", + "metrics", + "metrics.changed_on", + "metrics.created_on", + "metrics.d3format", + "metrics.description", + "metrics.expression", + "metrics.extra", + "metrics.id", + "metrics.metric_name", + "metrics.metric_type", + "metrics.verbose_name", + "metrics.warning_text", + "datasource_type", + "url", + "extra", + "kind", + "created_on", + "created_on_humanized", + "created_by.first_name", + "created_by.last_name", + "changed_on", + "changed_on_humanized", + "changed_by.first_name", + "changed_by.last_name", + "columns.type_generic", + "database.backend", + "columns.advanced_data_type", + "is_managed_externally" + ], + "show_title": "Show Sqla Table" + }, + "database": { + "description_columns": {}, + "id": 1, + "label_columns": { + "allow_ctas": "Allow Ctas", + "allow_cvas": "Allow Cvas", + "allow_dml": "Allow Dml", + "allow_file_upload": "Allow File Upload", + "allow_run_async": "Allow Run Async", + "backend": "Backend", + "cache_timeout": "Cache Timeout", + "configuration_method": "Configuration Method", + "database_name": "Database Name", + "driver": "Driver", + "engine_information": "Engine Information", + "expose_in_sqllab": "Expose In Sqllab", + "extra": "Extra", + "force_ctas_schema": "Force Ctas Schema", + "id": "Id", + "impersonate_user": "Impersonate User", + "is_managed_externally": "Is Managed Externally", + "masked_encrypted_extra": "Masked Encrypted Extra", + "parameters": "Parameters", + "parameters_schema": "Parameters Schema", + "server_cert": "Server Cert", + "sqlalchemy_uri": "Sqlalchemy Uri" + }, + "result": { + "allow_ctas": false, + "allow_cvas": false, + "allow_dml": false, + "allow_file_upload": false, + "allow_run_async": false, + "backend": "sqlite", + "cache_timeout": null, + "configuration_method": "sqlalchemy_form", + "database_name": "examples", + "driver": "pysqlite", + "engine_information": { + "supports_file_upload": true + }, + "expose_in_sqllab": true, + "extra": "{\n \"metadata_params\": {},\n \"engine_params\": {},\n \"metadata_cache_timeout\": {},\n \"schemas_allowed_for_file_upload\": []\n}\n", + "force_ctas_schema": null, + "id": 1, + "impersonate_user": false, + "is_managed_externally": false, + "masked_encrypted_extra": null, + "parameters": { + "database": "examples" + }, + "parameters_schema": {}, + "server_cert": null, + "sqlalchemy_uri": "sqlite:////app/superset_home/superset.db" + }, + "show_columns": [ + "id", + "database_name", + "cache_timeout", + "expose_in_sqllab", + "allow_run_async", + "allow_file_upload", + "configuration_method", + "allow_ctas", + "allow_cvas", + "allow_dml", + "backend", + "driver", + "force_ctas_schema", + "impersonate_user", + "masked_encrypted_extra", + "extra", + "parameters", + "parameters_schema", + "server_cert", + "sqlalchemy_uri", + "is_managed_externally", + "engine_information" + ], + "show_title": "Show Database" + }, + "dashboard-db": { + "id": 14, + "dashboard_title": "My DASH", + "position_json":"{\"CHART-dwSXo_0t5X\":{\"children\":[],\"id\":\"CHART-dwSXo_0t5X\",\"meta\":{\"chartId\":37,\"height\":50,\"sliceName\":\"% Rural\",\"uuid\":\"8f663401-854a-4da7-8e50-4b8e4ebb4f22\",\"width\":4},\"parents\":[\"ROOT_ID\",\"GRID_ID\",\"ROW-z_7odBWenK\"],\"type\":\"CHART\"},\"DASHBOARD_VERSION_KEY\":\"v2\",\"GRID_ID\":{\"children\":[\"ROW-z_7odBWenK\"],\"id\":\"GRID_ID\",\"parents\":[\"ROOT_ID\"],\"type\":\"GRID\"},\"HEADER_ID\":{\"id\":\"HEADER_ID\",\"meta\":{\"text\":\"My DASH\"},\"type\":\"HEADER\"},\"ROOT_ID\":{\"children\":[\"GRID_ID\"],\"id\":\"ROOT_ID\",\"type\":\"ROOT\"},\"ROW-z_7odBWenK\":{\"children\":[\"CHART-dwSXo_0t5X\"],\"id\":\"ROW-z_7odBWenK\",\"meta\":{\"background\":\"BACKGROUND_TRANSPARENT\"},\"parents\":[\"ROOT_ID\",\"GRID_ID\"],\"type\":\"ROW\"}}", + "email" : "admin@openmetadata.org" + }, + "chart-db": [ + { + "id": 37, + "slice_name": "% Rural", + "description": "TEST DESCRIPTION", + "table_name": "wb_health_population", + "schema": "main", + "database_name": "test", + "sqlalchemy_uri": "user:pass@localhost:5432/examples" + }] + +} \ No newline at end of file diff --git a/ingestion/tests/unit/topology/dashboard/test_superset.py b/ingestion/tests/unit/topology/dashboard/test_superset.py new file mode 100644 index 00000000000..fc3817ff8f1 --- /dev/null +++ b/ingestion/tests/unit/topology/dashboard/test_superset.py @@ -0,0 +1,352 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Test superset source +""" + +import json +import uuid +from pathlib import Path +from unittest import TestCase +from unittest.mock import patch + +from sqlalchemy.engine import Engine + +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.entity.data.chart import Chart, ChartType +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.api.source import InvalidSourceException +from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource +from metadata.ingestion.source.dashboard.superset.api_source import SupersetAPISource +from metadata.ingestion.source.dashboard.superset.client import SupersetAPIClient +from metadata.ingestion.source.dashboard.superset.db_source import SupersetDBSource +from metadata.ingestion.source.dashboard.superset.metadata import SupersetSource + +mock_file_path = ( + Path(__file__).parent.parent.parent / "resources/datasets/superset_dataset.json" +) +with open(mock_file_path, encoding="UTF-8") as file: + mock_data: dict = json.load(file) + +MOCK_DASHBOARD_RESP = mock_data["dashboard"] +MOCK_DASHBOARD = MOCK_DASHBOARD_RESP["result"][0] +MOCK_CHART_RESP = mock_data["chart"] +MOCK_CHART = MOCK_CHART_RESP["result"][0] + +MOCK_CHART_DB = mock_data["chart-db"][0] +MOCK_DASHBOARD_DB = mock_data["dashboard-db"] + +MOCK_SUPERSET_API_CONFIG = { + "source": { + "type": "superset", + "serviceName": "test_supserset", + "serviceConnection": { + "config": { + "hostPort": "https://my-superset.com", + "type": "Superset", + "connection": { + "username": "admin", + "password": "admin", + "provider": "db", + }, + } + }, + "sourceConfig": { + "config": { + "type": "DashboardMetadata", + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "token"}, + }, + }, +} + + +MOCK_SUPERSET_DB_CONFIG = { + "source": { + "type": "superset", + "serviceName": "test_supserset", + "serviceConnection": { + "config": { + "hostPort": "https://my-superset.com", + "type": "Superset", + "connection": { + "type": "Postgres", + "username": "superset", + "password": "superset", + "hostPort": "localhost:5432", + "database": "superset", + }, + } + }, + "sourceConfig": { + "config": { + "type": "DashboardMetadata", + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": {"jwtToken": "token"}, + }, + }, +} + +EXPECTED_DASH_SERVICE = EntityReference(id=uuid.uuid4(), type="dashboardService") +EXPECTED_USER = EntityReference(id=uuid.uuid4(), type="user") + + +EXPECTED_CHATRT_ENTITY = [ + Chart( + id=uuid.uuid4(), + name=37, + service=EXPECTED_DASH_SERVICE, + ) +] + +EXPECTED_DASH = CreateDashboardRequest( + name=14, + displayName="My DASH", + description="", + dashboardUrl="/superset/dashboard/14/", + owner=EXPECTED_USER, + charts=[ + EntityReference(id=chart.id.__root__, type="chart") + for chart in EXPECTED_CHATRT_ENTITY + ], + service=EXPECTED_DASH_SERVICE, +) + +EXPECTED_CHART = CreateChartRequest( + name=37, + displayName="% Rural", + description="TEST DESCRIPTION", + chartType=ChartType.Other.value, + chartUrl="/explore/?slice_id=37", + service=EXPECTED_DASH_SERVICE, +) + +EXPECTED_ALL_CHARTS = {37: MOCK_CHART} +EXPECTED_ALL_CHARTS_DB = {37: MOCK_CHART_DB} + +NOT_FOUND_RESP = {"message": "Not found"} + +EXPECTED_DATASET_FQN = "demo.examples.main.wb_health_population" + + +class SupersetUnitTest(TestCase): + """ + Validate how we work with Superset metadata + """ + + def __init__(self, methodName) -> None: + super().__init__(methodName) + self.config = OpenMetadataWorkflowConfig.parse_obj(MOCK_SUPERSET_API_CONFIG) + + with patch.object( + DashboardServiceSource, "test_connection", return_value=False + ), patch.object(OMetaServerMixin, "validate_versions", return_value=True): + + # This already validates that the source can be initialized + self.superset_api: SupersetSource = SupersetSource.create( + MOCK_SUPERSET_API_CONFIG["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + + self.assertEqual(type(self.superset_api), SupersetAPISource) + + self.superset_api.context.__dict__[ + "dashboard_service" + ] = EXPECTED_DASH_SERVICE + + with patch.object( + SupersetAPIClient, "fetch_total_charts", return_value=1 + ), patch.object( + SupersetAPIClient, "fetch_charts", return_value=MOCK_CHART_RESP + ): + self.superset_api.prepare() + self.assertEqual(EXPECTED_ALL_CHARTS, self.superset_api.all_charts) + + with patch.object( + DashboardServiceSource, "test_connection", return_value=False + ), patch.object(OMetaServerMixin, "validate_versions", return_value=True): + + # This already validates that the source can be initialized + self.superset_db: SupersetSource = SupersetSource.create( + MOCK_SUPERSET_DB_CONFIG["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + + self.assertEqual(type(self.superset_db), SupersetDBSource) + + self.superset_db.context.__dict__[ + "dashboard_service" + ] = EXPECTED_DASH_SERVICE + + with patch.object(Engine, "execute", return_value=mock_data["chart-db"]): + self.superset_db.prepare() + self.assertEqual(EXPECTED_ALL_CHARTS_DB, self.superset_db.all_charts) + + def test_create(self): + """ + An invalid config raises an error + """ + not_superset_source = { + "type": "mysql", + "serviceName": "mysql_local", + "serviceConnection": { + "config": { + "type": "Mysql", + "username": "openmetadata_user", + "password": "openmetadata_password", + "hostPort": "localhost:3306", + "databaseSchema": "openmetadata_db", + } + }, + "sourceConfig": { + "config": { + "type": "DatabaseMetadata", + } + }, + } + + self.assertRaises( + InvalidSourceException, + SupersetSource.create, + not_superset_source, + self.config.workflowConfig.openMetadataServerConfig, + ) + + def test_api_perpare(self): + + pass + + def test_api_get_dashboards_list(self): + """ + Mock the client and check that we get a list + """ + + with patch.object( + SupersetAPIClient, "fetch_total_dashboards", return_value=1 + ), patch.object( + SupersetAPIClient, "fetch_dashboards", return_value=MOCK_DASHBOARD_RESP + ): + dashboard_list = self.superset_api.get_dashboards_list() + self.assertEqual(list(dashboard_list), [MOCK_DASHBOARD]) + + def test_charts_of_dashboard(self): + """ + Mock the client and check that we get a list + """ + result = self.superset_api._get_charts_of_dashboard( # pylint: disable=protected-access + MOCK_DASHBOARD + ) + self.assertEqual(result, [37]) + + def test_dashboard_name(self): + dashboard_name = self.superset_api.get_dashboard_name(MOCK_DASHBOARD) + self.assertEqual(dashboard_name, MOCK_DASHBOARD["dashboard_title"]) + + def test_yield_dashboard(self): + # TEST API SOURCE + with patch.object( + SupersetAPISource, "_get_user_by_email", return_value=EXPECTED_USER + ): + self.superset_api.context.__dict__["charts"] = EXPECTED_CHATRT_ENTITY + dashboard = self.superset_api.yield_dashboard(MOCK_DASHBOARD) + self.assertEqual(list(dashboard), [EXPECTED_DASH]) + + # TEST DB SOURCE + with patch.object( + SupersetDBSource, "_get_user_by_email", return_value=EXPECTED_USER + ): + self.superset_db.context.__dict__["charts"] = EXPECTED_CHATRT_ENTITY + dashboard = self.superset_db.yield_dashboard(MOCK_DASHBOARD_DB) + self.assertEqual(list(dashboard), [EXPECTED_DASH]) + + def test_yield_dashboard_chart(self): + # TEST API SOURCE + dashboard_charts = self.superset_api.yield_dashboard_chart(MOCK_DASHBOARD) + self.assertEqual(list(dashboard_charts), [EXPECTED_CHART]) + + # TEST DB SOURCE + dashboard_charts = self.superset_db.yield_dashboard_chart(MOCK_DASHBOARD_DB) + self.assertEqual(list(dashboard_charts), [EXPECTED_CHART]) + + def test_api_get_datasource_fqn(self): + """ + Test generated datasource fqn for api source + """ + with patch.object( + OpenMetadata, "es_search_from_fqn", return_value=None + ), patch.object( + SupersetAPIClient, + "fetch_datasource", + return_value=mock_data.get("datasource"), + ), patch.object( + SupersetAPIClient, "fetch_database", return_value=mock_data.get("database") + ): + fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access + 1, "demo" + ) + self.assertEqual(fqn, EXPECTED_DATASET_FQN) + + with patch.object( + OpenMetadata, "es_search_from_fqn", return_value=None + ), patch.object( + SupersetAPIClient, + "fetch_datasource", + return_value=mock_data.get("datasource"), + ), patch.object( + SupersetAPIClient, "fetch_database", return_value=NOT_FOUND_RESP + ): + fqn = self.superset_api._get_datasource_fqn( # pylint: disable=protected-access + 1, "demo" + ) + self.assertEqual(fqn, None) + + def test_db_get_datasource_fqn_for_lineage(self): + fqn = self.superset_db._get_datasource_fqn_for_lineage( # pylint: disable=protected-access + MOCK_CHART_DB, "demo" + ) + self.assertEqual(fqn, EXPECTED_DATASET_FQN) + + def test_db_get_database_name(self): + sqa_str1 = "postgres://user:pass@localhost:8888/database" + self.assertEqual( + self.superset_db._get_database_name( # pylint: disable=protected-access + sqa_str1 + ), + "database", + ) + + sqa_str2 = "sqlite:////app/superset_home/superset.db" + self.assertEqual( + self.superset_db._get_database_name( # pylint: disable=protected-access + sqa_str2 + ), + "superset.db", + )