mirror of
				https://github.com/datahub-project/datahub.git
				synced 2025-11-03 20:27:50 +00:00 
			
		
		
		
	fix(ingest): change redash sql parse error to warnining (#14785)
This commit is contained in:
		
							parent
							
								
									667b7cb12c
								
							
						
					
					
						commit
						002cc398d0
					
				@ -447,7 +447,7 @@ class RedashSource(StatefulIngestionSourceBase):
 | 
				
			|||||||
                dataset_urns = sql_parser_in_tables.in_tables
 | 
					                dataset_urns = sql_parser_in_tables.in_tables
 | 
				
			||||||
                if sql_parser_in_tables.debug_info.table_error:
 | 
					                if sql_parser_in_tables.debug_info.table_error:
 | 
				
			||||||
                    self.report.queries_problem_parsing.add(str(query_id))
 | 
					                    self.report.queries_problem_parsing.add(str(query_id))
 | 
				
			||||||
                    self.error(
 | 
					                    self.warn(
 | 
				
			||||||
                        logger,
 | 
					                        logger,
 | 
				
			||||||
                        "sql-parsing",
 | 
					                        "sql-parsing",
 | 
				
			||||||
                        f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",
 | 
					                        f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",
 | 
				
			||||||
 | 
				
			|||||||
@ -16,6 +16,7 @@ from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import (
 | 
				
			|||||||
    DashboardSnapshot,
 | 
					    DashboardSnapshot,
 | 
				
			||||||
)
 | 
					)
 | 
				
			||||||
from datahub.metadata.schema_classes import ChartInfoClass, DashboardInfoClass
 | 
					from datahub.metadata.schema_classes import ChartInfoClass, DashboardInfoClass
 | 
				
			||||||
 | 
					from datahub.sql_parsing.sqlglot_lineage import SqlParsingDebugInfo
 | 
				
			||||||
 | 
					
 | 
				
			||||||
mock_dashboard_response = {
 | 
					mock_dashboard_response = {
 | 
				
			||||||
    "tags": [],
 | 
					    "tags": [],
 | 
				
			||||||
@ -724,3 +725,30 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source):
 | 
				
			|||||||
    )
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    assert result == expected
 | 
					    assert result == expected
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					@patch("datahub.ingestion.source.redash.create_lineage_sql_parsed_result")
 | 
				
			||||||
 | 
					@patch("datahub.ingestion.source.redash.RedashSource._get_chart_data_source")
 | 
				
			||||||
 | 
					def test_sql_parsing_error_generates_warning(mocked_data_source, mocked_sql_parser):
 | 
				
			||||||
 | 
					    mocked_data_source.return_value = {
 | 
				
			||||||
 | 
					        **mock_mysql_data_source_response,
 | 
				
			||||||
 | 
					        "syntax": "sql",
 | 
				
			||||||
 | 
					    }
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    class MockResult:
 | 
				
			||||||
 | 
					        def __init__(self):
 | 
				
			||||||
 | 
					            self.in_tables = []
 | 
				
			||||||
 | 
					            self.debug_info = SqlParsingDebugInfo()
 | 
				
			||||||
 | 
					            self.debug_info.table_error = Exception("Invalid SQL syntax")
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    mocked_sql_parser.return_value = MockResult()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    source = redash_source_parse_table_names_from_sql()
 | 
				
			||||||
 | 
					    result = source._get_datasource_urns(
 | 
				
			||||||
 | 
					        mocked_data_source.return_value,
 | 
				
			||||||
 | 
					        {"id": 123, "query": "SELECT * FROM invalid_table;"},
 | 
				
			||||||
 | 
					    )
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					    assert result is None
 | 
				
			||||||
 | 
					    assert "123" in source.report.queries_problem_parsing
 | 
				
			||||||
 | 
					    assert len(source.report.warnings) > 0
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user