mirror of
https://github.com/datahub-project/datahub.git
synced 2025-12-12 10:35:51 +00:00
1 line
47 KiB
JavaScript
1 line
47 KiB
JavaScript
"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[90837],{31165:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>m,contentTitle:()=>d,default:()=>f,frontMatter:()=>p,metadata:()=>g,toc:()=>c});t(96540);var n=t(15680),s=t(53720),l=t(5400);function i(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function r(e,a){return a=null!=a?a:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):function(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))})),e}function o(e,a){if(null==e)return{};var t,n,s=function(e,a){if(null==e)return{};var t,n,s={},l=Object.keys(e);for(n=0;n<l.length;n++)t=l[n],a.indexOf(t)>=0||(s[t]=e[t]);return s}(e,a);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)t=l[n],a.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(s[t]=e[t])}return s}const p={sidebar_position:69,title:"SQL Queries",slug:"/generated/ingestion/sources/sql-queries",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/sql-queries.md"},d="SQL Queries",g={unversionedId:"docs/generated/ingestion/sources/sql-queries",id:"docs/generated/ingestion/sources/sql-queries",title:"SQL Queries",description:"Incubating",source:"@site/genDocs/docs/generated/ingestion/sources/sql-queries.md",sourceDirName:"docs/generated/ingestion/sources",slug:"/generated/ingestion/sources/sql-queries",permalink:"/docs/generated/ingestion/sources/sql-queries",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/sql-queries.md",tags:[],version:"current",sidebarPosition:69,frontMatter:{sidebar_position:69,title:"SQL Queries",slug:"/generated/ingestion/sources/sql-queries",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/sql-queries.md"},sidebar:"overviewSidebar",previous:{title:"Snowflake",permalink:"/docs/generated/ingestion/sources/snowflake"},next:{title:"SQLAlchemy",permalink:"/docs/generated/ingestion/sources/sqlalchemy"}},m={},c=[{value:"Important Capabilities",id:"important-capabilities",level:3},{value:"Query File Format",id:"query-file-format",level:3},{value:"Incremental Lineage",id:"incremental-lineage",level:3},{value:"Temporary Table Support",id:"temporary-table-support",level:3},{value:"Example Queries File",id:"example-queries-file",level:4},{value:"CLI based Ingestion",id:"cli-based-ingestion",level:3},{value:"Starter Recipe",id:"starter-recipe",level:3},{value:"Config Details",id:"config-details",level:3},{value:"Code Coordinates",id:"code-coordinates",level:3}],y={toc:c},u="wrapper";function f(e){var{components:a}=e,t=o(e,["components"]);return(0,n.yg)(u,r(function(e){for(var a=1;a<arguments.length;a++){var t=null!=arguments[a]?arguments[a]:{},n=Object.keys(t);"function"==typeof Object.getOwnPropertySymbols&&(n=n.concat(Object.getOwnPropertySymbols(t).filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable})))),n.forEach((function(a){i(e,a,t[a])}))}return e}({},y,t),{components:a,mdxType:"MDXLayout"}),(0,n.yg)("h1",{id:"sql-queries"},"SQL Queries"),(0,n.yg)("p",null,(0,n.yg)("img",{parentName:"p",src:"https://img.shields.io/badge/support%20status-incubating-blue",alt:"Incubating"})),(0,n.yg)("h3",{id:"important-capabilities"},"Important Capabilities"),(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:null},"Capability"),(0,n.yg)("th",{parentName:"tr",align:null},"Status"),(0,n.yg)("th",{parentName:"tr",align:null},"Notes"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Column-level Lineage"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Parsed from SQL queries.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Table-Level Lineage"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Parsed from SQL queries.")))),(0,n.yg)("p",null,"This source reads a newline-delimited JSON file containing SQL queries and parses them to generate lineage."),(0,n.yg)("h3",{id:"query-file-format"},"Query File Format"),(0,n.yg)("p",null,"This file should contain one JSON object per line, with the following fields:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"query: string - The SQL query to parse."),(0,n.yg)("li",{parentName:"ul"},"timestamp (optional): number - The timestamp of the query, in seconds since the epoch."),(0,n.yg)("li",{parentName:"ul"},"user (optional): string - The user who ran the query.\nThis user value will be directly converted into a DataHub user urn."),(0,n.yg)("li",{parentName:"ul"},"operation_type (optional): string - Platform-specific operation type, used if the operation type can't be parsed."),(0,n.yg)("li",{parentName:"ul"},"session_id (optional): string - Session identifier for temporary table resolution across queries."),(0,n.yg)("li",{parentName:"ul"},"downstream_tables (optional): string[] - Fallback list of tables that the query writes to,\nused if the query can't be parsed."),(0,n.yg)("li",{parentName:"ul"},"upstream_tables (optional): string[] - Fallback list of tables the query reads from,\nused if the query can't be parsed.")),(0,n.yg)("p",null,(0,n.yg)("strong",{parentName:"p"},"Lazy Schema Loading"),":"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Fetches schemas on-demand during query parsing instead of bulk loading all schemas upfront"),(0,n.yg)("li",{parentName:"ul"},"Caches fetched schemas for future lookups to avoid repeated network requests"),(0,n.yg)("li",{parentName:"ul"},"Reduces initial startup time and memory usage significantly"),(0,n.yg)("li",{parentName:"ul"},"Automatically handles large platforms efficiently without memory issues")),(0,n.yg)("p",null,(0,n.yg)("strong",{parentName:"p"},"Query Processing"),":"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Loads the entire query file into memory at once"),(0,n.yg)("li",{parentName:"ul"},"Processes all queries sequentially before generating metadata work units"),(0,n.yg)("li",{parentName:"ul"},"Preserves temp table mappings and lineage relationships to ensure consistent lineage tracking"),(0,n.yg)("li",{parentName:"ul"},"Query deduplication is handled automatically by the SQL parsing aggregator")),(0,n.yg)("h3",{id:"incremental-lineage"},"Incremental Lineage"),(0,n.yg)("p",null,"When ",(0,n.yg)("inlineCode",{parentName:"p"},"incremental_lineage")," is enabled, this source will emit lineage as patches rather than full overwrites.\nThis allows you to add lineage edges without removing existing ones, which is useful for:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Gradually building up lineage from multiple sources"),(0,n.yg)("li",{parentName:"ul"},"Preserving manually curated lineage"),(0,n.yg)("li",{parentName:"ul"},"Avoiding conflicts when multiple ingestion processes target the same datasets")),(0,n.yg)("p",null,"Note: Incremental lineage only applies to UpstreamLineage aspects. Other aspects like queries and usage\nstatistics will still be emitted normally."),(0,n.yg)("h3",{id:"temporary-table-support"},"Temporary Table Support"),(0,n.yg)("p",null,"For platforms like Athena that don't have native temporary tables, you can use the ",(0,n.yg)("inlineCode",{parentName:"p"},"temp_table_patterns"),"\nconfiguration to specify regex patterns that identify fake temporary tables. This allows the source to\nprocess these tables like other sources that support native temp tables, enabling proper lineage tracking\nacross temporary table operations."),(0,n.yg)("h4",{id:"example-queries-file"},"Example Queries File"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-json"},'{"query": "SELECT x FROM my_table", "timestamp": 1689232738.051, "user": "user_a", "downstream_tables": [], "upstream_tables": ["my_database.my_schema.my_table"]}\n{"query": "INSERT INTO my_table VALUES (1, \'a\')", "timestamp": 1689232737.669, "user": "user_b", "downstream_tables": ["my_database.my_schema.my_table"], "upstream_tables": []}\n')),(0,n.yg)("p",null,"Note that this file does not represent a single JSON object, but instead newline-delimited JSON, in which\neach line is a separate JSON object."),(0,n.yg)("h3",{id:"cli-based-ingestion"},"CLI based Ingestion"),(0,n.yg)("h3",{id:"starter-recipe"},"Starter Recipe"),(0,n.yg)("p",null,"Check out the following recipe to get started with ingestion! See ",(0,n.yg)("a",{parentName:"p",href:"#config-details"},"below")," for full configuration options."),(0,n.yg)("p",null,"For general pointers on writing and running a recipe, see our ",(0,n.yg)("a",{parentName:"p",href:"/docs/metadata-ingestion#recipes"},"main recipe guide"),"."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-yaml"},'datahub_api: # Only necessary if using a non-DataHub sink, e.g. the file sink\n server: http://localhost:8080\n timeout_sec: 60\nsource:\n type: sql-queries\n config:\n platform: "snowflake"\n default_db: "SNOWFLAKE"\n query_file: "./queries.json"\n\n')),(0,n.yg)("h3",{id:"config-details"},"Config Details"),(0,n.yg)(s.A,{mdxType:"Tabs"},(0,n.yg)(l.A,{value:"options",label:"Options",default:!0,mdxType:"TabItem"},(0,n.yg)("p",null,"Note that a ",(0,n.yg)("inlineCode",{parentName:"p"},".")," is used to denote nested fields in the YAML recipe."),(0,n.yg)("div",{className:"config-table"},(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:"left"},"Field"),(0,n.yg)("th",{parentName:"tr",align:"left"},"Description"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"platform"),"\xa0",(0,n.yg)("abbr",{title:"Required"},"\u2705"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The platform for which to generate data, e.g. snowflake")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"query_file"),"\xa0",(0,n.yg)("abbr",{title:"Required"},"\u2705"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Path to file to ingest")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"default_db"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The default database to use for unqualified table names ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"default_schema"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The default schema to use for unqualified table names ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"enable_lazy_schema_loading"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Enable lazy schema loading for better performance. When enabled, schemas are fetched on-demand instead of bulk loading all schemas upfront, reducing startup time and memory usage. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"incremental_lineage"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"override_dialect"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The SQL dialect to use when parsing queries. Overrides automatic dialect detection. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"platform_instance"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See ",(0,n.yg)("a",{parentName:"td",href:"https://docs.datahub.com/docs/platform-instances/"},"https://docs.datahub.com/docs/platform-instances/")," for more details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"env"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The environment that all assets produced by this connector belong to ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"PROD")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"aws_config"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of AwsConnectionConfig, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS configuration for S3 access. Required when query_file is an S3 URI (s3://). ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_access_key_id"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS access key ID. Can be auto-detected, see ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html"},"the AWS boto3 docs")," for details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_advanced_config"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"object"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Advanced AWS configuration options. These are passed directly to ",(0,n.yg)("a",{parentName:"td",href:"https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html"},"botocore.config.Config"),".")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_endpoint_url"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The AWS service endpoint. This is normally ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html"},"constructed automatically"),", but can be overridden here. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_profile"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The ",(0,n.yg)("a",{parentName:"td",href:"https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html"},"named profile")," to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_proxy"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"A set of proxy configs to use with AWS. See the ",(0,n.yg)("a",{parentName:"td",href:"https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html"},"botocore.config")," docs for details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_region"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS region code. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_retry_mode"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"Enum"))),(0,n.yg)("td",{parentName:"tr",align:"left"},'One of: "legacy", "standard", "adaptive" ',(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"standard")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_retry_num"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"integer"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Number of times to retry failed AWS requests. See the ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html"},"botocore.retry")," docs for details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"5")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_secret_access_key"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS secret access key. Can be auto-detected, see ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html"},"the AWS boto3 docs")," for details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_session_token"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS session token. Can be auto-detected, see ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html"},"the AWS boto3 docs")," for details. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"read_timeout"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"number"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The timeout for reading from the connection (in seconds). ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"60")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config."),(0,n.yg)("span",{className:"path-main"},"aws_role"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, array, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as ",(0,n.yg)("a",{parentName:"td",href:"https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role"},"boto3's STS.Client.assume_role"),". ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config.aws_role."),(0,n.yg)("span",{className:"path-main"},"union"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, AwsAssumeRoleConfig"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config.aws_role.union."),(0,n.yg)("span",{className:"path-main"},"RoleArn"),"\xa0",(0,n.yg)("abbr",{title:"Required if union is set"},"\u2753"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"ARN of the role to assume.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"aws_config.aws_role.union."),(0,n.yg)("span",{className:"path-main"},"ExternalId"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of string, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"External ID to use when assuming the role. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"temp_table_patterns"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"array"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to match the entire table name. This is useful for platforms like Athena that don't have native temp tables but use naming patterns for fake temp tables. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"[","]")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"temp_table_patterns."),(0,n.yg)("span",{className:"path-main"},"string"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"usage"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"BaseUsageConfig"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"bucket_duration"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"Enum"))),(0,n.yg)("td",{parentName:"tr",align:"left"},'One of: "DAY", "HOUR"')),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"end_time"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string(date-time)"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Latest date of lineage/usage to consider. Default: Current time in UTC")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"format_sql_queries"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to format sql queries ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"include_operational_stats"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to display operational stats. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"include_read_operational_stats"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to report read operational stats. Experimental. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"include_top_n_queries"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ingest the top_n_queries. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"start_time"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string(date-time)"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on ",(0,n.yg)("inlineCode",{parentName:"td"},"bucket_duration"),"). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"None")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"top_n_queries"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"integer"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Number of top queries to save to each table. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"10")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage."),(0,n.yg)("span",{className:"path-main"},"user_email_pattern"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"AllowDenyPattern"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"A class to store allow deny regexes")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"usage.user_email_pattern."),(0,n.yg)("span",{className:"path-main"},"ignoreCase"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of boolean, null"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ignore case sensitivity during pattern matching. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))))))),(0,n.yg)(l.A,{value:"schema",label:"Schema",mdxType:"TabItem"},(0,n.yg)("p",null,"The ",(0,n.yg)("a",{parentName:"p",href:"https://json-schema.org/"},"JSONSchema")," for this configuration is inlined below."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "$defs": {\n "AllowDenyPattern": {\n "additionalProperties": false,\n "description": "A class to store allow deny regexes",\n "properties": {\n "allow": {\n "default": [\n ".*"\n ],\n "description": "List of regex patterns to include in ingestion",\n "items": {\n "type": "string"\n },\n "title": "Allow",\n "type": "array"\n },\n "deny": {\n "default": [],\n "description": "List of regex patterns to exclude from ingestion.",\n "items": {\n "type": "string"\n },\n "title": "Deny",\n "type": "array"\n },\n "ignoreCase": {\n "anyOf": [\n {\n "type": "boolean"\n },\n {\n "type": "null"\n }\n ],\n "default": true,\n "description": "Whether to ignore case sensitivity during pattern matching.",\n "title": "Ignorecase"\n }\n },\n "title": "AllowDenyPattern",\n "type": "object"\n },\n "AwsAssumeRoleConfig": {\n "additionalProperties": true,\n "properties": {\n "RoleArn": {\n "description": "ARN of the role to assume.",\n "title": "Rolearn",\n "type": "string"\n },\n "ExternalId": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "External ID to use when assuming the role.",\n "title": "Externalid"\n }\n },\n "required": [\n "RoleArn"\n ],\n "title": "AwsAssumeRoleConfig",\n "type": "object"\n },\n "AwsConnectionConfig": {\n "additionalProperties": false,\n "description": "Common AWS credentials config.\\n\\nCurrently used by:\\n - Glue source\\n - SageMaker source\\n - dbt source",\n "properties": {\n "aws_access_key_id": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.",\n "title": "Aws Access Key Id"\n },\n "aws_secret_access_key": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.",\n "title": "Aws Secret Access Key"\n },\n "aws_session_token": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.",\n "title": "Aws Session Token"\n },\n "aws_role": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "items": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "$ref": "#/$defs/AwsAssumeRoleConfig"\n }\n ]\n },\n "type": "array"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3\'s STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).",\n "title": "Aws Role"\n },\n "aws_profile": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.",\n "title": "Aws Profile"\n },\n "aws_region": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS region code.",\n "title": "Aws Region"\n },\n "aws_endpoint_url": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.",\n "title": "Aws Endpoint Url"\n },\n "aws_proxy": {\n "anyOf": [\n {\n "additionalProperties": {\n "type": "string"\n },\n "type": "object"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.",\n "title": "Aws Proxy"\n },\n "aws_retry_num": {\n "default": 5,\n "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.",\n "title": "Aws Retry Num",\n "type": "integer"\n },\n "aws_retry_mode": {\n "default": "standard",\n "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.",\n "enum": [\n "legacy",\n "standard",\n "adaptive"\n ],\n "title": "Aws Retry Mode",\n "type": "string"\n },\n "read_timeout": {\n "default": 60,\n "description": "The timeout for reading from the connection (in seconds).",\n "title": "Read Timeout",\n "type": "number"\n },\n "aws_advanced_config": {\n "additionalProperties": true,\n "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).",\n "title": "Aws Advanced Config",\n "type": "object"\n }\n },\n "title": "AwsConnectionConfig",\n "type": "object"\n },\n "BaseUsageConfig": {\n "additionalProperties": false,\n "properties": {\n "bucket_duration": {\n "$ref": "#/$defs/BucketDuration",\n "default": "DAY",\n "description": "Size of the time window to aggregate usage stats."\n },\n "end_time": {\n "description": "Latest date of lineage/usage to consider. Default: Current time in UTC",\n "format": "date-time",\n "title": "End Time",\n "type": "string"\n },\n "start_time": {\n "default": null,\n "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as \'-7 days\' Or \'-7d\'.",\n "format": "date-time",\n "title": "Start Time",\n "type": "string"\n },\n "top_n_queries": {\n "default": 10,\n "description": "Number of top queries to save to each table.",\n "exclusiveMinimum": 0,\n "title": "Top N Queries",\n "type": "integer"\n },\n "user_email_pattern": {\n "$ref": "#/$defs/AllowDenyPattern",\n "default": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "description": "regex patterns for user emails to filter in usage."\n },\n "include_operational_stats": {\n "default": true,\n "description": "Whether to display operational stats.",\n "title": "Include Operational Stats",\n "type": "boolean"\n },\n "include_read_operational_stats": {\n "default": false,\n "description": "Whether to report read operational stats. Experimental.",\n "title": "Include Read Operational Stats",\n "type": "boolean"\n },\n "format_sql_queries": {\n "default": false,\n "description": "Whether to format sql queries",\n "title": "Format Sql Queries",\n "type": "boolean"\n },\n "include_top_n_queries": {\n "default": true,\n "description": "Whether to ingest the top_n_queries.",\n "title": "Include Top N Queries",\n "type": "boolean"\n }\n },\n "title": "BaseUsageConfig",\n "type": "object"\n },\n "BucketDuration": {\n "enum": [\n "DAY",\n "HOUR"\n ],\n "title": "BucketDuration",\n "type": "string"\n }\n },\n "additionalProperties": false,\n "properties": {\n "incremental_lineage": {\n "default": false,\n "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",\n "title": "Incremental Lineage",\n "type": "boolean"\n },\n "env": {\n "default": "PROD",\n "description": "The environment that all assets produced by this connector belong to",\n "title": "Env",\n "type": "string"\n },\n "platform_instance": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.",\n "title": "Platform Instance"\n },\n "query_file": {\n "description": "Path to file to ingest",\n "title": "Query File",\n "type": "string"\n },\n "platform": {\n "description": "The platform for which to generate data, e.g. snowflake",\n "title": "Platform",\n "type": "string"\n },\n "usage": {\n "$ref": "#/$defs/BaseUsageConfig",\n "default": {\n "bucket_duration": "DAY",\n "end_time": "2025-12-10T03:45:03.864729Z",\n "start_time": "2025-12-09T00:00:00Z",\n "queries_character_limit": 24000,\n "top_n_queries": 10,\n "user_email_pattern": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "include_operational_stats": true,\n "include_read_operational_stats": false,\n "format_sql_queries": false,\n "include_top_n_queries": true\n },\n "description": "The usage config to use when generating usage statistics"\n },\n "default_db": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The default database to use for unqualified table names",\n "title": "Default Db"\n },\n "default_schema": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The default schema to use for unqualified table names",\n "title": "Default Schema"\n },\n "override_dialect": {\n "anyOf": [\n {\n "type": "string"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "The SQL dialect to use when parsing queries. Overrides automatic dialect detection.",\n "title": "Override Dialect"\n },\n "temp_table_patterns": {\n "default": [],\n "description": "Regex patterns for temporary tables to filter in lineage ingestion. Specify regex to match the entire table name. This is useful for platforms like Athena that don\'t have native temp tables but use naming patterns for fake temp tables.",\n "items": {\n "type": "string"\n },\n "title": "Temp Table Patterns",\n "type": "array"\n },\n "enable_lazy_schema_loading": {\n "default": true,\n "description": "Enable lazy schema loading for better performance. When enabled, schemas are fetched on-demand instead of bulk loading all schemas upfront, reducing startup time and memory usage.",\n "title": "Enable Lazy Schema Loading",\n "type": "boolean"\n },\n "aws_config": {\n "anyOf": [\n {\n "$ref": "#/$defs/AwsConnectionConfig"\n },\n {\n "type": "null"\n }\n ],\n "default": null,\n "description": "AWS configuration for S3 access. Required when query_file is an S3 URI (s3://)."\n }\n },\n "required": [\n "query_file",\n "platform"\n ],\n "title": "SqlQueriesSourceConfig",\n "type": "object"\n}\n')))),(0,n.yg)("h3",{id:"code-coordinates"},"Code Coordinates"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Class Name: ",(0,n.yg)("inlineCode",{parentName:"li"},"datahub.ingestion.source.sql_queries.SqlQueriesSource")),(0,n.yg)("li",{parentName:"ul"},"Browse on ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py"},"GitHub"))),(0,n.yg)("h2",null,"Questions"),(0,n.yg)("p",null,"If you've got any questions on configuring ingestion for SQL Queries, feel free to ping us on ",(0,n.yg)("a",{parentName:"p",href:"https://datahub.com/slack"},"our Slack"),"."))}f.isMDXComponent=!0}}]); |