feat(ingest): define Redshift as a Postgres Source (#2540)

This commit is contained in:
Albert Franzi 2021-05-12 19:00:34 +02:00 committed by GitHub
parent 0cdadd810a
commit 7fce505ffb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 0 deletions

View File

@ -43,6 +43,7 @@ We use a plugin architecture so that you can install only the dependencies you a
| mysql | `pip install 'acryl-datahub[mysql]'` | MySQL source |
| oracle | `pip install 'acryl-datahub[oracle]'` | Oracle source |
| postgres | `pip install 'acryl-datahub[postgres]'` | Postgres source |
| redshift | `pip install 'acryl-datahub[redshift]'` | Redshift source |
| sqlalchemy | `pip install 'acryl-datahub[sqlalchemy]'` | Generic SQLAlchemy source |
| snowflake | `pip install 'acryl-datahub[snowflake]'` | Snowflake source |
| superset | `pip install 'acryl-datahub[superset]'` | Supserset source |
@ -279,6 +280,26 @@ source:
# options is same as above
```
### Redshift `redshift`
Extracts:
- List of databases, schema, and tables
- Column types associated with each table
- Also supports PostGIS extensions
```yml
source:
type: redshift
config:
username: user
password: pass
host_port: localhost:5432
database: DemoDatabase
# table_pattern/schema_pattern is same as above
# options is same as above
```
### Snowflake `snowflake`
Extracts:

View File

@ -74,6 +74,7 @@ plugins: Dict[str, Set[str]] = {
"mssql": sql_common | {"sqlalchemy-pytds>=0.3"},
"mysql": sql_common | {"pymysql>=1.0.2"},
"postgres": sql_common | {"psycopg2-binary", "GeoAlchemy2"},
"redshift": sql_common | {"psycopg2-binary", "GeoAlchemy2"},
"snowflake": sql_common | {"snowflake-sqlalchemy"},
"oracle": sql_common | {"cx_Oracle"},
"ldap": {"python-ldap>=2.4"},
@ -192,6 +193,7 @@ setuptools.setup(
"mysql = datahub.ingestion.source.mysql:MySQLSource",
"oracle = datahub.ingestion.source.oracle:OracleSource",
"postgres = datahub.ingestion.source.postgres:PostgresSource",
"redshift = datahub.ingestion.source.redshift:RedshiftSource",
"snowflake = datahub.ingestion.source.snowflake:SnowflakeSource",
"superset = datahub.ingestion.source.superset:SupersetSource",
],

View File

@ -0,0 +1,7 @@
from datahub.ingestion.source.postgres import PostgresSource
from datahub.ingestion.source.sql_common import SQLAlchemySource
class RedshiftSource(PostgresSource):
def __init__(self, config, ctx):
SQLAlchemySource.__init__(self, config, ctx, "redshift")