yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #120 from open-metadata/pypi-publish

Browse Source 
Pypi publish
This commit is contained in:
Ayush Shah 2021-08-13 02:38:34 +05:30 committed by GitHub
commit c721a280a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 28 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestion/Dockerfile
View File

@ -2,7 +2,10 @@ FROM python:3.9.2
EXPOSE 7777
COPY . /openmetadata-ingestion
COPY ./examples /openmetadata-ingestion/examples
COPY ./pipelines /openmetadata-ingestion/pipelines
COPY ./ingestion_scheduler /openmetadata-ingestion/ingestion_scheduler
COPY ./ingestion_dependency.sh /openmetadata-ingestion/ingestion_dependency.sh
WORKDIR /openmetadata-ingestion

4
ingestion/README.md
View File

@ -12,14 +12,14 @@ OpenMetadata Ingesiton is a simple framework to build connectors and ingest meta
### Install From PyPI
```text
python3 -m pip install --upgrade pip wheel setuptools
python3 -m pip install --upgrade openmetadata-ingestion
python3 -m pip install --upgrade pip wheel setuptools openmetadata-ingestion
python3 -m spacy download en_core_web_sm
```
### Install Ingestion Connector Dependencies
Click here to go to [Ingestion Connector's Documentation](https://docs.open-metadata.org/install/metadata-ingestion)
#### Generate Redshift Data
```text

0
ingestion/pipelines/bigquery.json → ingestion/examples/bigquery.json
View File

0
ingestion/pipelines/creds/bigquery-cred.json → ingestion/examples/creds/bigquery-cred.json
View File

0
ingestion/pipelines/hive.json → ingestion/examples/hive.json
View File

0
ingestion/pipelines/ldap_user_to_catalog.json → ingestion/examples/ldap_user_to_catalog.json
View File

0
ingestion/pipelines/mssql.json → ingestion/examples/mssql.json
View File

0
ingestion/pipelines/postgres.json → ingestion/examples/postgres.json
View File

0
ingestion/pipelines/redshift.json → ingestion/examples/redshift.json
View File

0
ingestion/pipelines/redshift_usage.json → ingestion/examples/redshift_usage.json
View File

6
ingestion/pipelines/snowflake.json → ingestion/examples/snowflake.json
View File

@ -11,9 +11,9 @@
"service_type": "Snowflake",
"include_pattern": {
"includes": [
"(\\w)*.tpcds_sf100tcl.catalog_page",
"(\\w)*.tpcds_sf100tcl.time_dim",
"(\\w)*.tpcds_sf10tcl.catalog_page"
"(\\w)*tpcds_sf100tcl",
"(\\w)*tpcds_sf100tcl",
"(\\w)*tpcds_sf10tcl"
]
}
}

0
ingestion/pipelines/snowflake_usage.json → ingestion/examples/snowflake_usage.json
View File

6
ingestion/ingestion_dependency.sh
View File

@ -17,5 +17,7 @@
#
set -euo pipefail
pip install --upgrade pip setuptools
pip install '.[all]'
pip install --upgrade pip setuptools openmetadata-ingestion==0.2.1 apns
pip install openmetadata-ingestion[mysql,sample-tables,elasticsearch]
python -m spacy download en_core_web_sm
pip install "simplescheduler@git+https://github.com/StreamlineData/sdscheduler.git#egg=simplescheduler"

12
ingestion/pipelines/creds/custom-name-320505-04c223f37dcf.json
View File

@ -1,12 +0,0 @@
{
"type": "service_account",
"project_id": "custom-name-320505",
"private_key_id": "04c223f37dcf9590a3033f22679e1a74c39776e5",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCYC15S0LekClU2\nuixtrvHQtKqnND3SbN5ffluU5JV+DV8XXiuNY/VSDGID7/OJ1FlcryA/leudFWRQ\njjqUa0F7incdoaYTMgPDtvvar7BCyqBUmOnElUODpsgbTx2KmSYCgnFfrex4iDp2\nWt+BcuLnJyS+ZCi4+OdxbVR8M2QK5ipNzgtMfXxTa+sh/pjPAJPI7/Ew1eVsr1Ya\nvzgqFkbMKZ87KkmqUQ10S0JX8UjYe7xB+JuaweccbdxqTYy8txIhAMBjLEfoieZL\nukaFnCslWsCRJeBQ9NyeLrWeKes4GnI/Xu4d1G5uoWUXeiyEXyYGyFxwXP1LJxAf\n2C9tDOYfAgMBAAECggEAC4o6r6zGxhfAraWGGhG//yK58cnjHVWvr8Nz6QJLybmd\nwN91YnG2nMfIKAYDeoUYU+gOLwqjy5Q5kmxRIZMIcSYqcfXNw0GLC++XVbBQVP7I\nb338VDQuCnp9+qX6bIVrifTjl6r9j+JrFDDugNvllpIf1abdeQ3vPGqGEeKGp9Wq\npUPe9+NGVZRX8H9Ff+txzB+tIh+KaZNPRnEOm3yaRVXSvw4SYAGEyj+hoM8YQ1ps\ncOZnFZAK5tOOlkyUwpqXybUEotPdFRoqqf/9SSRyXNENMs22vT3zvtiv9Xl4YJa0\ng1Uk9RZ2zKPurLDSydQ4gH+fXOqsvyOVNN2MDVFuIQKBgQDVfcuBxohPfv4Z+aGD\nebXV1Kb9abrWgfyBX9cWqKVA9GFmi653Yle8a8a2iV6DD0D9ukLSvDdBbwTWW8TZ\nZzGu6lIgzeygqfISHAU4A5EUMa+NU48e69TBfBcg/xbK0mDuNyLQ2OQgx1rn4LMn\n8ErO4RXu1AzpoOsDRtLkI71LIQKBgQC2UXaFUMOGWoo72s1dbVRUSFsbQy5Y+pTe\nTGRxuusJBRNq4Sy/ZdItuybBf11JiGprvTeKIldlxaTtBHa3/aRZkMHnyf4duIUo\npFRlx1+CuZxtvTHNiOA0AGcNzoNFEoaaJuiy65J6bGLsy1ep9M8lBnwugLdwDBsT\nuswReKtJPwKBgH9mezwtgoiWCwLrMhQoXHeHWEOfhayek1WNIOjXqQ4TB9DPCEUb\n/almU4X4gdsKDkmbjmb9hKfanXNh/h2SdFaiZHJJEK1r3RYrfH3Juxjm8ZEPzTex\nmhwAdjjtoOdWFhvx8SfaVoCtFrWLJL5vqeS88367Mh11NumUqSMVpM+hAoGAbHg1\nCoYNfg659Xhs/GvCY06VHbgvh/K1xTgv/lYV5czmS41igREEJf8VxMOQ7eHCZw0x\nUPazhrI7OGSSeU7ni/4Kz2umy2nvl0mFiPRAR9dEUSnhyoXn1ihzJ4y5Hswx1t7V\nTTXIdRutE3K7WjtzTnT4RvZ/rKMXLPrS1IY1aiMCgYEA0Tw5bLYHCh3xyJ+DyWgx\nUDbzYIebZ/OtlH8XndSOH3kdWSumNSLcasWYeGnHyPzdBoaN2PJgw+56Juz0fj98\nw0nsyn5b6rGHZNwmieypE5vSadImGySGDq05b0ZTwgaex52JZPOLgoHhnFdinxDQ\nUgbmMEhft+Tes2JtTxn+8Ak=\n-----END PRIVATE KEY-----\n",
"client_email": "ingestion-bot@custom-name-320505.iam.gserviceaccount.com",
"client_id": "106137785698555844583",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ingestion-bot%40custom-name-320505.iam.gserviceaccount.com"
}

34
ingestion/setup.py
View File

@ -37,9 +37,7 @@ def get_long_description():
base_requirements = {
"elasticsearch>=7.0.0,<8.0.0",
"commonregex",
"requests>=2.25.1",
"click<7.2.0,>=7.1.1",
"expandvars>=0.6.5"
"dataclasses>=0.8"
@ -55,9 +53,7 @@ base_requirements = {
"wheel~=0.36.2",
"python-jose==3.3.0",
"okta==1.7.0",
"pandas~=1.3.1"
}
connector_requirements = {
"pandas~=1.3.1",
"sqlalchemy>=1.3.24",
"sql-metadata~=2.0.0",
"spacy==3.0.5",
@ -71,28 +67,28 @@ base_plugins = {
"sql-metadata~=2.0.0",
}
plugins: Dict[str, Set[str]] = {
"athena": connector_requirements | {"PyAthena[SQLAlchemy]"},
"bigquery": connector_requirements | {"pybigquery >= 0.6.0"},
"athena": {"PyAthena[SQLAlchemy]"},
"bigquery": {"pybigquery >= 0.6.0"},
"bigquery-usage": {"google-cloud-logging", "cachetools"},
"elasticsearch": {"elasticsearch~=7.13.1"},
"hive": connector_requirements | {"pyhive~=0.6.3", "thrift~=0.13.0", "sasl==0.3.1", "thrift-sasl==0.4.3"},
"hive": {"pyhive~=0.6.3", "thrift~=0.13.0", "sasl==0.3.1", "thrift-sasl==0.4.3"},
"ldap-users": {"ldap3==2.9.1"},
"mssql": connector_requirements | {"sqlalchemy-pytds>=0.3"},
"mssql-odbc": connector_requirements | {"pyodbc"},
"mysql": connector_requirements | {"pymysql>=1.0.2"},
"oracle": connector_requirements | {"cx_Oracle"},
"postgres": connector_requirements | {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"},
"redshift": connector_requirements | {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
"redshift-usage": connector_requirements | {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
"snowflake": connector_requirements | {"snowflake-sqlalchemy<=1.2.4"},
"snowflake-usage": connector_requirements | {"snowflake-sqlalchemy<=1.2.4"},
"sample-tables": connector_requirements | {"faker~=8.1.1", }
"mssql": {"sqlalchemy-pytds>=0.3"},
"mssql-odbc": {"pyodbc"},
"mysql": {"pymysql>=1.0.2"},
"oracle": {"cx_Oracle"},
"postgres": {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"},
"redshift": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
"redshift-usage": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
"snowflake": {"snowflake-sqlalchemy<=1.2.4"},
"snowflake-usage": {"snowflake-sqlalchemy<=1.2.4"},
"sample-tables": {"faker~=8.1.1", }
}
build_options = {"includes": ["_cffi_backend"]}
setup(
name="openmetadata-ingestion",
version="0.2.0",
version="0.2.1",
url="https://open-metadata.org/",
author="OpenMetadata Committers",
license="Apache License 2.0",