Merge pull request #120 from open-metadata/pypi-publish

Pypi publish

ingestion/Dockerfile

@@ -2,7 +2,10 @@ FROM python:3.9.2
 
 EXPOSE 7777
 
-COPY . /openmetadata-ingestion
+COPY ./examples /openmetadata-ingestion/examples
+COPY ./pipelines /openmetadata-ingestion/pipelines
+COPY ./ingestion_scheduler /openmetadata-ingestion/ingestion_scheduler
+COPY ./ingestion_dependency.sh /openmetadata-ingestion/ingestion_dependency.sh
 
 WORKDIR /openmetadata-ingestion
 

ingestion/README.md

@@ -12,14 +12,14 @@ OpenMetadata Ingesiton is a simple framework to build connectors and ingest meta
 ### Install From PyPI
 
 ```text
-python3 -m pip install --upgrade pip wheel setuptools
+python3 -m pip install --upgrade pip wheel setuptools openmetadata-ingestion
-python3 -m pip install --upgrade openmetadata-ingestion
 python3 -m spacy download en_core_web_sm
 ```
 
 ### Install Ingestion Connector Dependencies
 
 Click here to go to [Ingestion Connector's Documentation](https://docs.open-metadata.org/install/metadata-ingestion)
+
 #### Generate Redshift Data
 
 ```text

ingestion/examples/snowflake.json

@@ -11,9 +11,9 @@
       "service_type": "Snowflake",
       "include_pattern": {
         "includes": [
-          "(\\w)*.tpcds_sf100tcl.catalog_page",
+          "(\\w)*tpcds_sf100tcl",
-          "(\\w)*.tpcds_sf100tcl.time_dim",
+          "(\\w)*tpcds_sf100tcl",
-          "(\\w)*.tpcds_sf10tcl.catalog_page"
+          "(\\w)*tpcds_sf10tcl"
         ]
       }
     }

ingestion/ingestion_dependency.sh

@@ -17,5 +17,7 @@
 #
 
 set -euo pipefail
-pip install --upgrade pip setuptools
+pip install --upgrade pip setuptools openmetadata-ingestion==0.2.1 apns
-pip install '.[all]'
+pip install openmetadata-ingestion[mysql,sample-tables,elasticsearch]
+python -m spacy download en_core_web_sm
+pip install "simplescheduler@git+https://github.com/StreamlineData/sdscheduler.git#egg=simplescheduler"

ingestion/pipelines/creds/custom-name-320505-04c223f37dcf.json

@@ -1,12 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "custom-name-320505",
-  "private_key_id": "04c223f37dcf9590a3033f22679e1a74c39776e5",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCYC15S0LekClU2\nuixtrvHQtKqnND3SbN5ffluU5JV+DV8XXiuNY/VSDGID7/OJ1FlcryA/leudFWRQ\njjqUa0F7incdoaYTMgPDtvvar7BCyqBUmOnElUODpsgbTx2KmSYCgnFfrex4iDp2\nWt+BcuLnJyS+ZCi4+OdxbVR8M2QK5ipNzgtMfXxTa+sh/pjPAJPI7/Ew1eVsr1Ya\nvzgqFkbMKZ87KkmqUQ10S0JX8UjYe7xB+JuaweccbdxqTYy8txIhAMBjLEfoieZL\nukaFnCslWsCRJeBQ9NyeLrWeKes4GnI/Xu4d1G5uoWUXeiyEXyYGyFxwXP1LJxAf\n2C9tDOYfAgMBAAECggEAC4o6r6zGxhfAraWGGhG//yK58cnjHVWvr8Nz6QJLybmd\nwN91YnG2nMfIKAYDeoUYU+gOLwqjy5Q5kmxRIZMIcSYqcfXNw0GLC++XVbBQVP7I\nb338VDQuCnp9+qX6bIVrifTjl6r9j+JrFDDugNvllpIf1abdeQ3vPGqGEeKGp9Wq\npUPe9+NGVZRX8H9Ff+txzB+tIh+KaZNPRnEOm3yaRVXSvw4SYAGEyj+hoM8YQ1ps\ncOZnFZAK5tOOlkyUwpqXybUEotPdFRoqqf/9SSRyXNENMs22vT3zvtiv9Xl4YJa0\ng1Uk9RZ2zKPurLDSydQ4gH+fXOqsvyOVNN2MDVFuIQKBgQDVfcuBxohPfv4Z+aGD\nebXV1Kb9abrWgfyBX9cWqKVA9GFmi653Yle8a8a2iV6DD0D9ukLSvDdBbwTWW8TZ\nZzGu6lIgzeygqfISHAU4A5EUMa+NU48e69TBfBcg/xbK0mDuNyLQ2OQgx1rn4LMn\n8ErO4RXu1AzpoOsDRtLkI71LIQKBgQC2UXaFUMOGWoo72s1dbVRUSFsbQy5Y+pTe\nTGRxuusJBRNq4Sy/ZdItuybBf11JiGprvTeKIldlxaTtBHa3/aRZkMHnyf4duIUo\npFRlx1+CuZxtvTHNiOA0AGcNzoNFEoaaJuiy65J6bGLsy1ep9M8lBnwugLdwDBsT\nuswReKtJPwKBgH9mezwtgoiWCwLrMhQoXHeHWEOfhayek1WNIOjXqQ4TB9DPCEUb\n/almU4X4gdsKDkmbjmb9hKfanXNh/h2SdFaiZHJJEK1r3RYrfH3Juxjm8ZEPzTex\nmhwAdjjtoOdWFhvx8SfaVoCtFrWLJL5vqeS88367Mh11NumUqSMVpM+hAoGAbHg1\nCoYNfg659Xhs/GvCY06VHbgvh/K1xTgv/lYV5czmS41igREEJf8VxMOQ7eHCZw0x\nUPazhrI7OGSSeU7ni/4Kz2umy2nvl0mFiPRAR9dEUSnhyoXn1ihzJ4y5Hswx1t7V\nTTXIdRutE3K7WjtzTnT4RvZ/rKMXLPrS1IY1aiMCgYEA0Tw5bLYHCh3xyJ+DyWgx\nUDbzYIebZ/OtlH8XndSOH3kdWSumNSLcasWYeGnHyPzdBoaN2PJgw+56Juz0fj98\nw0nsyn5b6rGHZNwmieypE5vSadImGySGDq05b0ZTwgaex52JZPOLgoHhnFdinxDQ\nUgbmMEhft+Tes2JtTxn+8Ak=\n-----END PRIVATE KEY-----\n",
-  "client_email": "ingestion-bot@custom-name-320505.iam.gserviceaccount.com",
-  "client_id": "106137785698555844583",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/ingestion-bot%40custom-name-320505.iam.gserviceaccount.com"
-}

ingestion/setup.py

@@ -37,9 +37,7 @@ def get_long_description():
 
 
 base_requirements = {
-    "elasticsearch>=7.0.0,<8.0.0",
     "commonregex",
-    "requests>=2.25.1",
     "click<7.2.0,>=7.1.1",
     "expandvars>=0.6.5"
     "dataclasses>=0.8"
@@ -55,9 +53,7 @@ base_requirements = {
     "wheel~=0.36.2",
     "python-jose==3.3.0",
     "okta==1.7.0",
-    "pandas~=1.3.1"
+    "pandas~=1.3.1",
-}
-connector_requirements = {
     "sqlalchemy>=1.3.24",
     "sql-metadata~=2.0.0",
     "spacy==3.0.5",
@@ -71,28 +67,28 @@ base_plugins = {
     "sql-metadata~=2.0.0",
 }
 plugins: Dict[str, Set[str]] = {
-    "athena": connector_requirements | {"PyAthena[SQLAlchemy]"},
+    "athena": {"PyAthena[SQLAlchemy]"},
-    "bigquery": connector_requirements | {"pybigquery >= 0.6.0"},
+    "bigquery": {"pybigquery >= 0.6.0"},
     "bigquery-usage": {"google-cloud-logging", "cachetools"},
     "elasticsearch": {"elasticsearch~=7.13.1"},
-    "hive": connector_requirements | {"pyhive~=0.6.3", "thrift~=0.13.0", "sasl==0.3.1", "thrift-sasl==0.4.3"},
+    "hive": {"pyhive~=0.6.3", "thrift~=0.13.0", "sasl==0.3.1", "thrift-sasl==0.4.3"},
     "ldap-users": {"ldap3==2.9.1"},
-    "mssql": connector_requirements | {"sqlalchemy-pytds>=0.3"},
+    "mssql": {"sqlalchemy-pytds>=0.3"},
-    "mssql-odbc": connector_requirements | {"pyodbc"},
+    "mssql-odbc": {"pyodbc"},
-    "mysql": connector_requirements | {"pymysql>=1.0.2"},
+    "mysql": {"pymysql>=1.0.2"},
-    "oracle": connector_requirements | {"cx_Oracle"},
+    "oracle": {"cx_Oracle"},
-    "postgres": connector_requirements | {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"},
+    "postgres": {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"},
-    "redshift": connector_requirements | {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
+    "redshift": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
-    "redshift-usage": connector_requirements | {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
+    "redshift-usage": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"},
-    "snowflake": connector_requirements | {"snowflake-sqlalchemy<=1.2.4"},
+    "snowflake": {"snowflake-sqlalchemy<=1.2.4"},
-    "snowflake-usage": connector_requirements | {"snowflake-sqlalchemy<=1.2.4"},
+    "snowflake-usage": {"snowflake-sqlalchemy<=1.2.4"},
-    "sample-tables": connector_requirements | {"faker~=8.1.1", }
+    "sample-tables": {"faker~=8.1.1", }
 }
 
 build_options = {"includes": ["_cffi_backend"]}
 setup(
     name="openmetadata-ingestion",
-    version="0.2.0",
+    version="0.2.1",
     url="https://open-metadata.org/",
     author="OpenMetadata Committers",
     license="Apache License 2.0",