# Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Dict, Set import os from setuptools import setup, find_namespace_packages def get_version(): root = os.path.dirname(__file__) changelog = os.path.join(root, "CHANGELOG") with open(changelog) as f: return f.readline().strip() def get_long_description(): root = os.path.dirname(__file__) with open(os.path.join(root, "README.md")) as f: description = f.read() description += "\n\nChangelog\n=========\n\n" with open(os.path.join(root, "CHANGELOG")) as f: description += f.read() return description base_requirements = { "commonregex", "click<7.2.0,>=7.1.1", "expandvars>=0.6.5" "dataclasses>=0.8" "typing_extensions>=3.7.4" "mypy_extensions>=0.4.3", "typing-inspect", "pydantic~=1.7.4", "pydantic[email]>=1.7.2", "google>=3.0.0", "google-auth>=1.33.0", "python-dateutil>=2.8.1", "email-validator>=1.0.3", "wheel~=0.36.2", "python-jose==3.3.0", "okta==1.7.0", "pandas~=1.3.1", "sqlalchemy>=1.3.24", "sql-metadata~=2.0.0", "spacy==3.0.5", "requests~=2.25.1" } base_plugins = { "pii-tags", "query-parser", "metadata-usage", "file-stage", "sql-metadata~=2.0.0", } plugins: Dict[str, Set[str]] = { "athena": {"PyAthena[SQLAlchemy]"}, "bigquery": {"pybigquery >= 0.6.0"}, "bigquery-usage": {"google-cloud-logging", "cachetools"}, "elasticsearch": {"elasticsearch~=7.13.1"}, "hive": {"pyhive~=0.6.3", "thrift~=0.13.0", "sasl==0.3.1", "thrift-sasl==0.4.3"}, "ldap-users": {"ldap3==2.9.1"}, "mssql": {"sqlalchemy-pytds>=0.3"}, "mssql-odbc": {"pyodbc"}, "mysql": {"pymysql>=1.0.2"}, "oracle": {"cx_Oracle"}, "postgres": {"pymysql>=1.0.2", "psycopg2-binary", "GeoAlchemy2"}, "redshift": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"}, "redshift-usage": {"sqlalchemy-redshift", "psycopg2-binary", "GeoAlchemy2"}, "snowflake": {"snowflake-sqlalchemy<=1.2.4"}, "snowflake-usage": {"snowflake-sqlalchemy<=1.2.4"}, "sample-tables": {"faker~=8.1.1", } } build_options = {"includes": ["_cffi_backend"]} setup( name="openmetadata-ingestion", version="0.2.1", url="https://open-metadata.org/", author="OpenMetadata Committers", license="Apache License 2.0", description="Ingestion Framework for OpenMetadata", long_description=get_long_description(), long_description_content_type="text/markdown", python_requires=">=3.8", options={"build_exe": build_options}, package_dir={"": "src"}, zip_safe=False, dependency_links=[ "apns@git+git://github.com/djacobs/PyAPNs.git#egg=apns", "simplescheduler@git+https://github.com/StreamlineData/sdscheduler.git#egg=simplescheduler" ], project_urls={ "Documentation": "https://docs.open-metadata.org/", "Source": "https://github.com/open-metadata/OpenMetadata", }, packages=find_namespace_packages(where='./src', exclude=['tests*']), entry_points={ "console_scripts": ["metadata = metadata.cmd:metadata"], "metadata.ingestion.source.plugins": [ "mysql = metadata.ingestion.source.mysql:MySQLSource", "postgres = metadata.ingestion.source.postgres:PostgresSource", "snowflake = metadata.ingestion.source.snowflake:SnowflakeSource", "redshift = metadata.ingestion.source.redshift:RedshiftSource", "redshift-sql = metadata.ingestion.source.redshift_sql:RedshiftSQLSource", "bigquery = metadata.ingestion.source.bigquery:BigQuerySource", "athena = metadata.ingestion.source.athena:AthenaSource", "oracle = metadata.ingestion.source.oracle:OracleSource", "mssql = metadata.ingestion.source.mssql:SQLServerSource", "hive = metadata.ingestion.source.hive:HiveSource", "sample-tables = metadata.ingestion.source.sample_data_generator:SampleTableSource", "sample-users = metadata.ingestion.source.sample_data_generator:SampleUserSource", "metadata-rest-tables = metadata.ingestion.source.metadata_rest:MetadataTablesRestSource", "redshift-usage = metadata.ingestion.source.redshift_usage:RedshiftUsageSource", "snowflake-usage = metadata.ingestion.source.snowflake_usage:SnowflakeUsageSource", "ldap-users = metadata.ingestion.source.ldap_source:LDAPUserSource" ], "metadata.ingestion.sink.plugins": [ "file = metadata.ingestion.sink.file:FileSink", "console = metadata.ingestion.sink.console:ConsoleSink", "metadata-rest-tables = metadata.ingestion.sink.metadata_tables_rest:MetadataTablesRestSink", "metadata-rest-users = metadata.ingestion.sink.metadata_users_rest:MetadataUsersRestSink", "ldap-rest-users = metadata.ingestion.sink.ldap_add_user:LdapUserRestSink" ], "metadata.ingestion.processor.plugins": [ "pii-tags = metadata.ingestion.processor.pii_processor:PIIProcessor", "query-parser = metadata.ingestion.processor.query_parser:QueryParserProcessor", ], "metadata.ingestion.stage.plugins": [ "file-stage = metadata.ingestion.stage.file:FileStage", "table-usage-stage = metadata.ingestion.stage.table_usage_stage:TableUsageStage" ], "metadata.ingestion.bulksink.plugins": [ "elasticsearch = metadata.ingestion.bulksink.elastic_search:ElasticSearchBulkSink", "metadata-usage = metadata.ingestion.bulksink.metadata_usage_rest:MetadataUsageBulkSink", ], }, install_requires=list(base_requirements), extras_require={ "base": list(base_requirements), **{ plugin: list(dependencies) for (plugin, dependencies) in plugins.items() }, "all": list(base_requirements.union( *[ requirements for plugin, requirements in plugins.items() ] ) ) } )