mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2026-01-05 20:17:07 +00:00
* test * Use pyproject.toml * Fix pylint * makefile * makefile * Fix pylint * isort * pyproject * Airflow apis pyproject * Remove ingestion core * isort * Fix makefile help
This commit is contained in:
Pere Miquel Brull
GitHub
parent
06d9ca3f49
commit
caaf0e7a1d
@ -34,6 +34,6 @@ jobs:
|
||||
run: |
|
||||
make install_dev install_apis
|
||||
cd openmetadata-airflow-apis; \
|
||||
python setup.py build sdist bdist_wheel; \
|
||||
python -m build; \
|
||||
twine check dist/*; \
|
||||
twine upload dist/* --verbose
|
||||
|
||||
49
.github/workflows/py-ingestion-core-publish.yml
vendored
49
.github/workflows/py-ingestion-core-publish.yml
vendored
@ -1,49 +0,0 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# When a new push happens to the main branch that
|
||||
# modifies the JSON Schemas, we are going to
|
||||
# generate the updated pydantic models and
|
||||
# publish a new version of openmetadata-core
|
||||
|
||||
name: Publish openmetadata-ingestion-core packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: py-ingestion-core-publish-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
py-core-build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install Ubuntu related dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
|
||||
- name: Install, Generate and Publish Test PyPi packages
|
||||
env:
|
||||
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME_TEST }}
|
||||
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
sudo make install_antlr_cli
|
||||
make install_dev generate install
|
||||
cd ingestion-core; \
|
||||
python setup.py build sdist bdist_wheel; \
|
||||
twine check dist/*; \
|
||||
twine upload dist/* --verbose
|
||||
2
.github/workflows/py-ingestion-publish.yml
vendored
2
.github/workflows/py-ingestion-publish.yml
vendored
@ -37,6 +37,6 @@ jobs:
|
||||
sudo make install_antlr_cli
|
||||
make install_dev generate install
|
||||
cd ingestion; \
|
||||
python setup.py build sdist bdist_wheel; \
|
||||
python -m build; \
|
||||
twine check dist/*; \
|
||||
twine upload dist/* --verbose
|
||||
|
||||
48
Makefile
48
Makefile
@ -4,7 +4,7 @@ include ingestion/Makefile
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[35m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":"}; {printf "\033[35m%-35s\033[0m %s\n", $$2, $$3}'
|
||||
|
||||
.PHONY: install_e2e_tests
|
||||
install_e2e_tests: ## Install the ingestion module with e2e test dependencies (playwright)
|
||||
@ -24,40 +24,6 @@ yarn_install_cache: ## Use Yarn to install UI dependencies
|
||||
yarn_start_dev_ui: ## Run the UI locally with Yarn
|
||||
cd openmetadata-ui/src/main/resources/ui && yarn start
|
||||
|
||||
## Ingestion Core
|
||||
.PHONY: core_install_dev
|
||||
core_install_dev: ## Prepare a venv for the ingestion-core module
|
||||
cd ingestion-core; \
|
||||
rm -rf venv; \
|
||||
python3 -m venv venv; \
|
||||
. venv/bin/activate; \
|
||||
python3 -m pip install ".[dev]"
|
||||
|
||||
.PHONY: core_clean
|
||||
core_clean: ## Clean the ingestion-core generated files
|
||||
rm -rf ingestion-core/src/metadata/generated
|
||||
rm -rf ingestion-core/build
|
||||
rm -rf ingestion-core/dist
|
||||
|
||||
.PHONY: core_generate
|
||||
core_generate: ## Generate the pydantic models from the JSON Schemas to the ingestion-core module
|
||||
$(MAKE) core_install_dev
|
||||
mkdir -p ingestion-core/src/metadata/generated; \
|
||||
. ingestion-core/venv/bin/activate; \
|
||||
datamodel-codegen --input openmetadata-spec/src/main/resources/json/schema --input-file-type jsonschema --output ingestion-core/src/metadata/generated/schema
|
||||
$(MAKE) core_py_antlr
|
||||
|
||||
.PHONY: core_bump_version_dev
|
||||
core_bump_version_dev: ## Bump a `dev` version to the ingestion-core module. To be used when schemas are updated
|
||||
$(MAKE) core_install_dev
|
||||
cd ingestion-core; \
|
||||
. venv/bin/activate; \
|
||||
python -m incremental.update metadata --dev
|
||||
|
||||
.PHONY: core_py_antlr
|
||||
core_py_antlr: ## Generate the Python core code for parsing FQNs under ingestion-core
|
||||
antlr4 -Dlanguage=Python3 -o ingestion-core/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4
|
||||
|
||||
.PHONY: py_antlr
|
||||
py_antlr: ## Generate the Python code for parsing FQNs
|
||||
antlr4 -Dlanguage=Python3 -o ingestion/src/metadata/generated/antlr ${PWD}/openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4
|
||||
@ -171,7 +137,7 @@ generate-schema-docs: ## Generates markdown files for documenting the JSON Sche
|
||||
|
||||
#Upgrade release automation scripts below
|
||||
.PHONY: update_all
|
||||
update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
|
||||
update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
|
||||
@echo "The release version is: $(RELEASE_VERSION)" ; \
|
||||
echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \
|
||||
$(MAKE) update_maven ; \
|
||||
@ -184,7 +150,7 @@ update_all: ## To update all the release related files run make update_all RELEA
|
||||
#make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
|
||||
|
||||
.PHONY: update_maven
|
||||
update_maven: ## To update the common and pom.xml maven version
|
||||
update_maven: ## To update the common and pom.xml maven version
|
||||
@echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \
|
||||
mvn versions:set -DnewVersion=$(RELEASE_VERSION)
|
||||
#remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions
|
||||
@ -192,7 +158,7 @@ update_maven: ## To update the common and pom.xml maven version
|
||||
|
||||
|
||||
.PHONY: update_github_action_paths
|
||||
update_github_action_paths: ## To update the github action ci docker files
|
||||
update_github_action_paths: ## To update the github action ci docker files
|
||||
@echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
|
||||
file_paths="docker/docker-compose-quickstart/Dockerfile \
|
||||
.github/workflows/docker-openmetadata-db.yml \
|
||||
@ -212,7 +178,7 @@ update_github_action_paths: ## To update the github action ci docker files
|
||||
#make update_github_action_paths RELEASE_VERSION=2.2.2
|
||||
|
||||
.PHONY: update_python_release_paths
|
||||
update_python_release_paths: ## To update the setup.py files
|
||||
update_python_release_paths: ## To update the setup.py files
|
||||
file_paths="ingestion/setup.py \
|
||||
openmetadata-airflow-apis/setup.py"; \
|
||||
echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \
|
||||
@ -223,7 +189,7 @@ update_python_release_paths: ## To update the setup.py files
|
||||
#make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2
|
||||
|
||||
.PHONY: update_dockerfile_version
|
||||
update_dockerfile_version: ## To update the dockerfiles version
|
||||
update_dockerfile_version: ## To update the dockerfiles version
|
||||
@file_paths="docker/docker-compose-ingestion/docker-compose-ingestion.yml \
|
||||
docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \
|
||||
docker/docker-compose-quickstart/docker-compose-postgres.yml \
|
||||
@ -236,7 +202,7 @@ update_dockerfile_version: ## To update the dockerfiles version
|
||||
#make update_dockerfile_version RELEASE_VERSION=2.2.2
|
||||
|
||||
.PHONY: update_ingestion_dockerfile_version
|
||||
update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version
|
||||
update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version
|
||||
@file_paths="ingestion/Dockerfile \
|
||||
ingestion/operators/docker/Dockerfile"; \
|
||||
echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
---
|
||||
This guide will help you set up OpenMetadata Core Models
|
||||
---
|
||||
|
||||
|
||||
|
||||
These models are `pydantic` models automatically generated from the
|
||||
central JSON Schemas that define our APIs and Entities.
|
||||
|
||||
**Prerequisites**
|
||||
|
||||
- Python >= 3.8.x
|
||||
|
||||
### Docs
|
||||
|
||||
Please refer to the documentation here https://docs.open-metadata.org/openmetadata/connectors
|
||||
|
||||
### Contribution
|
||||
|
||||
In order to contribute to this package:
|
||||
|
||||
```bash
|
||||
cd ingestion-core
|
||||
python -m virtualenv venv
|
||||
source venv/bin/activate
|
||||
python -m pip install ".[dev]"
|
||||
```
|
||||
|
||||
> OBS: During development we might need to treat this in a different
|
||||
virtual environment if we are yet to update the reference to the core
|
||||
package in `openmetadata-ingestion`.
|
||||
@ -1,56 +0,0 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
|
||||
from setuptools import find_namespace_packages, setup
|
||||
|
||||
|
||||
def get_long_description():
|
||||
root = os.path.dirname(__file__)
|
||||
with open(os.path.join(root, "README.md")) as f:
|
||||
description = f.read()
|
||||
return description
|
||||
|
||||
|
||||
dev = {
|
||||
"datamodel-code-generator==0.13.0",
|
||||
"black==21.12b0",
|
||||
"incremental",
|
||||
"twine",
|
||||
"twisted",
|
||||
"wheel",
|
||||
"click",
|
||||
"pydantic[email]==1.9.0",
|
||||
}
|
||||
|
||||
setup(
|
||||
name="openmetadata-ingestion-core",
|
||||
url="https://open-metadata.org/",
|
||||
author="OpenMetadata Committers",
|
||||
license="Apache License 2.0",
|
||||
description="These are the generated Python classes from JSON Schema",
|
||||
long_description=get_long_description(),
|
||||
long_description_content_type="text/markdown",
|
||||
python_requires=">=3.7",
|
||||
package_dir={"": "src"},
|
||||
zip_safe=False,
|
||||
use_incremental=True,
|
||||
setup_requires=["incremental"],
|
||||
install_requires=["incremental"],
|
||||
project_urls={
|
||||
"Documentation": "https://docs.open-metadata.org/",
|
||||
"Source": "https://github.com/open-metadata/OpenMetadata",
|
||||
},
|
||||
packages=find_namespace_packages(where="./src", exclude=["tests*"]),
|
||||
extras_require={
|
||||
"dev": list(dev),
|
||||
},
|
||||
)
|
||||
@ -1,11 +0,0 @@
|
||||
"""
|
||||
Provides metadata version information.
|
||||
"""
|
||||
|
||||
# This file is auto-generated! Do not edit!
|
||||
# Use `python -m incremental.update metadata` to change this file.
|
||||
|
||||
from incremental import Version
|
||||
|
||||
__version__ = Version("metadata", 0, 12, 0, dev=20)
|
||||
__all__ = ["__version__"]
|
||||
@ -1,11 +0,0 @@
|
||||
[run]
|
||||
source = env/lib/python3.9/site-packages/metadata
|
||||
relative_files = True
|
||||
branch = True
|
||||
[report]
|
||||
omit =
|
||||
*__init__*
|
||||
*/generated/*
|
||||
tests/*
|
||||
ingestion/src/*
|
||||
*/src/metadata/ingestion/source/database/sample_*
|
||||
@ -30,8 +30,8 @@ install_apis: ## Install the REST APIs module to the current environment
|
||||
python -m pip install $(ROOT_DIR)/openmetadata-airflow-apis/
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## Run pylint on the Python sources to analyze the codebase
|
||||
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint
|
||||
lint: ## Run pylint on the Python sources to analyze the codebase
|
||||
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint --rcfile=$(INGESTION_DIR)/pyproject.toml
|
||||
|
||||
.PHONY: precommit_install
|
||||
precommit_install: ## Install the project's precommit hooks from .pre-commit-config.yaml
|
||||
@ -41,32 +41,32 @@ precommit_install: ## Install the project's precommit hooks from .pre-commit-co
|
||||
|
||||
.PHONY: py_format
|
||||
py_format: ## Run black and isort to format the Python codebase
|
||||
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated --all
|
||||
isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/env --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3
|
||||
black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated
|
||||
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
|
||||
isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml
|
||||
black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
|
||||
|
||||
.PHONY: py_format_check
|
||||
py_format_check: ## Check if Python sources are correctly formatted
|
||||
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --extend-exclude $(PY_SOURCE)/metadata/generated --all
|
||||
isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3
|
||||
black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated
|
||||
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1)
|
||||
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --config $(INGESTION_DIR)/pyproject.toml
|
||||
isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --settings-file $(INGESTION_DIR)/pyproject.toml
|
||||
black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --config $(INGESTION_DIR)/pyproject.toml
|
||||
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --rcfile=$(INGESTION_DIR)/pyproject.toml --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1)
|
||||
|
||||
.PHONY: unit_ingestion
|
||||
unit_ingestion: ## Run Python unit tests
|
||||
coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-unit.xml --ignore=$(INGESTION_DIR)/tests/unit/source $(INGESTION_DIR)/tests/unit
|
||||
coverage run --rcfile $(INGESTION_DIR)/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(INGESTION_DIR)/junit/test-results-unit.xml --ignore=$(INGESTION_DIR)/tests/unit/source $(INGESTION_DIR)/tests/unit
|
||||
|
||||
## Ingestion tests & QA
|
||||
.PHONY: run_ometa_integration_tests
|
||||
run_ometa_integration_tests: ## Run Python integration tests
|
||||
coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-integration.xml $(INGESTION_DIR)/tests/integration/ometa $(INGESTION_DIR)/tests/integration/orm_profiler $(INGESTION_DIR)/tests/integration/test_suite $(INGESTION_DIR)/tests/integration/data_insight $(INGESTION_DIR)/tests/integration/lineage
|
||||
coverage run --rcfile $(INGESTION_DIR)/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(INGESTION_DIR)/junit/test-results-integration.xml $(INGESTION_DIR)/tests/integration/ometa $(INGESTION_DIR)/tests/integration/orm_profiler $(INGESTION_DIR)/tests/integration/test_suite $(INGESTION_DIR)/tests/integration/data_insight $(INGESTION_DIR)/tests/integration/lineage
|
||||
|
||||
.PHONY: run_python_tests
|
||||
run_python_tests: ## Run all Python tests with coverage
|
||||
coverage erase
|
||||
$(MAKE) unit_ingestion
|
||||
$(MAKE) run_ometa_integration_tests
|
||||
coverage report --rcfile $(INGESTION_DIR)/.coveragerc || true
|
||||
coverage report --rcfile $(INGESTION_DIR)/pyproject.toml || true
|
||||
|
||||
.PHONY: sonar_ingestion
|
||||
sonar_ingestion: ## Run the Sonar analysis based on the tests results and push it to SonarCloud
|
||||
@ -82,20 +82,17 @@ sonar_ingestion: ## Run the Sonar analysis based on the tests results and push
|
||||
.PHONY: run_apis_tests
|
||||
run_apis_tests: ## Run the openmetadata airflow apis tests
|
||||
coverage erase
|
||||
coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -a --branch -m pytest --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests
|
||||
coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc
|
||||
|
||||
coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -a --branch -m pytest -c $(INGESTION_DIR)/pyproject.toml --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests
|
||||
coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml
|
||||
|
||||
.PHONY: coverage_apis
|
||||
coverage_apis: ## Run the python tests on openmetadata-airflow-apis
|
||||
$(MAKE) run_apis_tests
|
||||
coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml
|
||||
coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/pyproject.toml -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml
|
||||
sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")\///g" $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml >> $(ROOT_DIR)/openmetadata-airflow-apis/ci-coverage.xml
|
||||
|
||||
|
||||
|
||||
.PHONY: coverage
|
||||
coverage: ## Run all Python tests and generate the coverage XML report
|
||||
$(MAKE) run_python_tests
|
||||
coverage xml --rcfile $(INGESTION_DIR)/.coveragerc -o $(INGESTION_DIR)/coverage.xml || true
|
||||
coverage xml --rcfile $(INGESTION_DIR)/pyproject.toml -o $(INGESTION_DIR)/coverage.xml || true
|
||||
sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")/src/g" $(INGESTION_DIR)/coverage.xml >> $(INGESTION_DIR)/ci-coverage.xml
|
||||
|
||||
125
ingestion/pyproject.toml
Normal file
125
ingestion/pyproject.toml
Normal file
@ -0,0 +1,125 @@
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
# We will keep handling dependencies in setup.py
|
||||
# since it helps us organize and isolate version management
|
||||
[project]
|
||||
name = "openmetadata-ingestion"
|
||||
version = "1.3.0.0.dev0"
|
||||
dynamic = ["readme"]
|
||||
authors = [
|
||||
{name = "OpenMetadata Committers"}
|
||||
]
|
||||
license = {file = "LICENSE"}
|
||||
description = "Ingestion Framework for OpenMetadata"
|
||||
requires-python = ">=3.8"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://open-metadata.org/"
|
||||
Documentation = "https://docs.open-metadata.org/"
|
||||
Source = "https://github.com/open-metadata/OpenMetadata"
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
readme = {file = ["README.md"]}
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["./src"]
|
||||
exclude = ["tests*"]
|
||||
namespaces = true
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
"metadata.examples" = ["workflows/*.yaml"]
|
||||
|
||||
[project.scripts]
|
||||
metadata-cli = "metadata.cmd:metadata"
|
||||
|
||||
[project.entry-points.apache_airflow_provider]
|
||||
provider_info = "airflow_provider_openmetadata:get_provider_config"
|
||||
|
||||
[tool.coverage.run]
|
||||
source = [
|
||||
"env/lib/python3.9/site-packages/metadata"
|
||||
]
|
||||
relative_files = true
|
||||
branch = true
|
||||
|
||||
[tool.coverage.report]
|
||||
omit = [
|
||||
"*__init__*",
|
||||
"*/generated/*",
|
||||
"tests/*",
|
||||
"ingestion/src/*",
|
||||
"*/src/metadata/ingestion/source/database/sample_*"
|
||||
]
|
||||
|
||||
[tool.mypy]
|
||||
mypy_path = "src"
|
||||
plugins = [
|
||||
"sqlalchemy.ext.mypy.plugin",
|
||||
"pydantic.mypy"
|
||||
]
|
||||
ignore_missing_imports = true
|
||||
namespace_packages = true
|
||||
strict_optional = true
|
||||
check_untyped_defs = true
|
||||
# eventually we'd like to enable these
|
||||
disallow_untyped_defs = false
|
||||
disallow_incomplete_defs = false
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
"slow: marks tests as slow (deselect with '-m \"not slow\"')"
|
||||
]
|
||||
|
||||
[tool.pylint.BASIC]
|
||||
# W1203: logging-fstring-interpolation - f-string brings better readability and unifies style
|
||||
# W1202: logging-format-interpolation - lazy formatting in logging functions
|
||||
# R0903: too-few-public-methods - False negatives in pydantic classes
|
||||
# W0707: raise-missing-from - Tends to be a false positive as exception are closely encapsulated
|
||||
# R0901: too-many-ancestors - We are already inheriting from SQA classes with a bunch of ancestors
|
||||
# W0703: broad-except - We are dealing with many different source systems, but we want to make sure workflows run until the end
|
||||
# W0511: fixme - These are internal notes and guides
|
||||
# W1518: method-cache-max-size-none - allow us to use LRU Cache with maxsize `None` to speed up certain calls
|
||||
disable = "W1203,W1202,R0903,W0707,R0901,W1201,W0703,W0511,W1518"
|
||||
|
||||
docstring-min-length = 20
|
||||
max-args = 7
|
||||
max-attributes = 12
|
||||
|
||||
# usual typevar naming
|
||||
good-names = "T,C,fn,db,df,i"
|
||||
module-rgx = "(([a-z_][a-z0-9_]*)|([a-zA-Z0-9]+))$"
|
||||
|
||||
[tool.pylint.MASTER]
|
||||
fail-under = 6.0
|
||||
init-hook = "from pylint.config import find_default_config_files; import os, sys; sys.path.append(os.path.dirname(next(find_default_config_files())))"
|
||||
extension-pkg-allow-list = "pydantic"
|
||||
load-plugins = "ingestion.plugins.print_checker"
|
||||
max-public-methods = 25
|
||||
|
||||
[tool.pylint."MESSAGES CONTROL"]
|
||||
disable = "no-name-in-module,import-error,duplicate-code"
|
||||
enable = "useless-suppression"
|
||||
|
||||
[tool.pylint.FORMAT]
|
||||
# We all have big monitors now
|
||||
max-line-length = 120
|
||||
|
||||
[tool.black]
|
||||
extend-exclude = "src/metadata/generated"
|
||||
|
||||
[tool.pycln]
|
||||
all = true
|
||||
extend-exclude = "src/metadata/generated"
|
||||
|
||||
[tool.isort]
|
||||
skip_glob = [
|
||||
"src/metadata/generated/*",
|
||||
"build/*",
|
||||
"env/*",
|
||||
"../openmetadata-airflow-apis/build/*"
|
||||
]
|
||||
profile = "black"
|
||||
indent = " "
|
||||
multi_line_output = 3
|
||||
@ -1,52 +0,0 @@
|
||||
[flake8]
|
||||
# We ignore the line length issues here, since black will take care of them.
|
||||
max-line-length = 150
|
||||
max-complexity = 15
|
||||
ignore =
|
||||
# Ignore: 1 blank line required before class docstring.
|
||||
D203,
|
||||
W503
|
||||
exclude =
|
||||
.git,
|
||||
__pycache__
|
||||
per-file-ignores =
|
||||
# imported but unused
|
||||
__init__.py: F401
|
||||
[metadata]
|
||||
license_files = LICENSE
|
||||
[mypy]
|
||||
mypy_path = src
|
||||
plugins =
|
||||
sqlalchemy.ext.mypy.plugin,
|
||||
pydantic.mypy
|
||||
ignore_missing_imports = yes
|
||||
namespace_packages = true
|
||||
strict_optional = yes
|
||||
check_untyped_defs = yes
|
||||
# eventually we'd like to enable these
|
||||
disallow_untyped_defs = no
|
||||
disallow_incomplete_defs = no
|
||||
|
||||
[isort]
|
||||
profile = black
|
||||
indent=' '
|
||||
sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
|
||||
|
||||
[tool:pytest]
|
||||
markers =
|
||||
slow: marks tests as slow (deselect with '-m "not slow"')
|
||||
testpaths =
|
||||
tests/unit
|
||||
|
||||
[options]
|
||||
packages = find:
|
||||
package_dir =
|
||||
=src
|
||||
|
||||
[options.packages.find]
|
||||
where = src
|
||||
include = *
|
||||
|
||||
[options.package_data]
|
||||
metadata.examples =
|
||||
workflows/*.yaml
|
||||
@ -13,18 +13,9 @@
|
||||
Python Dependencies
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Dict, Set
|
||||
|
||||
from setuptools import find_namespace_packages, setup
|
||||
|
||||
|
||||
def get_long_description():
|
||||
root = os.path.dirname(__file__)
|
||||
with open(os.path.join(root, "README.md"), encoding="UTF-8") as file:
|
||||
description = file.read()
|
||||
return description
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
# Add here versions required for multiple plugins
|
||||
VERSIONS = {
|
||||
@ -123,7 +114,6 @@ base_requirements = {
|
||||
"sqlalchemy>=1.4.0,<2",
|
||||
"collate-sqllineage>=1.0.4",
|
||||
"tabulate==0.9.0",
|
||||
"typing-compat~=0.1.0", # compatibility requirements for 3.7
|
||||
"typing_extensions<=4.5.0", # We need to have this fixed due to a yanked release 4.6.0
|
||||
"typing-inspect",
|
||||
"wheel~=0.38.4",
|
||||
@ -261,12 +251,13 @@ plugins: Dict[str, Set[str]] = {
|
||||
dev = {
|
||||
"black==22.3.0",
|
||||
"datamodel-code-generator==0.22.0",
|
||||
"docker",
|
||||
"isort",
|
||||
"pre-commit",
|
||||
"pycln",
|
||||
"pylint~=3.0.0",
|
||||
# For publishing
|
||||
"twine",
|
||||
"build",
|
||||
}
|
||||
|
||||
test = {
|
||||
@ -306,34 +297,7 @@ e2e_test = {
|
||||
"pytest-base-url",
|
||||
}
|
||||
|
||||
build_options = {"includes": ["_cffi_backend"]}
|
||||
setup(
|
||||
name="openmetadata-ingestion",
|
||||
version="1.3.0.0.dev0",
|
||||
url="https://open-metadata.org/",
|
||||
author="OpenMetadata Committers",
|
||||
license="Apache License 2.0",
|
||||
description="Ingestion Framework for OpenMetadata",
|
||||
long_description=get_long_description(),
|
||||
long_description_content_type="text/markdown",
|
||||
python_requires=">=3.8",
|
||||
options={"build_exe": build_options},
|
||||
package_dir={"": "src"},
|
||||
package_data={"metadata.examples": ["workflows/*.yaml"]},
|
||||
zip_safe=False,
|
||||
dependency_links=[],
|
||||
project_urls={
|
||||
"Documentation": "https://docs.open-metadata.org/",
|
||||
"Source": "https://github.com/open-metadata/OpenMetadata",
|
||||
},
|
||||
packages=find_namespace_packages(where="./src", exclude=["tests*"]),
|
||||
namespace_package=["metadata"],
|
||||
entry_points={
|
||||
"console_scripts": ["metadata = metadata.cmd:metadata"],
|
||||
"apache_airflow_provider": [
|
||||
"provider_info = airflow_provider_openmetadata:get_provider_config"
|
||||
],
|
||||
},
|
||||
install_requires=list(base_requirements),
|
||||
extras_require={
|
||||
"base": list(base_requirements),
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
[run]
|
||||
source = env/lib/python3.9/site-packages/openmetadata_managed_apis
|
||||
relative_files = True
|
||||
branch = True
|
||||
[report]
|
||||
omit =
|
||||
*__init__*
|
||||
tests/*
|
||||
views/**
|
||||
plugin.py
|
||||
201
openmetadata-airflow-apis/LICENSE
Normal file
201
openmetadata-airflow-apis/LICENSE
Normal file
@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
65
openmetadata-airflow-apis/pyproject.toml
Normal file
65
openmetadata-airflow-apis/pyproject.toml
Normal file
@ -0,0 +1,65 @@
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
# We will keep handling dependencies in setup.py
|
||||
# since it helps us organize and isolate version management
|
||||
[project]
|
||||
name = "openmetadata_managed_apis"
|
||||
version = "1.3.0.0.dev0"
|
||||
dynamic = ["readme"]
|
||||
authors = [
|
||||
{name = "OpenMetadata Committers"}
|
||||
]
|
||||
license = {file = "LICENSE"}
|
||||
description = "Airflow REST APIs to create and manage DAGS"
|
||||
requires-python = ">=3.8"
|
||||
dependencies = [
|
||||
"pendulum~=2.1.2",
|
||||
"apache-airflow>=2.2.2",
|
||||
"Flask>=1.1.4",
|
||||
"Flask-Admin==1.6.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"black==22.3.0",
|
||||
"pytest",
|
||||
"pylint",
|
||||
"pytest-cov",
|
||||
"isort",
|
||||
"pycln",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://open-metadata.org/"
|
||||
Documentation = "https://docs.open-metadata.org/"
|
||||
Source = "https://github.com/open-metadata/OpenMetadata"
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
readme = {file = ["README.md"]}
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["openmetadata_managed_apis.*", "openmetadata_managed_apis"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
"openmetadata_managed_apis" = ["views/templates/rest_api/index.html", "resources/dag_runner.j2"]
|
||||
|
||||
[project.entry-points."airflow.plugins"]
|
||||
openmetadata_managed_apis = "openmetadata_managed_apis.plugin:RestApiPlugin"
|
||||
|
||||
|
||||
[tool.coverage.run]
|
||||
source = [
|
||||
"env/lib/python3.9/site-packages/openmetadata_managed_apis"
|
||||
]
|
||||
relative_files = true
|
||||
branch = true
|
||||
|
||||
[tool.coverage.report]
|
||||
omit = [
|
||||
"*__init__*",
|
||||
"tests/*",
|
||||
"views/**",
|
||||
"plugin.py"
|
||||
]
|
||||
@ -1,96 +0,0 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import glob
|
||||
import os
|
||||
import pathlib
|
||||
from itertools import chain
|
||||
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
PLUGIN_NAME = "openmetadata_managed_apis"
|
||||
PLUGIN_ROOT_FILE = "plugin"
|
||||
PLUGIN_ENTRY_POINT = "RestApiPlugin"
|
||||
HOME = pathlib.Path(__file__).parent
|
||||
README = (HOME / "README.md").read_text()
|
||||
CONFIG = HOME / PLUGIN_NAME / "config.py"
|
||||
|
||||
|
||||
def get_package_data():
|
||||
"""
|
||||
Add templates and html files to the built
|
||||
package for easier deployments
|
||||
"""
|
||||
extensions = ["html", "j2"]
|
||||
|
||||
files = list(
|
||||
chain(
|
||||
*[
|
||||
glob.glob(f"{PLUGIN_NAME}/**/*.%s" % x, recursive=True)
|
||||
for x in extensions
|
||||
]
|
||||
)
|
||||
)
|
||||
return [x.split(os.sep, 1)[1] for x in files]
|
||||
|
||||
|
||||
def get_long_description():
|
||||
root = os.path.dirname(__file__)
|
||||
with open(os.path.join(root, "README.md")) as f:
|
||||
description = f.read()
|
||||
return description
|
||||
|
||||
|
||||
base_requirements = {
|
||||
"pendulum~=2.1.2",
|
||||
"apache-airflow>=2.2.2",
|
||||
"Flask>=1.1.4",
|
||||
"Flask-Admin==1.6.0",
|
||||
}
|
||||
|
||||
dev_requirements = {
|
||||
"black==22.3.0",
|
||||
"pytest",
|
||||
"pylint",
|
||||
"pytest-cov",
|
||||
"isort",
|
||||
"pycln",
|
||||
}
|
||||
|
||||
setup(
|
||||
name=PLUGIN_NAME,
|
||||
packages=find_packages(include=[f"{PLUGIN_NAME}.*", PLUGIN_NAME]),
|
||||
include_package_data=True,
|
||||
package_data={PLUGIN_NAME: get_package_data()},
|
||||
version="1.3.0.0.dev0",
|
||||
url="https://open-metadata.org/",
|
||||
author="OpenMetadata Committers",
|
||||
license="Apache License 2.0",
|
||||
description="Airflow REST APIs to create and manage DAGS",
|
||||
long_description=get_long_description(),
|
||||
long_description_content_type="text/markdown",
|
||||
entry_points={
|
||||
"airflow.plugins": [
|
||||
f"{PLUGIN_NAME} = {PLUGIN_NAME}.{PLUGIN_ROOT_FILE}:{PLUGIN_ENTRY_POINT}"
|
||||
]
|
||||
},
|
||||
python_requires=">=3.8",
|
||||
zip_safe=False,
|
||||
dependency_links=[],
|
||||
project_urls={
|
||||
"Documentation": "https://docs.open-metadata.org/",
|
||||
"Source": "https://github.com/open-metadata/OpenMetadata",
|
||||
},
|
||||
install_requires=list(base_requirements),
|
||||
extras_require={
|
||||
"base": list(base_requirements),
|
||||
"dev": list(dev_requirements),
|
||||
},
|
||||
)
|
||||
Loading…
x
Reference in New Issue
Block a user