mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-11-01 19:18:05 +00:00
33fe3b5ee5
* Moved more recipes into ingestion/Makefile * Removed some recipes into ingestion/Makefile and added import statement * Modified file paths so that 'make generate' works from the ingestion directory * Modified checks for current directory * Fixed function names to be in snake case * Reverted function names back to camel case * Reverted changes to js_antlr and py_antlr and moved generate command back into root directory Makefile * Updated run_ometa_integration_testsrecipe in ingestion/Makefile --------- Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
102 lines
5.3 KiB
Makefile
102 lines
5.3 KiB
Makefile
DIRECTORY_NAME := $(notdir $(CURDIR))
|
|
PY_SOURCE ?= ./src
|
|
|
|
ifeq (ingestion,$(DIRECTORY_NAME))
|
|
INGESTION_DIR := .
|
|
ROOT_DIR := ..
|
|
else
|
|
INGESTION_DIR := ingestion
|
|
ROOT_DIR := .
|
|
endif
|
|
|
|
.PHONY: install
|
|
install: ## Install the ingestion module to the current environment
|
|
python -m pip install $(INGESTION_DIR)/
|
|
|
|
.PHONY: install_dev
|
|
install_dev: ## Install the ingestion module with dev dependencies
|
|
python -m pip install "$(INGESTION_DIR)[dev]/"
|
|
|
|
.PHONY: install_test
|
|
install_test: ## Install the ingestion module with test dependencies
|
|
python -m pip install "$(INGESTION_DIR)[test]/"
|
|
|
|
.PHONY: install_all
|
|
install_all: ## Install the ingestion module with all dependencies
|
|
python -m pip install "$(INGESTION_DIR)[all]/"
|
|
|
|
.PHONY: install_apis
|
|
install_apis: ## Install the REST APIs module to the current environment
|
|
python -m pip install $(ROOT_DIR)openmetadata-airflow-apis/
|
|
|
|
.PHONY: lint
|
|
lint: ## Run pylint on the Python sources to analyze the codebase
|
|
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint
|
|
|
|
.PHONY: precommit_install
|
|
precommit_install: ## Install the project's precommit hooks from .pre-commit-config.yaml
|
|
@echo "Installing pre-commit hooks"
|
|
@echo "Make sure to first run install_test first"
|
|
pre-commit install
|
|
|
|
.PHONY: py_format
|
|
py_format: ## Run black and isort to format the Python codebase
|
|
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated --all
|
|
isort $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/env --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3
|
|
black $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated
|
|
|
|
.PHONY: py_format_check
|
|
py_format_check: ## Check if Python sources are correctly formatted
|
|
pycln $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --diff --extend-exclude $(PY_SOURCE)/metadata/generated --all
|
|
isort --check-only $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --skip $(PY_SOURCE)/metadata/generated --skip $(INGESTION_DIR)/build --skip $(ROOT_DIR)/openmetadata-airflow-apis/build --profile black --multi-line 3
|
|
black --check --diff $(INGESTION_DIR)/ $(ROOT_DIR)/openmetadata-airflow-apis/ --extend-exclude $(PY_SOURCE)/metadata/generated
|
|
PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" pylint --fail-under=10 $(PY_SOURCE)/metadata --ignore-paths $(PY_SOURCE)/metadata/generated || (echo "PyLint error code $$?"; exit 1)
|
|
|
|
.PHONY: unit_ingestion
|
|
unit_ingestion: ## Run Python unit tests
|
|
coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-unit.xml --ignore=$(INGESTION_DIR)/tests/unit/source $(INGESTION_DIR)/tests/unit
|
|
|
|
## Ingestion tests & QA
|
|
.PHONY: run_ometa_integration_tests
|
|
run_ometa_integration_tests: ## Run Python integration tests
|
|
coverage run --rcfile $(INGESTION_DIR)/.coveragerc -a --branch -m pytest -c $(INGESTION_DIR)/setup.cfg --junitxml=$(INGESTION_DIR)/junit/test-results-integration.xml $(INGESTION_DIR)/tests/integration/ometa $(INGESTION_DIR)/tests/integration/orm_profiler $(INGESTION_DIR)/tests/integration/test_suite $(INGESTION_DIR)/tests/integration/data_insight $(INGESTION_DIR)/tests/integration/lineage
|
|
|
|
.PHONY: run_python_tests
|
|
run_python_tests: ## Run all Python tests with coverage
|
|
coverage erase
|
|
$(MAKE) unit_ingestion
|
|
$(MAKE) run_ometa_integration_tests
|
|
coverage report --rcfile $(INGESTION_DIR)/.coveragerc || true
|
|
|
|
.PHONY: sonar_ingestion
|
|
sonar_ingestion: ## Run the Sonar analysis based on the tests results and push it to SonarCloud
|
|
docker run \
|
|
--rm \
|
|
-e SONAR_HOST_URL="https://sonarcloud.io" \
|
|
-e SONAR_SCANNER_OPTS="-Xmx1g" \
|
|
-e SONAR_LOGIN=$(token) \
|
|
-v ${PWD}/$(INGESTION_DIR):/usr/src \
|
|
sonarsource/sonar-scanner-cli \
|
|
-Dproject.settings=sonar-project.properties
|
|
|
|
.PHONY: run_apis_tests
|
|
run_apis_tests: ## Run the openmetadata airflow apis tests
|
|
coverage erase
|
|
coverage run --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -a --branch -m pytest --junitxml=$(ROOT_DIR)/openmetadata-airflow-apis/junit/test-results.xml $(ROOT_DIR)/openmetadata-airflow-apis/tests
|
|
coverage report --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc
|
|
|
|
|
|
.PHONY: coverage_apis
|
|
coverage_apis: ## Run the python tests on openmetadata-airflow-apis
|
|
$(MAKE) run_apis_tests
|
|
coverage xml --rcfile $(ROOT_DIR)/openmetadata-airflow-apis/.coveragerc -o $(ROOT_DIR)/openmetadata-airflow-apis/coverage.xml
|
|
sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")\///g" $(INGESTION_DIR)/openmetadata-airflow-apis/coverage.xml >> $(INGESTION_DIR)/openmetadata-airflow-apis/ci-coverage.xml
|
|
|
|
|
|
|
|
.PHONY: coverage
|
|
coverage: ## Run all Python tests and generate the coverage XML report
|
|
$(MAKE) run_python_tests
|
|
coverage xml --rcfile $(INGESTION_DIR)/.coveragerc -o $(INGESTION_DIR)/coverage.xml || true
|
|
sed -e "s/$(shell python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())).replace('/','\/'))")/src/g" $(INGESTION_DIR)/coverage.xml >> $(INGESTION_DIR)/ci-coverage.xml
|