mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-28 07:58:31 +00:00
2ec32017bb
* Fix typo * Clean setup * Update ingestion local image to be barebone on connector dependencies * Prepare ingestion connectors base image * Add system dependencies * Prepare docker CLI * Add docker provider * Prepare entrypoint for the image * Remove DBT pipeline as per Issue 1658 * Add TODO for ingestion build * Bind docker socket * Update comment * Update README * Use DockerOperator in sample data * Build images with latest tag * Prepare symlink to pass the volume to the DockerOperator * Update README * Prepare Base image for CI * COPY multiple files into dir * COPY multiple files into dir * Remove DBT source as is now part of table ingestion * Build docker base in run_local_docker
63 lines
1.8 KiB
Makefile
63 lines
1.8 KiB
Makefile
.PHONY: env38
|
|
PY_SOURCE ?= ingestion/src
|
|
|
|
env38:
|
|
# virtualenv -p python3.8 env38
|
|
python3.8 -m venv env38
|
|
clean_env37:
|
|
rm -rf env38
|
|
|
|
install:
|
|
python3 -m pip install ingestion/
|
|
|
|
install_test:
|
|
python3 -m pip install -r ingestion/requirements-test.txt
|
|
|
|
install_dev:
|
|
python3 -m pip install -r ingestion/requirements-dev.txt
|
|
|
|
precommit_install:
|
|
@echo "Installing pre-commit hooks"
|
|
@echo "Make sure to first run `make install_test`"
|
|
pre-commit install --config ingestion/.pre-commit-config.yaml
|
|
|
|
isort:
|
|
isort $(PY_SOURCE) --skip $(PY_SOURCE)/metadata/generated --profile black --multi-line 3
|
|
|
|
lint:
|
|
find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint
|
|
|
|
black:
|
|
black $(PY_SOURCE) --exclude $(PY_SOURCE)/metadata/generated
|
|
|
|
black_check:
|
|
black --check --diff $(PY_SOURCE) --exclude $(PY_SOURCE)/metadata/generated
|
|
|
|
generate:
|
|
@echo "Running Datamodel Code Generator"
|
|
@echo "Make sure to first run `make install_dev`"
|
|
datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion/src/metadata/generated
|
|
|
|
run_ometa_integration_tests:
|
|
cd ingestion; \
|
|
pytest -c setup.cfg --override-ini=testpaths="tests/integration/ometa tests/unit/stage_test.py"
|
|
|
|
publish:
|
|
make install_dev generate
|
|
cd ingestion; \
|
|
python setup.py install sdist bdist_wheel; \
|
|
twine check dist/*; \
|
|
twine upload dist/*
|
|
|
|
build_docker_base:
|
|
make install_dev generate
|
|
docker build -f ingestion/connectors/Dockerfile-base ingestion/ -t openmetadata/ingestion-connector-base
|
|
|
|
build_docker_connectors:
|
|
@echo "Building Docker connectors. Make sure to run build_docker_base first"
|
|
python ingestion/connectors/docker-cli.py build
|
|
|
|
push_docker_connectors:
|
|
@echo "Pushing Docker connectors. Make sure to run build_docker_connectors first"
|
|
python ingestion/connectors/docker-cli.py push
|