yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-02 21:53:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion): make ingestion image runtime configurable (#3087)

Browse Source 
Co-authored-by: Jorgen Evens <jorgen@evens.eu>
This commit is contained in:
Sriharsha Chintalapani 2022-03-16 22:54:14 -07:00 committed by GitHub
parent 83de95c3f2
commit 3f91ae33c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 31 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
ingestion/ingestion_dependency.sh
View File

@ -10,20 +10,44 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done MYSQL_HOST=${MYSQL_HOST:-mysql}
MYSQL_PORT=${MYSQL_PORT:-3306}
MYSQL_DB=${MYSQL_DB:-airflow_db}
MYSQL_USER=${MYSQL_USER:-airflow_user}
MYSQL_PASSWORD=${MYSQL_PASSWORD:-airflow_pass}
MYSQL_CONN="${MYSQL_USER}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}"
AIRFLOW_ADMIN_USER=${AIRFLOW_ADMIN_USER:-admin}
AIRFLOW_ADMIN_PASSWORD=${AIRFLOW_ADMIN_PASSWORD:-admin}
OPENMETADATA_SERVER=${OPENMETADATA_SERVER:-"http://openmetadata-server:8585"}
sed -i "s#\(sql_alchemy_conn = \).*#\1mysql+pymysql://${MYSQL_CONN}#" /airflow/airflow.cfg
while ! wget -O /dev/null -o /dev/null $MYSQL_HOST:$MYSQL_PORT; do sleep 5; done
airflow db init airflow db init
airflow users create \ airflow users create \
--username admin \ --username ${AIRFLOW_ADMIN_USER} \
--firstname Peter \ --firstname Peter \
--lastname Parker \ --lastname Parker \
--role Admin \ --role Admin \
--email spiderman@superhero.org \ --email spiderman@superhero.org \
--password admin --password ${AIRFLOW_ADMIN_PASSWORD}
(while ! wget -O /dev/null -o /dev/null http://ingestion:8080; do sleep 5; done; sleep 5; curl -u admin:admin --data '{"dag_run_id":"sample_data_1"}' -H "Content-type: application/json" -X POST http://ingestion:8080/api/v1/dags/sample_data/dagRuns) &
(while ! wget -O /dev/null -o /dev/null http://openmetadata-server:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 7; curl -u admin:admin --data '{"dag_run_id":"sample_usage_1"}' -H "Content-type: application/json" -X POST http://ingestion:8080/api/v1/dags/sample_usage/dagRuns) & if [ ! -z "${LOAD_SAMPLE}" ]; then
(while ! wget -O /dev/null -o /dev/null http://openmetadata-server:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 9; curl -u admin:admin --data '{"dag_run_id":"sample_dbt_1"}' -H "Content-type: application/json" -X POST http://ingestion:8080/api/v1/dags/sample_dbt/dagRuns) & AIRFLOW_CREDENTIALS="${AIRFLOW_ADMIN_USER}:${AIRFLOW_ADMIN_PASSWORD}"
(while ! wget -O /dev/null -o /dev/null http://openmetadata-server:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 10; curl -u admin:admin --data '{"dag_run_id":"index_metadata_1"}' -H "Content-type: application/json" -X POST http://ingestion:8080/api/v1/dags/index_metadata/dagRuns) & (while ! wget -O /dev/null -o /dev/null http://localhost:8080; do sleep 5; done; sleep 5; curl -u "${AIRFLOW_CREDENTIALS}" --data '{"dag_run_id":"sample_data_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/sample_data/dagRuns) &
(while ! wget -O /dev/null -o /dev/null $OPENMETADATA_SERVER/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 7; curl -u "${AIRFLOW_CREDENTIALS}" --data '{"dag_run_id":"sample_usage_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/sample_usage/dagRuns) &
(while ! wget -O /dev/null -o /dev/null $OPENMETADATA_SERVER/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 9; curl -u "${AIRFLOW_CREDENTIALS}" --data '{"dag_run_id":"sample_dbt_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/sample_dbt/dagRuns) &
(while ! wget -O /dev/null -o /dev/null $OPENMETADATA_SERVER/api/v1/tables/name/bigquery_gcp.shopify.fact_sale; do sleep 5; done; sleep 10; curl -u "${AIRFLOW_CREDENTIALS}" --data '{"dag_run_id":"index_metadata_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/index_metadata/dagRuns) &
fi
airflow webserver --port 8080 -D & airflow webserver --port 8080 -D &
(sleep 5; airflow db upgrade) (sleep 5; airflow db upgrade)
(sleep 5; airflow db upgrade) (sleep 5; airflow db upgrade)
airflow scheduler airflow scheduler