diff --git a/docker/local-metadata/Dockerfile b/docker/local-metadata/Dockerfile index 7640f9c7231..779a9087a36 100644 --- a/docker/local-metadata/Dockerfile +++ b/docker/local-metadata/Dockerfile @@ -17,7 +17,7 @@ RUN apk update \ && apk upgrade \ && apk add --update wget curl bash openjdk11 \ && rm -rf /var/cache/apk/* -COPY docker/metadata/openmetadata-start.sh docker/metadata/openmetadata.yaml ./ +COPY docker/metadata/openmetadata-start.sh ./ COPY openmetadata-dist/target/openmetadata-*.tar.gz / RUN tar zxvf openmetadata-*.tar.gz && \ rm openmetadata-*.tar.gz diff --git a/docker/local-metadata/docker-compose.yml b/docker/local-metadata/docker-compose.yml index 0d37757dd4d..0e73e948625 100644 --- a/docker/local-metadata/docker-compose.yml +++ b/docker/local-metadata/docker-compose.yml @@ -17,6 +17,8 @@ services: dockerfile: docker/local-metadata/Dockerfile_mysql container_name: openmetadata_mysql restart: always + depends_on: + - elasticsearch environment: MYSQL_ROOT_PASSWORD: password expose: @@ -71,7 +73,9 @@ services: ports: - 8585:8585 depends_on: + - elasticsearch - mysql + - ingestion networks: local_app_net: ipv4_address: 172.16.239.13 @@ -86,6 +90,7 @@ services: container_name: openmetadata_ingestion depends_on: - mysql + - elasticsearch expose: - 8080 ports: diff --git a/docker/metadata/openmetadata-start.sh b/docker/metadata/openmetadata-start.sh index 4270d932ad5..1bd0e857e81 100644 --- a/docker/metadata/openmetadata-start.sh +++ b/docker/metadata/openmetadata-start.sh @@ -14,7 +14,6 @@ MYSQL="${MYSQL_HOST:-mysql}":"${MYSQL_PORT:-3306}" while ! wget -O /dev/null -o /dev/null "${MYSQL}"; do echo "Trying to connect to ${MYSQL}"; sleep 5; done -cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml cd /openmetadata-*/ ./bootstrap/bootstrap_storage.sh migrate-all ./bin/openmetadata-server-start.sh conf/openmetadata.yaml diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 87305fed41e..d75fa3ae6f6 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -10,16 +10,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -cd "$( dirname "${BASH_SOURCE[0]}" )" +cd "$(dirname "${BASH_SOURCE[0]}")" echo "Maven Build - Skipping Tests" cd ../ && mvn -DskipTests clean package echo "Prepare Docker volume for the operators" cd docker/local-metadata echo "Starting Local Docker Containers" + docker compose down && docker compose up --build -d -until curl -s -f -o /dev/null "http://localhost:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale"; do - printf '.' - sleep 2 + +until curl -s -f -o /dev/null "http://localhost:9200/_cat/indices/team_search_index"; do + printf '.' + sleep 5 done -curl -u "admin:admin" --data '{"dag_run_id":"es_index_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/index_metadata/dagRuns -tput setaf 2; echo "✔ OpenMetadata is up and running" \ No newline at end of file +until curl -s -f -o /dev/null --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do + printf '.' + sleep 5 +done +curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_data' \ + --header 'Authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "is_paused": false + }' +until curl -s -f -o /dev/null "http://localhost:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale"; do + printf '.' + sleep 2 +done +sleep 5 +curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_usage' \ + --header 'Authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "is_paused": false + }' +sleep 5 +curl --location --request PATCH 'localhost:8080/api/v1/dags/index_metadata' \ + --header 'Authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "is_paused": false + }' +sleep 2 +tput setaf 2 +echo "✔ OpenMetadata is up and running" + + diff --git a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py index 7daac50a699..d565e90c983 100644 --- a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py +++ b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py @@ -77,7 +77,7 @@ with DAG( default_args=default_args, description="An example DAG which runs a OpenMetadata ingestion workflow", start_date=days_ago(1), - is_paused_upon_creation=False, + is_paused_upon_creation=True, catchup=False, ) as dag: ingest_task = PythonOperator( diff --git a/ingestion/examples/airflow/dags/airflow_sample_data.py b/ingestion/examples/airflow/dags/airflow_sample_data.py index a1f8f26bce9..640986eac7f 100644 --- a/ingestion/examples/airflow/dags/airflow_sample_data.py +++ b/ingestion/examples/airflow/dags/airflow_sample_data.py @@ -69,7 +69,7 @@ with DAG( default_args=default_args, description="An example DAG which runs a OpenMetadata ingestion workflow", start_date=days_ago(1), - is_paused_upon_creation=False, + is_paused_upon_creation=True, catchup=False, ) as dag: ingest_task = PythonOperator( diff --git a/ingestion/examples/airflow/dags/airflow_sample_usage.py b/ingestion/examples/airflow/dags/airflow_sample_usage.py index 104a49db351..80c7dc25595 100644 --- a/ingestion/examples/airflow/dags/airflow_sample_usage.py +++ b/ingestion/examples/airflow/dags/airflow_sample_usage.py @@ -87,7 +87,7 @@ with DAG( description="An example DAG which runs a OpenMetadata ingestion workflow", schedule_interval=timedelta(days=1), start_date=days_ago(1), - is_paused_upon_creation=False, + is_paused_upon_creation=True, catchup=False, ) as dag: ingest_task = PythonOperator(