Added fix for Sample_Usage not being ingested in Local docker (#3650)

docker/local-metadata/Dockerfile

@@ -17,7 +17,7 @@ RUN  apk update \
   && apk upgrade \
   && apk add --update wget curl bash openjdk11 \
   && rm -rf /var/cache/apk/*
-COPY docker/metadata/openmetadata-start.sh docker/metadata/openmetadata.yaml ./
+COPY docker/metadata/openmetadata-start.sh ./
 COPY openmetadata-dist/target/openmetadata-*.tar.gz /
 RUN tar zxvf openmetadata-*.tar.gz && \
     rm openmetadata-*.tar.gz

docker/local-metadata/docker-compose.yml

@@ -17,6 +17,8 @@ services:
       dockerfile: docker/local-metadata/Dockerfile_mysql
     container_name: openmetadata_mysql
     restart: always
+    depends_on:
+      - elasticsearch
     environment:
       MYSQL_ROOT_PASSWORD: password
     expose:
@@ -71,7 +73,9 @@ services:
     ports:
       - 8585:8585
     depends_on:
+      - elasticsearch
       - mysql
+      - ingestion
     networks:
       local_app_net:
         ipv4_address: 172.16.239.13
@@ -86,6 +90,7 @@ services:
     container_name: openmetadata_ingestion
     depends_on:
       - mysql
+      - elasticsearch
     expose:
       - 8080
     ports:

docker/metadata/openmetadata-start.sh

@@ -14,7 +14,6 @@ MYSQL="${MYSQL_HOST:-mysql}":"${MYSQL_PORT:-3306}"
 while ! wget -O /dev/null -o /dev/null "${MYSQL}";
   do echo "Trying to connect to ${MYSQL}"; sleep 5;
 done
-cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml
 cd /openmetadata-*/
 ./bootstrap/bootstrap_storage.sh migrate-all
 ./bin/openmetadata-server-start.sh conf/openmetadata.yaml

docker/run_local_docker.sh

@@ -10,16 +10,49 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-cd "$( dirname "${BASH_SOURCE[0]}" )"
+cd "$(dirname "${BASH_SOURCE[0]}")"
 echo "Maven Build - Skipping Tests"
 cd ../ && mvn -DskipTests clean package
 echo "Prepare Docker volume for the operators"
 cd docker/local-metadata
 echo "Starting Local Docker Containers"
+
 docker compose down && docker compose up --build -d
-until curl -s -f -o /dev/null "http://localhost:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale"; do
-    printf '.'
-    sleep 2
+
+until curl -s -f -o /dev/null "http://localhost:9200/_cat/indices/team_search_index"; do
+  printf '.'
+  sleep 5
 done
-curl -u "admin:admin" --data '{"dag_run_id":"es_index_1"}' -H "Content-type: application/json" -X POST http://localhost:8080/api/v1/dags/index_metadata/dagRuns
-tput setaf 2; echo "✔ OpenMetadata is up and running"
+until curl -s -f -o /dev/null --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do
+  printf '.'
+  sleep 5
+done
+curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_data' \
+  --header 'Authorization: Basic YWRtaW46YWRtaW4=' \
+  --header 'Content-Type: application/json' \
+  --data-raw '{
+        "is_paused": false
+      }'
+until curl -s -f -o /dev/null "http://localhost:8585/api/v1/tables/name/bigquery_gcp.shopify.fact_sale"; do
+  printf '.'
+  sleep 2
+done
+sleep 5
+curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_usage' \
+  --header 'Authorization: Basic YWRtaW46YWRtaW4=' \
+  --header 'Content-Type: application/json' \
+  --data-raw '{
+      "is_paused": false
+      }'
+sleep 5
+curl --location --request PATCH 'localhost:8080/api/v1/dags/index_metadata' \
+  --header 'Authorization: Basic YWRtaW46YWRtaW4=' \
+  --header 'Content-Type: application/json' \
+  --data-raw '{
+      "is_paused": false
+      }'
+sleep 2
+tput setaf 2
+echo "✔ OpenMetadata is up and running"
+
+

ingestion/examples/airflow/dags/airflow_metadata_to_es.py

@@ -77,7 +77,7 @@ with DAG(
     default_args=default_args,
     description="An example DAG which runs a OpenMetadata ingestion workflow",
     start_date=days_ago(1),
-    is_paused_upon_creation=False,
+    is_paused_upon_creation=True,
     catchup=False,
 ) as dag:
     ingest_task = PythonOperator(

ingestion/examples/airflow/dags/airflow_sample_data.py

@@ -69,7 +69,7 @@ with DAG(
     default_args=default_args,
     description="An example DAG which runs a OpenMetadata ingestion workflow",
     start_date=days_ago(1),
-    is_paused_upon_creation=False,
+    is_paused_upon_creation=True,
     catchup=False,
 ) as dag:
     ingest_task = PythonOperator(

ingestion/examples/airflow/dags/airflow_sample_usage.py

@@ -87,7 +87,7 @@ with DAG(
     description="An example DAG which runs a OpenMetadata ingestion workflow",
     schedule_interval=timedelta(days=1),
     start_date=days_ago(1),
-    is_paused_upon_creation=False,
+    is_paused_upon_creation=True,
     catchup=False,
 ) as dag:
     ingest_task = PythonOperator(