diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index dacd14617c6..3d417c49750 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -19,12 +19,12 @@ echo "Starting Local Docker Containers" docker compose down && docker compose up --build -d -until curl -s -f -o /dev/null "http://localhost:9200/_cat/indices/team_search_index"; do - printf '.' +until curl -s -f "http://localhost:9200/_cat/indices/team_search_index"; do + printf 'Checking if Elastic Search instance is up...\n' sleep 5 done -until curl -s -f -o /dev/null --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do - printf '.' +until curl -s -f --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do + printf 'Checking if Sample Data DAG is reachable...\n' sleep 5 done curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_data' \ @@ -33,9 +33,16 @@ curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_data' \ --data-raw '{ "is_paused": false }' -until curl -s -f -o /dev/null "http://localhost:8585/api/v1/tables/name/sample_data.ecommerce_db.shopify.fact_sale"; do - printf '.' - sleep 2 + +cd ../ +printf 'Validate sample data DAG...' +sleep 5 +python validate_compose.py + +until curl -s -f "http://localhost:8585/api/v1/tables/name/sample_data.ecommerce_db.shopify.fact_sale"; do + printf 'Waiting on Sample Data Ingestion to complete...\n' + curl -v "http://localhost:8585/api/v1/tables" + sleep 5 done sleep 5 curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_usage' \ diff --git a/docker/validate_compose.py b/docker/validate_compose.py new file mode 100644 index 00000000000..4a4bcf2d3e4 --- /dev/null +++ b/docker/validate_compose.py @@ -0,0 +1,65 @@ +from typing import Tuple + +from pprint import pprint +import requests +import time + +HEADER_AUTH = {"Authorization": "Basic YWRtaW46YWRtaW4="} +HEADER_JSON = {"Content-Type": "application/json"} + + +def get_apis_token_header() -> dict: + """ + Get our APIs token + """ + req = requests.post( + "http://localhost:8080/api/v1/security/login", + json={"username": "admin", "password": "admin", "refresh": "true", "provider": "db"}, + headers=HEADER_JSON, + ).json() + + return {"Authorization": f"Bearer {req.get('access_token')}"} + + +def get_last_run_info() -> Tuple[str, str]: + """ + Make sure we can pick up the latest run info + """ + dag_runs = None + while not dag_runs: + print("Waiting for DAG Run data...") + time.sleep(5) + runs = requests.get("http://localhost:8080/api/v1/dags/sample_data/dagRuns", headers=HEADER_AUTH).json() + dag_runs = runs.get("dag_runs") + + return dag_runs[0].get("dag_run_id"), dag_runs[0].get("state") + + +def print_last_run_logs() -> None: + """ + Show the logs + """ + token = get_apis_token_header() + logs = requests.get( + "http://localhost:8080/rest_api/api?api=last_dag_logs&dag_id=sample_data", + headers={**HEADER_JSON, **token} + ).text + pprint(logs) + + +def main(): + + state = None + while state != "success": + + print("Waiting for sample data ingestion to be a success. We'll show some logs along the way.") + + dag_run_id, state = get_last_run_info() + print(f"DAG run: [{dag_run_id}, {state}]") + + print_last_run_logs() + time.sleep(10) + + +if __name__ == "__main__": + main()