[ { "job_id": 11223344, "creator_user_name": "user.name@databricks.com", "settings": { "name": "OpenMetadata Databricks Workflow", "description": "This job contain multiple tasks that are required to produce the weekly shark sightings report.", "email_notifications": { "on_start": [ "user.name@databricks.com" ], "on_success": [ "user.name@databricks.com" ], "on_failure": [ "user.name@databricks.com" ], "on_duration_warning_threshold_exceeded": [ "user.name@databricks.com" ], "no_alert_for_skipped_runs": false }, "webhook_notifications": { "on_start": [ [ { "id": "0481e838-0a59-4eff-9541-a4ca6f149574" } ] ], "on_success": [ [ { "id": "0481e838-0a59-4eff-9541-a4ca6f149574" } ] ], "on_failure": [ [ { "id": "0481e838-0a59-4eff-9541-a4ca6f149574" } ] ], "on_duration_warning_threshold_exceeded": [ [ { "id": "0481e838-0a59-4eff-9541-a4ca6f149574" } ] ] }, "notification_settings": { "no_alert_for_skipped_runs": false, "no_alert_for_canceled_runs": false }, "timeout_seconds": 86400, "health": { "rules": [ { "metric": "RUN_DURATION_SECONDS", "op": "GREATER_THAN", "value": 10 } ] }, "schedule": { "quartz_cron_expression": "20 30 * * * ?", "timezone_id": "Europe/London", "pause_status": "UNPAUSED" }, "trigger": { "pause_status": "UNPAUSED", "file_arrival": { "url": "string", "min_time_between_triggers_seconds": 0, "wait_after_last_change_seconds": 0 } }, "continuous": { "pause_status": "UNPAUSED" }, "max_concurrent_runs": 10, "tasks": [ { "max_retries": 3, "task_key": "Sessionize", "description": "Extracts session data from events", "min_retry_interval_millis": 2000, "depends_on": [], "timeout_seconds": 86400, "spark_jar_task": { "main_class_name": "com.databricks.Sessionize", "parameters": [ "--data", "dbfs:/path/to/data.json" ] }, "libraries": [ { "jar": "dbfs:/mnt/databricks/Sessionize.jar" } ], "retry_on_timeout": false, "existing_cluster_id": "0923-164208-meows279" }, { "max_retries": 3, "task_key": "Orders_Ingest", "description": "Ingests order data", "job_cluster_key": "auto_scaling_cluster", "min_retry_interval_millis": 2000, "depends_on": [], "timeout_seconds": 86400, "spark_jar_task": { "main_class_name": "com.databricks.OrdersIngest", "parameters": [ "--data", "dbfs:/path/to/order-data.json" ] }, "libraries": [ { "jar": "dbfs:/mnt/databricks/OrderIngest.jar" } ], "retry_on_timeout": false }, { "max_retries": 3, "task_key": "Matched_Changed", "description": "Matches orders with user sessions", "notebook_task": { "base_parameters": { "age": "35", "name": "John Doe" }, "notebook_path": "/Users/user.name@databricks.com/Match" }, "min_retry_interval_millis": 2000, "depends_on": [ { "task_key": "Orders_Ingest" }, { "task_key": "Sessionize" }, { "task_key": "Sessionize_duplicated" } ], "new_cluster": { "autoscale": { "max_workers": 16, "min_workers": 2 }, "node_type_id": null, "spark_conf": { "spark.speculation": true }, "spark_version": "7.3.x-scala2.12" }, "timeout_seconds": 86400, "retry_on_timeout": false, "run_if": "ALL_SUCCESS" } ], "job_clusters": [ { "job_cluster_key": "auto_scaling_cluster", "new_cluster": { "autoscale": { "max_workers": 16, "min_workers": 2 }, "node_type_id": null, "spark_conf": { "spark.speculation": true }, "spark_version": "7.3.x-scala2.12" } } ], "git_source": { "git_branch": "main", "git_provider": "gitHub", "git_url": "https://github.com/databricks/databricks-cli" }, "tags": { "cost-center": "engineering", "team": "jobs" }, "format": "SINGLE_TASK", "queue": { "enabled": true }, "parameters": [ { "default": "users", "name": "table" } ], "run_as": { "user_name": "user@databricks.com", "service_principal_name": "692bc6d0-ffa3-11ed-be56-0242ac120002" }, "edit_mode": "UI_LOCKED", "deployment": { "kind": "BUNDLE", "metadata_file_path": "string" }, "environments": [ { "environment_key": "string", "spec": { "client": "1", "dependencies": [ "string" ] } } ] }, "created_time": 1601370337343 } ]