diff --git a/docs/install/metadata-ingestion/connectors/dashboard-services/looker.md b/docs/install/metadata-ingestion/connectors/dashboard-services/looker.md index 0ab5c7eb156..c3d46c72dda 100644 --- a/docs/install/metadata-ingestion/connectors/dashboard-services/looker.md +++ b/docs/install/metadata-ingestion/connectors/dashboard-services/looker.md @@ -82,13 +82,6 @@ Add Optionally`pii` processor and `metadata-rest-tables` sink along with `metada "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/dashboard-services/superset.md b/docs/install/metadata-ingestion/connectors/dashboard-services/superset.md index 8abc4f60ec2..815133bbc4d 100644 --- a/docs/install/metadata-ingestion/connectors/dashboard-services/superset.md +++ b/docs/install/metadata-ingestion/connectors/dashboard-services/superset.md @@ -82,13 +82,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/dashboard-services/tableau.md b/docs/install/metadata-ingestion/connectors/dashboard-services/tableau.md index a620f940cf0..e3816e36bb8 100644 --- a/docs/install/metadata-ingestion/connectors/dashboard-services/tableau.md +++ b/docs/install/metadata-ingestion/connectors/dashboard-services/tableau.md @@ -92,13 +92,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/athena.md b/docs/install/metadata-ingestion/connectors/database-services/athena.md index ae40907e71f..abffbdac8ec 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/athena.md +++ b/docs/install/metadata-ingestion/connectors/database-services/athena.md @@ -83,13 +83,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/bigquery.md b/docs/install/metadata-ingestion/connectors/database-services/bigquery.md index 275c55f0d5e..cb342f6dd24 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/bigquery.md +++ b/docs/install/metadata-ingestion/connectors/database-services/bigquery.md @@ -121,13 +121,6 @@ Add Optionally`pii` processor and `metadata-rest-tables` sink along with `metada "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/elastic-search.md b/docs/install/metadata-ingestion/connectors/database-services/elastic-search.md index 9bf788b3636..329611c0348 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/elastic-search.md +++ b/docs/install/metadata-ingestion/connectors/database-services/elastic-search.md @@ -77,13 +77,6 @@ Add Optionally `file` stage and `elasticsearch` bulk_sink along with `metadata-s "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/hive.md b/docs/install/metadata-ingestion/connectors/database-services/hive.md index 46840b7c292..587e3f25fa3 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/hive.md +++ b/docs/install/metadata-ingestion/connectors/database-services/hive.md @@ -69,13 +69,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/mssql.md b/docs/install/metadata-ingestion/connectors/database-services/mssql.md index 487e1187b32..9592ff9ab87 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/mssql.md +++ b/docs/install/metadata-ingestion/connectors/database-services/mssql.md @@ -92,13 +92,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/mysql.md b/docs/install/metadata-ingestion/connectors/database-services/mysql.md index eeeb654418a..1f952fd74e0 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/mysql.md +++ b/docs/install/metadata-ingestion/connectors/database-services/mysql.md @@ -86,13 +86,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/oracle.md b/docs/install/metadata-ingestion/connectors/database-services/oracle.md index 5d6f65aaf1b..c53fb0e8358 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/oracle.md +++ b/docs/install/metadata-ingestion/connectors/database-services/oracle.md @@ -83,13 +83,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/postgres.md b/docs/install/metadata-ingestion/connectors/database-services/postgres.md index d79089bf881..16c78ac2f80 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/postgres.md +++ b/docs/install/metadata-ingestion/connectors/database-services/postgres.md @@ -83,13 +83,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/presto.md b/docs/install/metadata-ingestion/connectors/database-services/presto.md index af77efd72f9..e7cae91b2c5 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/presto.md +++ b/docs/install/metadata-ingestion/connectors/database-services/presto.md @@ -76,13 +76,6 @@ add `metadata-rest-tables` sink along with `metadata-server` config "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/redshift-usage.md b/docs/install/metadata-ingestion/connectors/database-services/redshift-usage.md index c053d878e2a..047f072c6e0 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/redshift-usage.md +++ b/docs/install/metadata-ingestion/connectors/database-services/redshift-usage.md @@ -100,13 +100,6 @@ Add optionally`query-parser` processor, `table-usage` stage and `metadata-usage` "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/salesforce.md b/docs/install/metadata-ingestion/connectors/database-services/salesforce.md index cdc3628c779..003463ed5b9 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/salesforce.md +++ b/docs/install/metadata-ingestion/connectors/database-services/salesforce.md @@ -86,13 +86,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/snowflake-usage.md b/docs/install/metadata-ingestion/connectors/database-services/snowflake-usage.md index 46f0ce59e69..e0ea0db00cf 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/snowflake-usage.md +++ b/docs/install/metadata-ingestion/connectors/database-services/snowflake-usage.md @@ -100,13 +100,6 @@ Add Optionally `query-parser` processor, `table-usage` stage and`metadata-usage` "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/snowflake.md b/docs/install/metadata-ingestion/connectors/database-services/snowflake.md index 0aafbd4d0fe..80e4f471227 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/snowflake.md +++ b/docs/install/metadata-ingestion/connectors/database-services/snowflake.md @@ -94,13 +94,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/database-services/trino.md b/docs/install/metadata-ingestion/connectors/database-services/trino.md index e33b1d3e775..fd0d1de90be 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/trino.md +++ b/docs/install/metadata-ingestion/connectors/database-services/trino.md @@ -82,13 +82,6 @@ add `metadata-rest-tables` sink along with `metadata-server` config "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/docs/install/metadata-ingestion/connectors/database-services/vertica.md b/docs/install/metadata-ingestion/connectors/database-services/vertica.md index aeababb6fc9..cc1db54cdbd 100644 --- a/docs/install/metadata-ingestion/connectors/database-services/vertica.md +++ b/docs/install/metadata-ingestion/connectors/database-services/vertica.md @@ -85,13 +85,6 @@ Add Optionally `query-parser` processor, `table-usage` stage and`metadata-usage "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/docs/install/metadata-ingestion/connectors/messaging-services/kafka.md b/docs/install/metadata-ingestion/connectors/messaging-services/kafka.md index eaa7e4c6737..cf5551e6997 100644 --- a/docs/install/metadata-ingestion/connectors/messaging-services/kafka.md +++ b/docs/install/metadata-ingestion/connectors/messaging-services/kafka.md @@ -84,13 +84,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } ``` diff --git a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py index 759a10eba74..34a6a733a0d 100644 --- a/ingestion/examples/airflow/dags/airflow_metadata_to_es.py +++ b/ingestion/examples/airflow/dags/airflow_metadata_to_es.py @@ -39,8 +39,7 @@ config = """ def metadata_ingestion_workflow(): config_file = pathlib.Path("/ingestion/pipelines/metadata_to_es.json") workflow_config = load_config_file(config_file) - if workflow_config.get('cron'): - del workflow_config['cron'] + workflow = Workflow.create(workflow_config) workflow.execute() workflow.raise_from_status() diff --git a/ingestion/examples/airflow/dags/airflow_sample_data.py b/ingestion/examples/airflow/dags/airflow_sample_data.py index e16e7bb74c3..94fddd626d8 100644 --- a/ingestion/examples/airflow/dags/airflow_sample_data.py +++ b/ingestion/examples/airflow/dags/airflow_sample_data.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import pathlib +import json from datetime import timedelta from airflow import DAG @@ -24,7 +25,7 @@ except ModuleNotFoundError: from metadata.config.common import load_config_file from metadata.ingestion.api.workflow import Workflow from airflow.utils.dates import days_ago -import json + default_args = { "owner": "user_name", "email": ["username@org.com"], @@ -52,21 +53,12 @@ config = """ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/12", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } """ def metadata_ingestion_workflow(): workflow_config = json.loads(config) - if workflow_config.get('cron'): - del workflow_config['cron'] workflow = Workflow.create(workflow_config) workflow.execute() workflow.raise_from_status() diff --git a/ingestion/examples/airflow/dags/airflow_sample_usage.py b/ingestion/examples/airflow/dags/airflow_sample_usage.py index dfcf70ad14f..a8f59bc18cb 100644 --- a/ingestion/examples/airflow/dags/airflow_sample_usage.py +++ b/ingestion/examples/airflow/dags/airflow_sample_usage.py @@ -68,21 +68,13 @@ config = """ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/6", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } """ def metadata_ingestion_workflow(): workflow_config = json.loads(config) - if workflow_config.get('cron'): - del workflow_config['cron'] + workflow = Workflow.create(workflow_config) workflow.execute() workflow.raise_from_status() diff --git a/ingestion/examples/airflow/dags/airflow_sample_users.py b/ingestion/examples/airflow/dags/airflow_sample_users.py index b959843e134..291dfc6fa4c 100644 --- a/ingestion/examples/airflow/dags/airflow_sample_users.py +++ b/ingestion/examples/airflow/dags/airflow_sample_users.py @@ -54,20 +54,12 @@ config = """ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": null, - "hour": null, - "day": "*/7", - "month": "*", - "day_of_week": "*" } } """ def metadata_ingestion_workflow(): workflow_config = json.loads(config) - if workflow_config.get('cron'): - del workflow_config['cron'] + workflow = Workflow.create(workflow_config) workflow.execute() workflow.raise_from_status() diff --git a/ingestion/examples/airflow/sample_tables_airflow_example.py b/ingestion/examples/airflow/sample_tables_airflow_example.py index 51d0ee7aa91..700964e93c8 100644 --- a/ingestion/examples/airflow/sample_tables_airflow_example.py +++ b/ingestion/examples/airflow/sample_tables_airflow_example.py @@ -38,8 +38,7 @@ default_args = { def metadata_ingestion_workflow(): config_file = pathlib.Path("/tmp/sample_data.json") workflow_config = load_config_file(config_file) - if workflow_config.get('cron'): - del workflow_config['cron'] + workflow = Workflow.create(workflow_config) workflow.execute() workflow.raise_from_status() diff --git a/ingestion/examples/workflows/bigquery.json b/ingestion/examples/workflows/bigquery.json index 596689cd439..9f2ebf7b855 100644 --- a/ingestion/examples/workflows/bigquery.json +++ b/ingestion/examples/workflows/bigquery.json @@ -30,12 +30,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/bigquery_usage.json b/ingestion/examples/workflows/bigquery_usage.json index db9ad767cae..f439a23e448 100644 --- a/ingestion/examples/workflows/bigquery_usage.json +++ b/ingestion/examples/workflows/bigquery_usage.json @@ -36,12 +36,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/confluent_kafka.json b/ingestion/examples/workflows/confluent_kafka.json index 4066f444a3e..4acc7e0e182 100644 --- a/ingestion/examples/workflows/confluent_kafka.json +++ b/ingestion/examples/workflows/confluent_kafka.json @@ -21,12 +21,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/hive.json b/ingestion/examples/workflows/hive.json index 52b4786192b..62fbd6e8af2 100644 --- a/ingestion/examples/workflows/hive.json +++ b/ingestion/examples/workflows/hive.json @@ -16,12 +16,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/ldap_user_to_catalog.json b/ingestion/examples/workflows/ldap_user_to_catalog.json index 3f24fe3b0bb..d49ffa785cd 100644 --- a/ingestion/examples/workflows/ldap_user_to_catalog.json +++ b/ingestion/examples/workflows/ldap_user_to_catalog.json @@ -19,12 +19,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/looker.json b/ingestion/examples/workflows/looker.json index 4307a4588b2..f49ac4dec00 100644 --- a/ingestion/examples/workflows/looker.json +++ b/ingestion/examples/workflows/looker.json @@ -19,12 +19,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/mssql.json b/ingestion/examples/workflows/mssql.json index 6d1764ebf1a..94ec847da66 100644 --- a/ingestion/examples/workflows/mssql.json +++ b/ingestion/examples/workflows/mssql.json @@ -23,12 +23,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/mysql.json b/ingestion/examples/workflows/mysql.json index bf6ac893f69..934ae5ea10a 100644 --- a/ingestion/examples/workflows/mysql.json +++ b/ingestion/examples/workflows/mysql.json @@ -21,12 +21,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/postgres.json b/ingestion/examples/workflows/postgres.json index 8b66bee9c85..66c020cb215 100644 --- a/ingestion/examples/workflows/postgres.json +++ b/ingestion/examples/workflows/postgres.json @@ -19,12 +19,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/presto.json b/ingestion/examples/workflows/presto.json index 9475fe4489d..8a74cbe3f0e 100644 --- a/ingestion/examples/workflows/presto.json +++ b/ingestion/examples/workflows/presto.json @@ -18,12 +18,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/redash.json b/ingestion/examples/workflows/redash.json index 88e64681f63..849c7a41c04 100644 --- a/ingestion/examples/workflows/redash.json +++ b/ingestion/examples/workflows/redash.json @@ -17,12 +17,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } \ No newline at end of file diff --git a/ingestion/examples/workflows/redshift.json b/ingestion/examples/workflows/redshift.json index 1b7ebacb0aa..2894a88546b 100644 --- a/ingestion/examples/workflows/redshift.json +++ b/ingestion/examples/workflows/redshift.json @@ -22,12 +22,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } \ No newline at end of file diff --git a/ingestion/examples/workflows/redshift_usage.json b/ingestion/examples/workflows/redshift_usage.json index 6bea3c61029..e2b33d915b3 100644 --- a/ingestion/examples/workflows/redshift_usage.json +++ b/ingestion/examples/workflows/redshift_usage.json @@ -35,12 +35,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/salesforce.json b/ingestion/examples/workflows/salesforce.json index 9c0dabd5ec4..dbee7cec3e0 100644 --- a/ingestion/examples/workflows/salesforce.json +++ b/ingestion/examples/workflows/salesforce.json @@ -20,12 +20,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/sample_entity.json b/ingestion/examples/workflows/sample_entity.json index a3f18de5d62..a211e98e5c2 100644 --- a/ingestion/examples/workflows/sample_entity.json +++ b/ingestion/examples/workflows/sample_entity.json @@ -29,12 +29,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/snowflake.json b/ingestion/examples/workflows/snowflake.json index 843d42c17d8..46cfc02cdef 100644 --- a/ingestion/examples/workflows/snowflake.json +++ b/ingestion/examples/workflows/snowflake.json @@ -26,12 +26,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/snowflake_usage.json b/ingestion/examples/workflows/snowflake_usage.json index 9c5b5ac3aaf..285df234e5e 100644 --- a/ingestion/examples/workflows/snowflake_usage.json +++ b/ingestion/examples/workflows/snowflake_usage.json @@ -35,12 +35,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/superset.json b/ingestion/examples/workflows/superset.json index 64ec011323c..8e7ed34449f 100644 --- a/ingestion/examples/workflows/superset.json +++ b/ingestion/examples/workflows/superset.json @@ -19,12 +19,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/tableau.json b/ingestion/examples/workflows/tableau.json index 89ced0654c3..03b071aba6f 100644 --- a/ingestion/examples/workflows/tableau.json +++ b/ingestion/examples/workflows/tableau.json @@ -22,12 +22,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/trino.json b/ingestion/examples/workflows/trino.json index 989feb1c780..382c31051fb 100644 --- a/ingestion/examples/workflows/trino.json +++ b/ingestion/examples/workflows/trino.json @@ -18,12 +18,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/examples/workflows/vertica.json b/ingestion/examples/workflows/vertica.json index eca76f6a2cc..c0078b7e54c 100644 --- a/ingestion/examples/workflows/vertica.json +++ b/ingestion/examples/workflows/vertica.json @@ -21,12 +21,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/pipelines/metadata_to_es.json b/ingestion/pipelines/metadata_to_es.json index 987221594ac..a95361a838f 100644 --- a/ingestion/pipelines/metadata_to_es.json +++ b/ingestion/pipelines/metadata_to_es.json @@ -24,12 +24,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/pipelines/sample_data.json b/ingestion/pipelines/sample_data.json index ffd3993b33a..8727f7528f5 100644 --- a/ingestion/pipelines/sample_data.json +++ b/ingestion/pipelines/sample_data.json @@ -15,12 +15,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/12", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/pipelines/sample_usage.json b/ingestion/pipelines/sample_usage.json index fe5fc0af3ec..cc98bd45b48 100644 --- a/ingestion/pipelines/sample_usage.json +++ b/ingestion/pipelines/sample_usage.json @@ -31,12 +31,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/6", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/pipelines/sample_users.json b/ingestion/pipelines/sample_users.json index 3fd692fe7f9..21597e01fa3 100644 --- a/ingestion/pipelines/sample_users.json +++ b/ingestion/pipelines/sample_users.json @@ -16,12 +16,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": null, - "hour": null, - "day": "*/7", - "month": "*", - "day_of_week": "*" } } diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index e754af4433b..5565efcc8de 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -66,8 +66,6 @@ def ingest(config: str) -> None: try: logger.info(f"Using config: {workflow_config}") - if workflow_config.get("cron"): - del workflow_config["cron"] workflow = Workflow.create(workflow_config) except ValidationError as e: click.echo(e, err=True) @@ -95,8 +93,6 @@ def report(config: str) -> None: try: logger.info(f"Using config: {workflow_config}") - if workflow_config.get("cron"): - del workflow_config["cron"] if workflow_config.get("sink"): del workflow_config["sink"] workflow_config["sink"] = file_sink diff --git a/ingestion/tests/unit/mysql_test.json b/ingestion/tests/unit/mysql_test.json index e20fec8f9fc..0cbbeed2192 100644 --- a/ingestion/tests/unit/mysql_test.json +++ b/ingestion/tests/unit/mysql_test.json @@ -23,12 +23,5 @@ "api_endpoint": "http://localhost:8585/api", "auth_provider_type": "no-auth" } - }, - "cron": { - "minute": "*/5", - "hour": null, - "day": null, - "month": null, - "day_of_week": null } } diff --git a/ingestion/tests/unit/workflow_test.py b/ingestion/tests/unit/workflow_test.py index ff14ba58312..3f3286bfc98 100644 --- a/ingestion/tests/unit/workflow_test.py +++ b/ingestion/tests/unit/workflow_test.py @@ -47,8 +47,6 @@ class WorkflowTest(TestCase): def test_execute_200(self): config_file = pathlib.Path('tests/unit/mysql_test.json') workflow_config = load_config_file(config_file) - if workflow_config.get('cron'): - del workflow_config['cron'] workflow = Workflow.create(workflow_config) workflow.execute() workflow.stop()