yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 01:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removing Cron from all configs (#773)

Browse Source 
* Removing Cron from all configs

* Removing Cron from Documentation
This commit is contained in:
Ayush Shah 2021-10-15 03:51:47 +05:30 committed by GitHub
parent 5b22809be6
commit 72e4b8831f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 6 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/install/metadata-ingestion/connectors/dashboard-services/looker.md
View File

@ -82,13 +82,6 @@ Add Optionally`pii` processor and `metadata-rest-tables` sink along with `metada
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/dashboard-services/superset.md
View File

@ -82,13 +82,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/dashboard-services/tableau.md
View File

@ -92,13 +92,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/athena.md
View File

@ -83,13 +83,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/bigquery.md
View File

@ -121,13 +121,6 @@ Add Optionally`pii` processor and `metadata-rest-tables` sink along with `metada
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/elastic-search.md
View File

@ -77,13 +77,6 @@ Add Optionally `file` stage and `elasticsearch` bulk_sink along with `metadata-s
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/hive.md
View File

@ -69,13 +69,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/mssql.md
View File

@ -92,13 +92,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/mysql.md
View File

@ -86,13 +86,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/oracle.md
View File

@ -83,13 +83,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/postgres.md
View File

@ -83,13 +83,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/presto.md
View File

@ -76,13 +76,6 @@ add `metadata-rest-tables` sink along with `metadata-server` config
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/redshift-usage.md
View File

@ -100,13 +100,6 @@ Add optionally`query-parser` processor, `table-usage` stage and `metadata-usage`
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/salesforce.md
View File

@ -86,13 +86,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/snowflake-usage.md
View File

@ -100,13 +100,6 @@ Add Optionally `query-parser` processor, `table-usage` stage and`metadata-usage`
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/snowflake.md
View File

@ -94,13 +94,6 @@ Add Optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/database-services/trino.md
View File

@ -82,13 +82,6 @@ add `metadata-rest-tables` sink along with `metadata-server` config
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

7
docs/install/metadata-ingestion/connectors/database-services/vertica.md
View File

@ -85,13 +85,6 @@ Add Optionally `query-parser` processor, `table-usage` stage and`metadata-usage
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
docs/install/metadata-ingestion/connectors/messaging-services/kafka.md
View File

@ -84,13 +84,6 @@ Add optionally `pii` processor and `metadata-rest-tables` sink along with `metad
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
```

3
ingestion/examples/airflow/dags/airflow_metadata_to_es.py
View File

@ -39,8 +39,7 @@ config = """
def metadata_ingestion_workflow():
config_file = pathlib.Path("/ingestion/pipelines/metadata_to_es.json")
workflow_config = load_config_file(config_file)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()

12
ingestion/examples/airflow/dags/airflow_sample_data.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import pathlib
import json
from datetime import timedelta
from airflow import DAG
@ -24,7 +25,7 @@ except ModuleNotFoundError:
from metadata.config.common import load_config_file
from metadata.ingestion.api.workflow import Workflow
from airflow.utils.dates import days_ago
import json
default_args = {
"owner": "user_name",
"email": ["username@org.com"],
@ -52,21 +53,12 @@ config = """
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/12",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
"""
def metadata_ingestion_workflow():
workflow_config = json.loads(config)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()

10
ingestion/examples/airflow/dags/airflow_sample_usage.py
View File

@ -68,21 +68,13 @@ config = """
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/6",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}
"""
def metadata_ingestion_workflow():
workflow_config = json.loads(config)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()

10
ingestion/examples/airflow/dags/airflow_sample_users.py
View File

@ -54,20 +54,12 @@ config = """
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": null,
"hour": null,
"day": "*/7",
"month": "*",
"day_of_week": "*"
}
}
"""
def metadata_ingestion_workflow():
workflow_config = json.loads(config)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()

3
ingestion/examples/airflow/sample_tables_airflow_example.py
View File

@ -38,8 +38,7 @@ default_args = {
def metadata_ingestion_workflow():
config_file = pathlib.Path("/tmp/sample_data.json")
workflow_config = load_config_file(config_file)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()

7
ingestion/examples/workflows/bigquery.json
View File

@ -30,12 +30,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/bigquery_usage.json
View File

@ -36,12 +36,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/confluent_kafka.json
View File

@ -21,12 +21,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/hive.json
View File

@ -16,12 +16,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/ldap_user_to_catalog.json
View File

@ -19,12 +19,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/looker.json
View File

@ -19,12 +19,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/mssql.json
View File

@ -23,12 +23,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/mysql.json
View File

@ -21,12 +21,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/postgres.json
View File

@ -19,12 +19,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/presto.json
View File

@ -18,12 +18,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/redash.json
View File

@ -17,12 +17,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/redshift.json
View File

@ -22,12 +22,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/redshift_usage.json
View File

@ -35,12 +35,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/salesforce.json
View File

@ -20,12 +20,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/sample_entity.json
View File

@ -29,12 +29,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/snowflake.json
View File

@ -26,12 +26,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/snowflake_usage.json
View File

@ -35,12 +35,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/superset.json
View File

@ -19,12 +19,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/tableau.json
View File

@ -22,12 +22,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/trino.json
View File

@ -18,12 +18,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/examples/workflows/vertica.json
View File

@ -21,12 +21,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/pipelines/metadata_to_es.json
View File

@ -24,12 +24,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/pipelines/sample_data.json
View File

@ -15,12 +15,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/12",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/pipelines/sample_usage.json
View File

@ -31,12 +31,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/6",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

7
ingestion/pipelines/sample_users.json
View File

@ -16,12 +16,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": null,
"hour": null,
"day": "*/7",
"month": "*",
"day_of_week": "*"
}
}

4
ingestion/src/metadata/cmd.py
View File

@ -66,8 +66,6 @@ def ingest(config: str) -> None:
try:
logger.info(f"Using config: {workflow_config}")
if workflow_config.get("cron"):
del workflow_config["cron"]
workflow = Workflow.create(workflow_config)
except ValidationError as e:
click.echo(e, err=True)
@ -95,8 +93,6 @@ def report(config: str) -> None:
try:
logger.info(f"Using config: {workflow_config}")
if workflow_config.get("cron"):
del workflow_config["cron"]
if workflow_config.get("sink"):
del workflow_config["sink"]
workflow_config["sink"] = file_sink

7
ingestion/tests/unit/mysql_test.json
View File

@ -23,12 +23,5 @@
"api_endpoint": "http://localhost:8585/api",
"auth_provider_type": "no-auth"
}
},
"cron": {
"minute": "*/5",
"hour": null,
"day": null,
"month": null,
"day_of_week": null
}
}

2
ingestion/tests/unit/workflow_test.py
View File

@ -47,8 +47,6 @@ class WorkflowTest(TestCase):
def test_execute_200(self):
config_file = pathlib.Path('tests/unit/mysql_test.json')
workflow_config = load_config_file(config_file)
if workflow_config.get('cron'):
del workflow_config['cron']
workflow = Workflow.create(workflow_config)
workflow.execute()
workflow.stop()