2024-06-11 11:44:13 -07:00
|
|
|
# Ingest data from S3
|
2022-05-02 00:18:15 -07:00
|
|
|
source:
|
|
|
|
|
type: s3
|
|
|
|
|
config:
|
2022-05-05 19:09:47 +02:00
|
|
|
path_specs:
|
2024-06-11 11:44:13 -07:00
|
|
|
- include: "s3://covid19-lake/covid_knowledge_graph/csv/nodes/*.*"
|
2022-05-05 19:09:47 +02:00
|
|
|
|
2022-05-02 00:18:15 -07:00
|
|
|
aws_config:
|
|
|
|
|
aws_access_key_id: *****
|
|
|
|
|
aws_secret_access_key: *****
|
|
|
|
|
aws_region: us-east-2
|
|
|
|
|
env: "PROD"
|
|
|
|
|
profiling:
|
|
|
|
|
enabled: false
|
|
|
|
|
|
2024-06-11 11:44:13 -07:00
|
|
|
# Ingest data from local filesystem
|
|
|
|
|
source:
|
|
|
|
|
type: s3
|
|
|
|
|
config:
|
|
|
|
|
path_specs:
|
|
|
|
|
- include: "/absolute/path/*.csv"
|
