mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-03 20:27:50 +00:00
fix(ci): ingest - fix nifi integration tests (#5718)
This commit is contained in:
Ravindra Lanka
GitHub
parent
d64cca5c6b
commit
2a56d2c276
@ -102,7 +102,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-01-15 03:06:13.701000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=c5f6fc66-ffbb-3f60-9564-f2466ae32493\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:45.343000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=c5f6fc66-ffbb-3f60-9564-f2466ae32493\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
@ -132,6 +132,44 @@
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataPlatformInstance",
|
||||
"aspect": {
|
||||
"value": "{\"platform\": \"urn:li:dataPlatform:s3\"}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetProperties",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataJob",
|
||||
@ -140,7 +178,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-01-15 03:06:11.397000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8a218b6e-e6a0-36b6-bc4b-79d202a80167\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-08-23 19:44:39.852000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=1c61a8d9-3462-387f-8145-09e6e7785e5c&componentIds=8a218b6e-e6a0-36b6-bc4b-79d202a80167\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
@ -292,7 +330,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"listing-strategy\\\": \\\"timestamps\\\", \\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote Path\\\": \\\".\\\", \\\"record-writer\\\": null, \\\"Distributed Cache Service\\\": null, \\\"Search Recursively\\\": \\\"true\\\", \\\"follow-symlink\\\": \\\"false\\\", \\\"File Filter Regex\\\": null, \\\"Path Filter Regex\\\": null, \\\"Ignore Dotted Files\\\": \\\"true\\\", \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Host Key File\\\": null, \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"target-system-timestamp-precision\\\": \\\"auto-detect\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"et-state-cache\\\": null, \\\"et-time-window\\\": \\\"3 hours\\\", \\\"et-initial-listing-target\\\": \\\"all\\\", \\\"Minimum File Age\\\": \\\"0 sec\\\", \\\"Maximum File Age\\\": null, \\\"Minimum File Size\\\": \\\"0 B\\\", \\\"Maximum File Size\\\": null, \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-01-15 03:06:03.851000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb5263d-017d-1000-ffff-ffff911b23aa\", \"name\": \"ListSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"listing-strategy\\\": \\\"timestamps\\\", \\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote Path\\\": \\\".\\\", \\\"record-writer\\\": null, \\\"Distributed Cache Service\\\": null, \\\"Search Recursively\\\": \\\"true\\\", \\\"follow-symlink\\\": \\\"false\\\", \\\"File Filter Regex\\\": null, \\\"Path Filter Regex\\\": null, \\\"Ignore Dotted Files\\\": \\\"true\\\", \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Host Key File\\\": null, \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"target-system-timestamp-precision\\\": \\\"auto-detect\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"et-state-cache\\\": null, \\\"et-time-window\\\": \\\"3 hours\\\", \\\"et-initial-listing-target\\\": \\\"all\\\", \\\"Minimum File Age\\\": \\\"0 sec\\\", \\\"Maximum File Age\\\": null, \\\"Minimum File Size\\\": \\\"0 B\\\", \\\"Maximum File Size\\\": null, \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:15.866000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb5263d-017d-1000-ffff-ffff911b23aa\", \"name\": \"ListSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
@ -360,6 +398,82 @@
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataPlatformInstance",
|
||||
"aspect": {
|
||||
"value": "{\"platform\": \"urn:li:dataPlatform:file\"}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host.temperature,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetProperties",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host/temperature\"}, \"tags\": []}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataPlatformInstance",
|
||||
"aspect": {
|
||||
"value": "{\"platform\": \"urn:li:dataPlatform:file\"}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,sftp_public_host,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetProperties",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"uri\": \"sftp://sftp_public_host\"}, \"tags\": []}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-cluster",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataJob",
|
||||
@ -368,7 +482,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote File\\\": \\\"${path}/${filename}\\\", \\\"Completion Strategy\\\": \\\"None\\\", \\\"Move Destination Directory\\\": null, \\\"Create Directory\\\": \\\"false\\\", \\\"Disable Directory Listing\\\": \\\"false\\\", \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"Host Key File\\\": null, \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Use Compression\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"fetchfiletransfer-notfound-loglevel\\\": \\\"ERROR\\\", \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-01-15 03:06:07.830000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb55aeb-017d-1000-ffff-fffff475768d\", \"name\": \"FetchSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Hostname\\\": \\\"${sftp.host}\\\", \\\"Port\\\": \\\"22\\\", \\\"Username\\\": \\\"${sftp.username}\\\", \\\"Password\\\": \\\"********\\\", \\\"Private Key Path\\\": null, \\\"Private Key Passphrase\\\": null, \\\"Remote File\\\": \\\"${path}/${filename}\\\", \\\"Completion Strategy\\\": \\\"None\\\", \\\"Move Destination Directory\\\": null, \\\"Create Directory\\\": \\\"false\\\", \\\"Disable Directory Listing\\\": \\\"false\\\", \\\"Connection Timeout\\\": \\\"30 sec\\\", \\\"Data Timeout\\\": \\\"30 sec\\\", \\\"Send Keep Alive On Timeout\\\": \\\"true\\\", \\\"Host Key File\\\": null, \\\"Strict Host Key Checking\\\": \\\"false\\\", \\\"Use Compression\\\": \\\"false\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Type\\\": \\\"DIRECT\\\", \\\"Proxy Host\\\": null, \\\"Proxy Port\\\": null, \\\"Http Proxy Username\\\": null, \\\"Http Proxy Password\\\": null, \\\"fetchfiletransfer-notfound-loglevel\\\": \\\"ERROR\\\", \\\"Ciphers Allowed\\\": null, \\\"Key Algorithms Allowed\\\": null, \\\"Key Exchange Algorithms Allowed\\\": null, \\\"Message Authentication Codes Allowed\\\": null}\", \"last_event_time\": \"2022-08-23 19:44:20.256000+00:00\"}, \"externalUrl\": \"http://localhost:9080/nifi/?processGroupId=8eb4f034-017d-1000-ffff-ffffccebd06c&componentIds=8eb55aeb-017d-1000-ffff-fffff475768d\", \"name\": \"FetchSFTP\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
|
||||
@ -102,7 +102,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-01-15 03:05:07.136000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=91d59f03-1c2b-3f3f-48bc-f89296a328bd\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"ALL\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Object Key\\\": \\\"${filename}\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"Version\\\": null, \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"encryption-service\\\": null, \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"requester-pays\\\": \\\"false\\\", \\\"range-start\\\": null, \\\"range-length\\\": null}\", \"last_event_time\": \"2022-08-23 19:43:21.880000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=91d59f03-1c2b-3f3f-48bc-f89296a328bd\", \"name\": \"FetchS3Object\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
@ -132,6 +132,44 @@
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataPlatformInstance",
|
||||
"aspect": {
|
||||
"value": "{\"platform\": \"urn:li:dataPlatform:s3\"}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-standalone",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataset",
|
||||
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)",
|
||||
"entityKeyAspect": null,
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "datasetProperties",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"s3_uri\": \"s3://enriched-topical-chat\"}, \"tags\": []}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
"lastObserved": 1638532800000,
|
||||
"runId": "nifi-test-standalone",
|
||||
"registryName": null,
|
||||
"registryVersion": null,
|
||||
"properties": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"auditHeader": null,
|
||||
"entityType": "dataJob",
|
||||
@ -140,7 +178,7 @@
|
||||
"changeType": "UPSERT",
|
||||
"aspectName": "dataJobInfo",
|
||||
"aspect": {
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-01-15 03:05:01.720000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=cb7693ed-f93b-3340-3776-fe80e6283ddc\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"value": "{\"customProperties\": {\"schedulingPeriod\": \"0 sec\", \"schedulingStrategy\": \"TIMER_DRIVEN\", \"executionNode\": \"PRIMARY\", \"concurrentlySchedulableTaskCount\": \"1\", \"properties\": \"{\\\"Bucket\\\": \\\"enriched-topical-chat\\\", \\\"Region\\\": \\\"us-west-2\\\", \\\"Access Key\\\": null, \\\"Secret Key\\\": null, \\\"record-writer\\\": null, \\\"min-age\\\": \\\"0 sec\\\", \\\"Listing Batch Size\\\": \\\"100\\\", \\\"write-s3-object-tags\\\": \\\"false\\\", \\\"write-s3-user-metadata\\\": \\\"false\\\", \\\"Credentials File\\\": null, \\\"AWS Credentials Provider service\\\": null, \\\"Communications Timeout\\\": \\\"30 secs\\\", \\\"SSL Context Service\\\": null, \\\"Endpoint Override URL\\\": null, \\\"Signer Override\\\": \\\"Default Signature\\\", \\\"proxy-configuration-service\\\": null, \\\"Proxy Host\\\": null, \\\"Proxy Host Port\\\": null, \\\"proxy-user-name\\\": null, \\\"proxy-user-password\\\": null, \\\"delimiter\\\": null, \\\"prefix\\\": null, \\\"use-versions\\\": \\\"false\\\", \\\"list-type\\\": \\\"1\\\", \\\"requester-pays\\\": \\\"false\\\"}\", \"last_event_time\": \"2022-08-23 19:43:17.474000+00:00\"}, \"externalUrl\": \"http://localhost:9443/nifi/?processGroupId=80404c81-017d-1000-e8e8-af7420af06c1&componentIds=cb7693ed-f93b-3340-3776-fe80e6283ddc\", \"name\": \"ListS3\", \"description\": \"\", \"type\": {\"string\": \"NIFI_PROCESSOR\"}}",
|
||||
"contentType": "application/json"
|
||||
},
|
||||
"systemMetadata": {
|
||||
|
||||
@ -80,9 +80,8 @@ def test_nifi_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
|
||||
output_path="nifi_mces.json",
|
||||
golden_path=test_resources_dir / "nifi_mces_golden_standalone.json",
|
||||
ignore_paths=[
|
||||
r"root\[1\]\['aspect'\]\['value'\]",
|
||||
r"root\[5\]\['aspect'\]\['value'\]",
|
||||
r"root\[7\]\['aspect'\]\['value'\]",
|
||||
r"root\[9\]\['aspect'\]\['value'\]",
|
||||
],
|
||||
)
|
||||
|
||||
@ -118,8 +117,8 @@ def test_nifi_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time):
|
||||
golden_path=test_resources_dir / "nifi_mces_golden_cluster.json",
|
||||
ignore_paths=[
|
||||
r"root\[5\]\['aspect'\]\['value'\]",
|
||||
r"root\[7\]\['aspect'\]\['value'\]",
|
||||
r"root\[15\]\['aspect'\]\['value'\]",
|
||||
r"root\[19\]\['aspect'\]\['value'\]",
|
||||
r"root\[9\]\['aspect'\]\['value'\]",
|
||||
r"root\[17\]\['aspect'\]\['value'\]",
|
||||
r"root\[25\]\['aspect'\]\['value'\]",
|
||||
],
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user