yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-19 06:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/s3): fix test flakiness (#8416)

Browse Source
This commit is contained in:
Tamas Nemeth 2023-07-14 00:42:00 +02:00 committed by GitHub
parent 83ebeb207d
commit a91c78cf31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 42 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json
View File

@ -127,7 +127,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419674000
"lastUpdatedTimestamp": 1586848010000
}
},
"systemMetadata": {

8
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json
View File

@ -7,7 +7,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv"
},
"name": "food_csv",
"description": "",
@ -125,7 +125,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419673000
"lastUpdatedTimestamp": 1586847990000
}
},
"systemMetadata": {
@ -588,7 +588,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet"
},
"name": "food_parquet",
"description": "",
@ -706,7 +706,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419671000
"lastUpdatedTimestamp": 1586847810000
}
},
"systemMetadata": {

4
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json
View File

@ -7,7 +7,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet"
},
"name": "food_parquet",
"description": "",
@ -125,7 +125,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419671000
"lastUpdatedTimestamp": 1586847810000
}
},
"systemMetadata": {

8
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json
View File

@ -7,7 +7,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv"
},
"name": "folder_aaa.food_csv",
"description": "",
@ -125,7 +125,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419673000
"lastUpdatedTimestamp": 1586847990000
}
},
"systemMetadata": {
@ -588,7 +588,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet"
},
"name": "folder_aaa.food_parquet",
"description": "",
@ -706,7 +706,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419671000
"lastUpdatedTimestamp": 1586847810000
}
},
"systemMetadata": {

4
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json
View File

@ -7,7 +7,7 @@
"aspect": {
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet"
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet"
},
"name": "food_parquet",
"description": "",
@ -125,7 +125,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419671000
"lastUpdatedTimestamp": 1586847810000
}
},
"systemMetadata": {

2
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json
View File

@ -557,7 +557,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419673000
"lastUpdatedTimestamp": 1586847960000
}
},
"systemMetadata": {

2
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json
View File

@ -557,7 +557,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419673000
"lastUpdatedTimestamp": 1586847960000
}
},
"systemMetadata": {

2
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json
View File

@ -557,7 +557,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419673000
"lastUpdatedTimestamp": 1586847960000
}
},
"systemMetadata": {

14
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json
View File

@ -439,7 +439,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419669000
"lastUpdatedTimestamp": 1586847610000
}
},
"systemMetadata": {
@ -1030,7 +1030,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847620000
}
},
"systemMetadata": {
@ -1213,7 +1213,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847630000
}
},
"systemMetadata": {
@ -1372,7 +1372,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847640000
}
},
"systemMetadata": {
@ -1555,7 +1555,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847650000
}
},
"systemMetadata": {
@ -1738,7 +1738,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847660000
}
},
"systemMetadata": {
@ -2233,7 +2233,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847670000
}
},
"systemMetadata": {

4
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json
View File

@ -149,7 +149,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847620000
}
},
"systemMetadata": {
@ -754,7 +754,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847630000
}
},
"systemMetadata": {

4
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json
View File

@ -149,7 +149,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847620000
}
},
"systemMetadata": {
@ -754,7 +754,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419674000
"lastUpdatedTimestamp": 1586847630000
}
},
"systemMetadata": {

2
metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json
View File

@ -149,7 +149,7 @@
"partition": "FULL_TABLE_SNAPSHOT"
},
"operationType": "UPDATE",
"lastUpdatedTimestamp": 1688419668000
"lastUpdatedTimestamp": 1586847620000
}
},
"systemMetadata": {

15
metadata-ingestion/tests/integration/s3/test_s3.py
View File

@ -3,6 +3,7 @@ import logging
import os
from datetime import datetime
import moto.s3
import pytest
from boto3.session import Session
from moto import mock_s3
@ -55,6 +56,10 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names):
test_resources_dir = (
pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/"
)
current_time_sec = datetime.strptime(
FROZEN_TIME, "%Y-%m-%d %H:%M:%S"
).timestamp()
for root, _dirs, files in os.walk(test_resources_dir):
for file in sorted(files):
full_path = os.path.join(root, file)
@ -65,6 +70,13 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names):
Key=rel_path,
Tagging={"TagSet": [{"Key": "baz", "Value": "bob"}]},
)
key = (
moto.s3.models.s3_backends["123456789012"]["global"]
.buckets[bucket_name]
.keys[rel_path]
)
current_time_sec += 10
key.last_modified = datetime.fromtimestamp(current_time_sec)
yield
@ -73,7 +85,8 @@ def touch_local_files(pytestconfig):
test_resources_dir = (
pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/"
)
current_time_sec = datetime.now().timestamp()
current_time_sec = datetime.strptime(FROZEN_TIME, "%Y-%m-%d %H:%M:%S").timestamp()
for root, _dirs, files in os.walk(test_resources_dir):
_dirs.sort()
for file in sorted(files):