diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json index 5518e93600..cb7ccf87f7 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_file_without_extension.json @@ -127,7 +127,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419674000 + "lastUpdatedTimestamp": 1586848010000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json index 1b2ef998bd..0eefc4cafb 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" }, "name": "food_csv", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { @@ -588,7 +588,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -706,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json index fe50a19546..36f40cc918 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json index 6a1b339b2e..7a59a7f21f 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part1.csv" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv/part2.csv" }, "name": "folder_aaa.food_csv", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847990000 } }, "systemMetadata": { @@ -588,7 +588,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "folder_aaa.food_parquet", "description": "", @@ -706,7 +706,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json index 071ef3efff..0dc71b900b 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -7,7 +7,7 @@ "aspect": { "json": { "customProperties": { - "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet" + "schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet" }, "name": "food_parquet", "description": "", @@ -125,7 +125,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419671000 + "lastUpdatedTimestamp": 1586847810000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json index 188369c92d..aa683a2bcd 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json index 5ab5f043e9..095c1cbe31 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json index 6e42b68f1f..a5f135c303 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json @@ -557,7 +557,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419673000 + "lastUpdatedTimestamp": 1586847960000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json index 01ff0030e3..40e45150fa 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -439,7 +439,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419669000 + "lastUpdatedTimestamp": 1586847610000 } }, "systemMetadata": { @@ -1030,7 +1030,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -1213,7 +1213,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { @@ -1372,7 +1372,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847640000 } }, "systemMetadata": { @@ -1555,7 +1555,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847650000 } }, "systemMetadata": { @@ -1738,7 +1738,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847660000 } }, "systemMetadata": { @@ -2233,7 +2233,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847670000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json index 26d1636eae..84ace7d673 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_spec_for_files.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -754,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json index d7a831e949..f7f3cb8fb7 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_specs_of_different_buckets.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { @@ -754,7 +754,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419674000 + "lastUpdatedTimestamp": 1586847630000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json index caf989fa37..5353d95ada 100644 --- a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -149,7 +149,7 @@ "partition": "FULL_TABLE_SNAPSHOT" }, "operationType": "UPDATE", - "lastUpdatedTimestamp": 1688419668000 + "lastUpdatedTimestamp": 1586847620000 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/s3/test_s3.py b/metadata-ingestion/tests/integration/s3/test_s3.py index 6202fa5509..98ae2eaa39 100644 --- a/metadata-ingestion/tests/integration/s3/test_s3.py +++ b/metadata-ingestion/tests/integration/s3/test_s3.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime +import moto.s3 import pytest from boto3.session import Session from moto import mock_s3 @@ -55,6 +56,10 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): test_resources_dir = ( pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/" ) + + current_time_sec = datetime.strptime( + FROZEN_TIME, "%Y-%m-%d %H:%M:%S" + ).timestamp() for root, _dirs, files in os.walk(test_resources_dir): for file in sorted(files): full_path = os.path.join(root, file) @@ -65,6 +70,13 @@ def s3_populate(pytestconfig, s3_resource, s3_client, bucket_names): Key=rel_path, Tagging={"TagSet": [{"Key": "baz", "Value": "bob"}]}, ) + key = ( + moto.s3.models.s3_backends["123456789012"]["global"] + .buckets[bucket_name] + .keys[rel_path] + ) + current_time_sec += 10 + key.last_modified = datetime.fromtimestamp(current_time_sec) yield @@ -73,7 +85,8 @@ def touch_local_files(pytestconfig): test_resources_dir = ( pytestconfig.rootpath / "tests/integration/s3/test_data/local_system/" ) - current_time_sec = datetime.now().timestamp() + current_time_sec = datetime.strptime(FROZEN_TIME, "%Y-%m-%d %H:%M:%S").timestamp() + for root, _dirs, files in os.walk(test_resources_dir): _dirs.sort() for file in sorted(files):