diff --git a/ingestion/tests/integration/ometa/test_ometa_table_api.py b/ingestion/tests/integration/ometa/test_ometa_table_api.py index 8dc184741a9..ddb5ac9288a 100644 --- a/ingestion/tests/integration/ometa/test_ometa_table_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_table_api.py @@ -375,7 +375,7 @@ class OMetaTableTest(TestCase): ) table_profile = TableProfile( - timestamp=Timestamp(int(datetime.now().timestamp())), + timestamp=Timestamp(int(datetime.now().timestamp() * 1000)), columnCount=1.0, rowCount=3.0, ) @@ -389,18 +389,18 @@ class OMetaTableTest(TestCase): mean=1.5, sum=2, stddev=None, - timestamp=Timestamp(root=int(datetime.now().timestamp())), + timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000)), ) ] system_profile = [ SystemProfile( - timestamp=Timestamp(root=int(datetime.now().timestamp())), + timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000)), operation=DmlOperationType.INSERT, rowsAffected=11, ), SystemProfile( - timestamp=Timestamp(root=int(datetime.now().timestamp()) + 1), + timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000) + 1), operation=DmlOperationType.UPDATE, rowsAffected=110, ), diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 5a2a00b1349..c15c18ad2c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -395,6 +395,7 @@ public class TableRepository extends EntityRepository { public Table addTableProfileData(UUID tableId, CreateTableProfile createTableProfile) { // Validate the request content Table table = find(tableId, NON_DELETED); + validateProfilerTimestamps(createTableProfile); daoCollection .profilerDataTimeSeriesDao() .insert( @@ -448,6 +449,26 @@ public class TableRepository extends EntityRepository
{ return table.withProfile(createTableProfile.getTableProfile()); } + private void validateProfilerTimestamps(CreateTableProfile createTableProfile) { + if (createTableProfile.getTableProfile() != null) { + RestUtil.validateTimestampMilliseconds(createTableProfile.getTableProfile().getTimestamp()); + } + if (createTableProfile.getColumnProfile() != null) { + createTableProfile + .getColumnProfile() + .forEach( + columnProfile -> + RestUtil.validateTimestampMilliseconds(columnProfile.getTimestamp())); + } + if (createTableProfile.getSystemProfile() != null) { + createTableProfile + .getSystemProfile() + .forEach( + systemProfile -> + RestUtil.validateTimestampMilliseconds(systemProfile.getTimestamp())); + } + } + public void deleteTableProfile(String fqn, String entityType, Long timestamp) { // Validate the request content String extension; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index dd6c2264428..4fc3642b2d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -913,6 +913,7 @@ public class TestCaseResource extends EntityResource= 12; + if (!isMilliseconds) { + throw new BadRequestException( + String.format( + "Timestamp %s is not valid, it should be in milliseconds since epoch", timestamp)); + } + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 330976c1872..a1ea2989e05 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -136,6 +136,7 @@ import org.openmetadata.schema.type.JoinedWith; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.PartitionColumnDetails; import org.openmetadata.schema.type.PartitionIntervalTypes; +import org.openmetadata.schema.type.SystemProfile; import org.openmetadata.schema.type.TableConstraint; import org.openmetadata.schema.type.TableConstraint.ConstraintType; import org.openmetadata.schema.type.TableData; @@ -1236,6 +1237,47 @@ public class TableResourceTest extends EntityResourceTest { permissionNotAllowed(USER2.getName(), List.of(MetadataOperation.EDIT_DATA_PROFILE))); } + @Test + void create_profilerWrongTimestamp(TestInfo testInfo) throws IOException, ParseException { + Table table = createEntity(createRequest(testInfo), ADMIN_AUTH_HEADERS); + Long correctTimestamp = 1725525388000L; + Long wrongTimestamp = 1725525388L; + + ColumnProfile c1Profile = getColumnProfile(C1, 100.0, 10.0, 100.0, wrongTimestamp); + ColumnProfile c2Profile = getColumnProfile(C2, 99.0, 20.0, 89.0, correctTimestamp); + ColumnProfile c3Profile = getColumnProfile(C3, 75.0, 25.0, 77.0, correctTimestamp); + List columnProfiles = List.of(c1Profile, c2Profile, c3Profile); + TableProfile tableProfile = + new TableProfile() + .withRowCount(6.0) + .withColumnCount(3.0) + .withTimestamp(correctTimestamp) + .withProfileSample(10.0); + + CreateTableProfile createTableProfile = + new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles); + assertResponse( + () -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Timestamp 1725525388 is not valid, it should be in milliseconds since epoch"); + + tableProfile = tableProfile.withTimestamp(wrongTimestamp); + c1Profile = c1Profile.withTimestamp(correctTimestamp); + columnProfiles = List.of(c1Profile, c2Profile, c3Profile); + createTableProfile.withTableProfile(tableProfile).withColumnProfile(columnProfiles); + assertResponse( + () -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Timestamp 1725525388 is not valid, it should be in milliseconds since epoch"); + SystemProfile systemProfile = new SystemProfile().withTimestamp(wrongTimestamp); + tableProfile = tableProfile.withTimestamp(correctTimestamp); + createTableProfile.withTableProfile(tableProfile).withSystemProfile(listOf(systemProfile)); + assertResponse( + () -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Timestamp 1725525388 is not valid, it should be in milliseconds since epoch"); + } + void putTableProfile(Table table, Table table1, Map authHeaders) throws IOException, ParseException { Long timestamp = TestUtils.dateToTimestamp("2021-09-09"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index a751bbfae63..792c9abd800 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -2708,6 +2708,22 @@ public class TestCaseResourceTest extends EntityResourceTest + putTestCaseResult(testCase.getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Timestamp 1725521153 is not valid, it should be in milliseconds since epoch"); + } + private void putInspectionQuery(TestCase testCase, String sql, Map authHeaders) throws IOException { TestCase putResponse = putInspectionQuery(testCase.getId(), sql, authHeaders);