yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-16 13:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

GEN-1333 Add TS validation on DQ and Porfiler data ingestion (#17731)

Browse Source 
* fix: added ts validation for DQ and profiler data

* fix: change ts to ms in pytest

* style: ran python linting
This commit is contained in:
Teddy 2024-09-06 08:16:31 +02:00 committed by GitHub
parent 4f6dd7fcc3
commit a7b6279f2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 100 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ingestion/tests/integration/ometa/test_ometa_table_api.py
View File

@ -375,7 +375,7 @@ class OMetaTableTest(TestCase):
)
table_profile = TableProfile(
timestamp=Timestamp(int(datetime.now().timestamp())),
timestamp=Timestamp(int(datetime.now().timestamp() * 1000)),
columnCount=1.0,
rowCount=3.0,
)
@ -389,18 +389,18 @@ class OMetaTableTest(TestCase):
mean=1.5,
sum=2,
stddev=None,
timestamp=Timestamp(root=int(datetime.now().timestamp())),
timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000)),
)
]
system_profile = [
SystemProfile(
timestamp=Timestamp(root=int(datetime.now().timestamp())),
timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000)),
operation=DmlOperationType.INSERT,
rowsAffected=11,
),
SystemProfile(
timestamp=Timestamp(root=int(datetime.now().timestamp()) + 1),
timestamp=Timestamp(root=int(datetime.now().timestamp() * 1000) + 1),
operation=DmlOperationType.UPDATE,
rowsAffected=110,
),

21
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -395,6 +395,7 @@ public class TableRepository extends EntityRepository<Table> {
public Table addTableProfileData(UUID tableId, CreateTableProfile createTableProfile) {
// Validate the request content
Table table = find(tableId, NON_DELETED);
validateProfilerTimestamps(createTableProfile);
daoCollection
.profilerDataTimeSeriesDao()
.insert(
@ -448,6 +449,26 @@ public class TableRepository extends EntityRepository<Table> {
return table.withProfile(createTableProfile.getTableProfile());
}
private void validateProfilerTimestamps(CreateTableProfile createTableProfile) {
if (createTableProfile.getTableProfile() != null) {
RestUtil.validateTimestampMilliseconds(createTableProfile.getTableProfile().getTimestamp());
}
if (createTableProfile.getColumnProfile() != null) {
createTableProfile
.getColumnProfile()
.forEach(
columnProfile ->
RestUtil.validateTimestampMilliseconds(columnProfile.getTimestamp()));
}
if (createTableProfile.getSystemProfile() != null) {
createTableProfile
.getSystemProfile()
.forEach(
systemProfile ->
RestUtil.validateTimestampMilliseconds(systemProfile.getTimestamp()));
}
}
public void deleteTableProfile(String fqn, String entityType, Long timestamp) {
// Validate the request content
String extension;

1
openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
View File

@ -913,6 +913,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
TestCase testCase = repository.findByName(fqn, Include.ALL);
repository.deleteTestCaseFailedRowsSample(testCase.getId());
}
RestUtil.validateTimestampMilliseconds(testCaseResult.getTimestamp());
return repository
.addTestCaseResult(
securityContext.getUserPrincipal().getName(), uriInfo, fqn, testCaseResult)

16
openmetadata-service/src/main/java/org/openmetadata/service/util/RestUtil.java
View File

@ -28,6 +28,7 @@ import java.util.Base64;
import java.util.Date;
import java.util.TimeZone;
import java.util.UUID;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
@ -168,4 +169,19 @@ public final class RestUtil {
return responseBuilder.entity(entity).build();
}
}
public static void validateTimestampMilliseconds(Long timestamp) {
if (timestamp == null) {
throw new IllegalArgumentException("Timestamp is required");
}
// check if timestamp has 12 or more digits
// timestamp ms between 2001-09-09 and 2286-11-20 will have 13 digits
// timestamp ms between 1973-03-03 and 2001-09-09 will have 12 digits
boolean isMilliseconds = String.valueOf(timestamp).length() >= 12;
if (!isMilliseconds) {
throw new BadRequestException(
String.format(
"Timestamp %s is not valid, it should be in milliseconds since epoch", timestamp));
}
}
}

42
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -136,6 +136,7 @@ import org.openmetadata.schema.type.JoinedWith;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.PartitionColumnDetails;
import org.openmetadata.schema.type.PartitionIntervalTypes;
import org.openmetadata.schema.type.SystemProfile;
import org.openmetadata.schema.type.TableConstraint;
import org.openmetadata.schema.type.TableConstraint.ConstraintType;
import org.openmetadata.schema.type.TableData;
@ -1236,6 +1237,47 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
permissionNotAllowed(USER2.getName(), List.of(MetadataOperation.EDIT_DATA_PROFILE)));
}
@Test
void create_profilerWrongTimestamp(TestInfo testInfo) throws IOException, ParseException {
Table table = createEntity(createRequest(testInfo), ADMIN_AUTH_HEADERS);
Long correctTimestamp = 1725525388000L;
Long wrongTimestamp = 1725525388L;
ColumnProfile c1Profile = getColumnProfile(C1, 100.0, 10.0, 100.0, wrongTimestamp);
ColumnProfile c2Profile = getColumnProfile(C2, 99.0, 20.0, 89.0, correctTimestamp);
ColumnProfile c3Profile = getColumnProfile(C3, 75.0, 25.0, 77.0, correctTimestamp);
List<ColumnProfile> columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
TableProfile tableProfile =
new TableProfile()
.withRowCount(6.0)
.withColumnCount(3.0)
.withTimestamp(correctTimestamp)
.withProfileSample(10.0);
CreateTableProfile createTableProfile =
new CreateTableProfile().withTableProfile(tableProfile).withColumnProfile(columnProfiles);
assertResponse(
() -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"Timestamp 1725525388 is not valid, it should be in milliseconds since epoch");
tableProfile = tableProfile.withTimestamp(wrongTimestamp);
c1Profile = c1Profile.withTimestamp(correctTimestamp);
columnProfiles = List.of(c1Profile, c2Profile, c3Profile);
createTableProfile.withTableProfile(tableProfile).withColumnProfile(columnProfiles);
assertResponse(
() -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"Timestamp 1725525388 is not valid, it should be in milliseconds since epoch");
SystemProfile systemProfile = new SystemProfile().withTimestamp(wrongTimestamp);
tableProfile = tableProfile.withTimestamp(correctTimestamp);
createTableProfile.withTableProfile(tableProfile).withSystemProfile(listOf(systemProfile));
assertResponse(
() -> putTableProfileData(table.getId(), createTableProfile, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"Timestamp 1725525388 is not valid, it should be in milliseconds since epoch");
}
void putTableProfile(Table table, Table table1, Map<String, String> authHeaders)
throws IOException, ParseException {
Long timestamp = TestUtils.dateToTimestamp("2021-09-09");

16
openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
View File

@ -2708,6 +2708,22 @@ public class TestCaseResourceTest extends EntityResourceTest<TestCase, CreateTes
assertFalse(testCase.getUseDynamicAssertion());
}
@Test
void createTestCaseResults_wrongTs(TestInfo testInfo) throws IOException, HttpResponseException {
CreateTestCase create = createRequest(testInfo);
TestCase testCase = createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
TestCaseResult testCaseResult =
new TestCaseResult()
.withResult("result")
.withTestCaseStatus(TestCaseStatus.Failed)
.withTimestamp(1725521153L);
assertResponse(
() ->
putTestCaseResult(testCase.getFullyQualifiedName(), testCaseResult, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"Timestamp 1725521153 is not valid, it should be in milliseconds since epoch");
}
private void putInspectionQuery(TestCase testCase, String sql, Map<String, String> authHeaders)
throws IOException {
TestCase putResponse = putInspectionQuery(testCase.getId(), sql, authHeaders);