fix: add lowcardinality support (#24921)

2026-01-08 05:26:19 +00:00 · 2025-12-20 14:03:17 +01:00 · 2025-12-20 14:03:17 +01:00 · 66f2cb4d9c
commit 66f2cb4d9c
parent 8c855c6b41
10 changed files with 137 additions and 8 deletions
--- a/bootstrap/sql/migrations/native/1.11.4/mysql/postDataMigrationSQLScript.sql
+++ b/bootstrap/sql/migrations/native/1.11.4/mysql/postDataMigrationSQLScript.sql
@ -0,0 +1,48 @@
+-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValueLengthsToBeBetween'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesMissingCount'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeInSet'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeNotInSet'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeNotNull'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToBeUnique'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToMatchRegex'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
+
+UPDATE test_definition
+SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
+WHERE name = 'columnValuesToNotMatchRegex'
+  AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
+  AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
--- a/bootstrap/sql/migrations/native/1.11.4/postgres/postDataMigrationSQLScript.sql
+++ b/bootstrap/sql/migrations/native/1.11.4/postgres/postDataMigrationSQLScript.sql
@ -0,0 +1,80 @@
+-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValueLengthsToBeBetween'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesMissingCount'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToBeInSet'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToBeNotInSet'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToBeNotNull'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToBeUnique'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToMatchRegex'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
+
+UPDATE test_definition
+SET json = jsonb_set(
+    json::jsonb,
+    '{supportedDataTypes}',
+    (json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
+)
+WHERE name = 'columnValuesToNotMatchRegex'
+  AND json->'supportedDataTypes' @> '"STRING"'::jsonb
+  AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValueLengthsToBeBetween. Test the value lengths in a column to be between minimum and maximum value. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY"],
+  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY", "LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "minLength",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesMissingCount. Test the column values missing count to be equal to given number. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
+  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON","LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "missingCountValue",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesToBeInSet. Test the column values are in the set.",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN"],
+  "supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN", "LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "allowedValues",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesToBeNotInSet. Test the column values to not be in the set. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN"],
+  "supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN", "LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "forbiddenValues",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesToBeNotNull. Test the number of values in a column are null. Values must be explicitly null. Empty strings don't count as null. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
+  "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON","LOWCARDINALITY"],
  "supportsRowLevelPassedFailed": true,
  "provider": "system",
  "dataQualityDimension": "Completeness"
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json
@ -46,7 +46,8 @@
        "VARIANT",
        "GEOMETRY",
        "POINT",
-        "POLYGON"
+        "POLYGON",
+        "LOWCARDINALITY"
    ],
    "supportsRowLevelPassedFailed": true,
    "provider": "system",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesToMatchRegex. Test the values in a column to match a given regular expression. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"],
+  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "regex",
--- a/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json
@ -5,7 +5,7 @@
  "description": "This schema defines the test ColumnValuesToNotMatchRegex. Test the values in a column to not match a given regular expression. ",
  "entityType": "COLUMN",
  "testPlatforms": ["OpenMetadata"],
-  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"],
+  "supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "LOWCARDINALITY"],
  "parameterDefinition": [
    {
      "name": "forbiddenRegex",