mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-12-27 07:28:30 +00:00
[1.1.1] - Bump size for FQN (#12092)
* Bump size for FQN * Bump table entityName size * Bump table entityName size * Fix table resource tests * Remove pattern from fqn * Remove pattern from fqn * Remove pattern from fqn * Generalize get_by_name in ometa client * Generalize get_by_name in ometa client * Format * Fix test suite * Remove limit from FQN max size * Remove limit from FQN max size * Add sample data * Update lint names * Add more sample data * Bump column name size * 1024 max FQN length * 1024 max FQN length * 1024 max FQN length * Bump FQN
This commit is contained in:
Pere Miquel Brull
GitHub
parent
67b68a70c9
commit
6773541d15
@ -14,7 +14,7 @@ max-args=7
|
||||
max-attributes=12
|
||||
|
||||
# usual typevar naming
|
||||
good-names=T,C,fn,db,df
|
||||
good-names=T,C,fn,db,df,i
|
||||
module-rgx=(([a-z_][a-z0-9_]*)|([a-zA-Z0-9]+))$
|
||||
|
||||
[MASTER]
|
||||
|
||||
@ -4719,8 +4719,8 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "fact_line_item",
|
||||
"fullyQualifiedName": "sample_data.shopify.fact_line_item",
|
||||
"name": "fact_line_item_Lorem_ipsum_dolor_sit_amet_consectetur_adipiscing_elit_sed_do_eiusmod_tempor_incididunt_ut_labore_et_dolore_magna_aliqua_Ut_enim_ad_minim_veniam_quis_nostrud_exercitation_ullamco_laboris_nisi_ut_aliquip_ex_ea_commodo_consequat_Duis_aute_iru",
|
||||
"fullyQualifiedName": "sample_data.shopify.fact_line_item_Lorem_ipsum_dolor_sit_amet_consectetur_adipiscing_elit_sed_do_eiusmod_tempor_incididunt_ut_labore_et_dolore_magna_aliqua_Ut_enim_ad_minim_veniam_quis_nostrud_exercitation_ullamco_laboris_nisi_ut_aliquip_ex_ea_commodo_consequat_Duis_aute_iru",
|
||||
"description": "The fact table contains information about the line items in orders. Each row in the table is a line item in an order. It contains product and product variant details as they were at the time of the order. This table does not include information about returns. Join this table with the TODO fact_sales table to get the details of the product on the day it was sold. This data will match what appears on the order in your Shopify admin as well as the in the Sales reports.",
|
||||
"version": 0.1,
|
||||
"updatedAt": 1638354087696,
|
||||
|
||||
@ -14,13 +14,10 @@ Helper functions to handle OpenMetadata Entities' properties
|
||||
|
||||
import re
|
||||
import string
|
||||
from functools import singledispatch
|
||||
from typing import Type, TypeVar, Union
|
||||
from typing import Any, Type, TypeVar, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from metadata.generated.schema.type import basic
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
|
||||
@ -61,19 +58,14 @@ def get_entity_type(
|
||||
return class_name
|
||||
|
||||
|
||||
@singledispatch
|
||||
def model_str(arg) -> str:
|
||||
def model_str(arg: Any) -> str:
|
||||
"""
|
||||
Default model stringifying method
|
||||
Default model stringifying method.
|
||||
|
||||
Some elements such as FQN, EntityName, UUID
|
||||
have the actual value under the pydantic base __root__
|
||||
"""
|
||||
if hasattr(arg, "__root__"):
|
||||
return str(arg.__root__)
|
||||
|
||||
return str(arg)
|
||||
|
||||
|
||||
@model_str.register(basic.Uuid)
|
||||
@model_str.register(basic.FullyQualifiedEntityName)
|
||||
@model_str.register(basic.EntityName)
|
||||
def _(arg) -> str:
|
||||
"""
|
||||
Models with __root__
|
||||
"""
|
||||
return str(arg.__root__)
|
||||
|
||||
@ -14,6 +14,7 @@ Sample Data source ingestion
|
||||
# pylint: disable=too-many-lines,too-many-statements
|
||||
import json
|
||||
import random
|
||||
import string
|
||||
import traceback
|
||||
from collections import namedtuple
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@ -106,6 +107,7 @@ from metadata.parsers.schema_parsers import (
|
||||
)
|
||||
from metadata.utils import fqn
|
||||
from metadata.utils.constants import UTF_8
|
||||
from metadata.utils.fqn import FQN_SEPARATOR
|
||||
from metadata.utils.helpers import get_standard_chart_type
|
||||
from metadata.utils.logger import ingestion_logger
|
||||
|
||||
@ -1030,6 +1032,39 @@ class SampleDataSource(
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error ingesting Container [{container}]: {exc}")
|
||||
|
||||
# Create a very nested container structure:
|
||||
try:
|
||||
long_base_name = (
|
||||
"".join(random.choice(string.ascii_letters) for _ in range(100))
|
||||
+ "{suffix}"
|
||||
)
|
||||
for base_name in ("deep_nested_container_{suffix}", long_base_name):
|
||||
parent_container_fqns = []
|
||||
# We cannot go deeper than this
|
||||
for i in range(1, 6):
|
||||
parent_container: Container = (
|
||||
self.metadata.get_by_name(
|
||||
entity=Container,
|
||||
fqn=self.storage_service.fullyQualifiedName.__root__
|
||||
+ FQN_SEPARATOR
|
||||
+ FQN_SEPARATOR.join(parent_container_fqns),
|
||||
)
|
||||
if parent_container_fqns
|
||||
else None
|
||||
)
|
||||
name = base_name.format(suffix=i)
|
||||
parent_container_fqns.append(name)
|
||||
yield CreateContainerRequest(
|
||||
name=name,
|
||||
parent=EntityReference(id=parent_container.id, type="container")
|
||||
if parent_container
|
||||
else None,
|
||||
service=self.storage_service.fullyQualifiedName,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(traceback.format_exc())
|
||||
logger.warning(f"Error ingesting nested containers: {exc}")
|
||||
|
||||
def ingest_users(self) -> Iterable[OMetaUserProfile]:
|
||||
"""
|
||||
Ingest Sample User data
|
||||
|
||||
@ -85,7 +85,9 @@ import org.openmetadata.service.security.SecurityUtil;
|
||||
|
||||
@Slf4j
|
||||
public final class TestUtils {
|
||||
public static String LONG_ENTITY_NAME = "a".repeat(128 + 1);
|
||||
|
||||
// Setting length at +256 since this is the length of the longest EntityName for Test Suites
|
||||
public static String LONG_ENTITY_NAME = "a".repeat(256 + 1);
|
||||
public static final Map<String, String> ADMIN_AUTH_HEADERS = authHeaders(ADMIN_USER_NAME + "@open-metadata.org");
|
||||
public static final String INGESTION_BOT = "ingestion-bot";
|
||||
public static final Map<String, String> INGESTION_BOT_AUTH_HEADERS =
|
||||
|
||||
@ -9,9 +9,9 @@
|
||||
"definitions": {
|
||||
"testSuiteEntityName": {
|
||||
"description": "Name of a test suite entity. For executable testSuite, this should match the entity FQN in the platform.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 256
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 256
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
@ -32,7 +32,7 @@
|
||||
"$ref": "../../type/entityReference.json"
|
||||
},
|
||||
"executableEntityReference": {
|
||||
"description": "FQN of the entity the test suite is executed against.. Only applicable for executable test suites.",
|
||||
"description": "FQN of the entity the test suite is executed against. Only applicable for executable test suites.",
|
||||
"$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName"
|
||||
}
|
||||
},
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
"description": "Name of a table. Expected to be unique within a database.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128,
|
||||
"maxLength": 256,
|
||||
"pattern": "^((?!::).)*$"
|
||||
},
|
||||
"profileSampleType": {
|
||||
@ -205,7 +205,7 @@
|
||||
"description": "Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128,
|
||||
"maxLength": 256,
|
||||
"pattern": "^((?!::).)*$"
|
||||
},
|
||||
"tablePartition": {
|
||||
|
||||
@ -103,10 +103,10 @@
|
||||
"pattern": "^(?U)[\\w'\\- .&()%]+$"
|
||||
},
|
||||
"fullyQualifiedEntityName": {
|
||||
"description": "A unique name that identifies an entity. Example for table 'DatabaseService:Database:Table'.",
|
||||
"description": "A unique name that identifies an entity. Example for table 'DatabaseService.Database.Schema.Table'.",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 256
|
||||
"maxLength": 3072
|
||||
},
|
||||
"sqlQuery": {
|
||||
"$comment" : "@om-field-type",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user