yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-25 22:49:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Sample data, modify regex pattern (#14467)

Browse Source
This commit is contained in:
Ayush Shah 2024-01-11 14:23:33 +05:30 committed by GitHub
parent ff690d8dd4
commit 9c6d202555
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
67 changed files with 632 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
docker/run_local_docker.sh
View File

@ -16,12 +16,12 @@ helpFunction()
{
echo ""
echo "Usage: $0 -m mode -d database"
printf "\t-m Running mode: [ui, no-ui]. Default [ui]\n"
printf "\t-d Database: [mysql, postgresql]. Default [mysql]\n"
printf "\t-s Skip maven build: [true, false]. Default [false]\n"
printf "\t-x Open JVM debug port on 5005: [true, false]. Default [false]\n"
printf "\t-h For usage help\n"
printf "\t-r For Cleaning DB Volumes. [true, false]. Default [true]\n"
echo "\t-m Running mode: [ui, no-ui]. Default [ui]\n"
echo "\t-d Database: [mysql, postgresql]. Default [mysql]\n"
echo "\t-s Skip maven build: [true, false]. Default [false]\n"
echo "\t-x Open JVM debug port on 5005: [true, false]. Default [false]\n"
echo "\t-h For usage help\n"
echo "\t-r For Cleaning DB Volumes. [true, false]. Default [true]\n"
exit 1 # Exit script after printing help
}
@ -108,12 +108,17 @@ if [ $RESULT -ne 0 ]; then
fi
until curl -s -f "http://localhost:9200/_cat/indices/team_search_index"; do
printf 'Checking if Elastic Search instance is up...\n'
echo 'Checking if Elastic Search instance is up...\n'
sleep 5
done
until curl -s -f --header 'Authorization: Basic YWRtaW46YWRtaW4=' "http://localhost:8080/api/v1/dags/sample_data"; do
printf 'Checking if Sample Data DAG is reachable...\n'
echo 'Checking if Sample Data DAG is reachable...\n'
sleep 5
done
until curl -s -f --header "Authorization: Bearer $authorizationToken" "http://localhost:8585/api/v1/tables"; do
echo 'Checking if OM Server is reachable...\n'
sleep 5
done
@ -124,13 +129,20 @@ curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_data' \
"is_paused": false
}'
printf 'Validate sample data DAG...'
curl --location --request PATCH 'localhost:8080/api/v1/dags/extended_sample_data' \
--header 'Authorization: Basic YWRtaW46YWRtaW4=' \
--header 'Content-Type: application/json' \
--data-raw '{
"is_paused": false
}'
echo 'Validate sample data DAG...'
sleep 5
python -m pip install ingestion/
python docker/validate_compose.py
until curl -s -f --header "Authorization: Bearer $authorizationToken" "http://localhost:8585/api/v1/tables/name/sample_data.ecommerce_db.shopify.fact_sale"; do
printf 'Waiting on Sample Data Ingestion to complete...\n'
echo 'Waiting on Sample Data Ingestion to complete...\n'
curl -v --header "Authorization: Bearer $authorizationToken" "http://localhost:8585/api/v1/tables"
sleep 5
done
@ -155,6 +167,7 @@ curl --location --request PATCH 'localhost:8080/api/v1/dags/sample_lineage' \
--data-raw '{
"is_paused": false
}'
echo "✔running reindexing"
# Trigger ElasticSearch ReIndexing from UI
curl --location --request POST 'http://localhost:8585/api/v1/apps/trigger/SearchIndexingApplication' \

42
docker/validate_compose.py
View File

@ -1,7 +1,3 @@
"""
Helper functions used for ingestion of sample data into docker by calling airflow dags
"""
import time
from pprint import pprint
from typing import Tuple
@ -20,16 +16,23 @@ def get_last_run_info() -> Tuple[str, str]:
"""
Make sure we can pick up the latest run info
"""
max_retries = 15
retries = 0
dag_runs = None
while not dag_runs:
while retries < max_retries:
log_ansi_encoded_string(message="Waiting for DAG Run data...")
time.sleep(5)
runs = requests.get(
"http://localhost:8080/api/v1/dags/sample_data/dagRuns", auth=BASIC_AUTH, timeout=REQUESTS_TIMEOUT
).json()
dag_runs = runs.get("dag_runs")
if dag_runs[0].get("dag_run_id"):
return dag_runs[0].get("dag_run_id"), "success"
retries += 1
return None, None
return dag_runs[0].get("dag_run_id"), dag_runs[0].get("state")
def print_last_run_logs() -> None:
@ -45,17 +48,26 @@ def print_last_run_logs() -> None:
def main():
max_retries = 15
retries = 0
state = None
while state != "success":
log_ansi_encoded_string(
message="Waiting for sample data ingestion to be a success. We'll show some logs along the way.",
)
while retries < max_retries:
dag_run_id, state = get_last_run_info()
log_ansi_encoded_string(message=f"DAG run: [{dag_run_id}, {state}]")
print_last_run_logs()
time.sleep(10)
if state == "success":
log_ansi_encoded_string(message=f"DAG run: [{dag_run_id}, {state}]")
print_last_run_logs()
break
else:
log_ansi_encoded_string(
message="Waiting for sample data ingestion to be a success. We'll show some logs along the way.",
)
log_ansi_encoded_string(message=f"DAG run: [{dag_run_id}, {state}]")
print_last_run_logs()
time.sleep(10)
retries += 1
if retries == max_retries:
raise Exception("Max retries exceeded. Sample data ingestion was not successful.")
if __name__ == "__main__":

80
ingestion/examples/airflow/dags/airflow_extended_sample_data.py Normal file
View File

@ -0,0 +1,80 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import timedelta
import yaml
from airflow import DAG
from metadata.workflow.workflow_output_handler import print_status
try:
from airflow.operators.python import PythonOperator
except ModuleNotFoundError:
from airflow.operators.python_operator import PythonOperator
from airflow.utils.dates import days_ago
from metadata.workflow.metadata import MetadataWorkflow
default_args = {
"owner": "user_name",
"email": ["username@org.com"],
"email_on_failure": False,
"retries": 3,
"retry_delay": timedelta(seconds=10),
"execution_timeout": timedelta(minutes=60),
}
config = """
source:
type: custom-database
serviceName: extended_sample_data
serviceConnection:
config:
type: CustomDatabase
sourcePythonClass: metadata.ingestion.source.database.sample_data.ExtendedSampleDataSource
connectionOptions:
sampleDataFolder: "/home/airflow/ingestion/examples/sample_data"
sourceConfig: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://openmetadata-server:8585/api
authProvider: openmetadata
securityConfig:
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
"""
def metadata_ingestion_workflow():
workflow_config = yaml.safe_load(config)
workflow = MetadataWorkflow.create(workflow_config)
workflow.execute()
workflow.raise_from_status()
print_status(workflow)
workflow.stop()
with DAG(
"extended_sample_data",
default_args=default_args,
description="An example DAG which runs a OpenMetadata ingestion workflow",
start_date=days_ago(1),
is_paused_upon_creation=True,
catchup=False,
) as dag:
ingest_task = PythonOperator(
task_id="ingest_using_recipe",
python_callable=metadata_ingestion_workflow,
)

4
ingestion/ingestion_dependency.sh
View File

@ -39,8 +39,8 @@ airflow users create \
--email spiderman@superhero.org \
--password ${AIRFLOW_ADMIN_PASSWORD}
(sleep 5; airflow db upgrade)
(sleep 5; airflow db upgrade)
(sleep 5; airflow db migrate)
(sleep 5; airflow db migrate)
# we need to this in case the container is restarted and the scheduler exited without tidying up its lock file
rm -f /opt/airflow/airflow-webserver-monitor.pid

20
ingestion/pipelines/extended_sample_data.yaml Normal file
View File

@ -0,0 +1,20 @@
---
source:
type: custom-database
serviceName: sample_data
serviceConnection:
config:
type: CustomDatabase
sourcePythonClass: metadata.ingestion.source.database.extended_sample_data.ExtendedSampleDataSource
connectionOptions:
sampleDataFolder: "./examples/sample_data"
sourceConfig: {}
sink:
type: metadata-rest
config: {}
workflowConfig:
openMetadataServerConfig:
hostPort: http://localhost:8585/api
authProvider: openmetadata
securityConfig:
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

6
ingestion/setup.py
View File

@ -269,6 +269,7 @@ dev = {
"build",
}
test = {
# Install Airflow as it's not part of `all` plugin
VERSIONS["airflow"],
@ -306,6 +307,10 @@ e2e_test = {
"pytest-base-url",
}
extended_testing = {
"Faker", # For Sample Data Generation
}
def filter_requirements(filtered: Set[str]) -> List[str]:
"""Filter out requirements from base_requirements"""
@ -327,6 +332,7 @@ setup(
"dev": list(dev),
"test": list(test),
"e2e_test": list(e2e_test),
"extended_testing": list(extended_testing),
"data-insight": list(plugins["elasticsearch"]),
**{plugin: list(dependencies) for (plugin, dependencies) in plugins.items()},
"all": filter_requirements({"airflow", "db2", "great-expectations"}),

2
ingestion/sonar-project.properties
View File

@ -5,7 +5,7 @@ sonar.language=py
sonar.sources=src/metadata
sonar.tests=tests
sonar.exclusions=src/metadata_server/static/**,ingestion/src/metadata_server/templates/**,**/*.yaml,**/*.yml,**/*.json,src/metadata/ingestion/source/database/sample_*
sonar.exclusions=src/metadata_server/static/**,ingestion/src/metadata_server/templates/**,**/*.yaml,**/*.yml,**/*.json,src/metadata/ingestion/source/database/sample_*,src/metadata/ingestion/source/database/extended_sample_*
sonar.python.xunit.reportPath=junit/test-results-*.xml
sonar.python.coverage.reportPaths=ci-coverage.xml
sonar.python.version=3.7,3.8,3.9

5
ingestion/src/metadata/antlr/split_listener.py
View File

@ -35,7 +35,7 @@ class EntityLinkSplitListener(EntityLinkListener):
def __init__(self):
self._list = []
def enterEntityAttribute(self, ctx: EntityLinkParser.EntityAttributeContext):
def enterNameOrFQN(self, ctx: EntityLinkParser.NameOrFQNContext):
self._list.append(ctx.getText())
def enterEntityType(self, ctx: EntityLinkParser.EntityTypeContext):
@ -44,8 +44,5 @@ class EntityLinkSplitListener(EntityLinkListener):
def enterEntityField(self, ctx: EntityLinkParser.EntityFieldContext):
self._list.append(ctx.getText())
def enterEntityFqn(self, ctx: EntityLinkParser.EntityFqnContext):
self._list.append(ctx.getText())
def split(self):
return self._list

2
ingestion/src/metadata/ingestion/sink/metadata_rest.py
View File

@ -279,7 +279,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
for role in record.roles:
try:
role_entity = self.metadata.get_by_name(
entity=Role, fqn=str(role.name.__root__.__root__)
entity=Role, fqn=str(role.name.__root__)
)
except APIError:
role_entity = self._create_role(role)

4
ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py
View File

@ -29,7 +29,7 @@ from metadata.generated.schema.api.data.createDatabaseSchema import (
from metadata.generated.schema.api.data.createStoredProcedure import (
CreateStoredProcedureRequest,
)
from metadata.generated.schema.entity.data.database import Database, EntityName
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
from metadata.generated.schema.entity.data.table import (
@ -49,7 +49,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
from metadata.generated.schema.security.credentials.gcpValues import (
GcpCredentialsValues,
)
from metadata.generated.schema.type.basic import SourceUrl
from metadata.generated.schema.type.basic import EntityName, SourceUrl
from metadata.generated.schema.type.tagLabel import TagLabel
from metadata.ingestion.api.models import Either
from metadata.ingestion.api.steps import InvalidSourceException

299
ingestion/src/metadata/ingestion/source/database/extended_sample_data.py Normal file
View File

@ -0,0 +1,299 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Sample Data source ingestion
"""
import json
from collections import namedtuple
from typing import Iterable
from metadata.generated.schema.api.data.createDashboardDataModel import (
CreateDashboardDataModelRequest,
)
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
from metadata.generated.schema.api.data.createDatabaseSchema import (
CreateDatabaseSchemaRequest,
)
from metadata.generated.schema.api.data.createTable import CreateTableRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.dashboardDataModel import DashboardDataModel
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.services.connections.database.customDatabaseConnection import (
CustomDatabaseConnection,
)
from metadata.generated.schema.entity.services.dashboardService import DashboardService
from metadata.generated.schema.entity.services.databaseService import DatabaseService
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
from metadata.generated.schema.type.entityLineage import Source as LineageSource
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.models import Either
from metadata.ingestion.api.steps import InvalidSourceException, Source
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils import fqn
from metadata.utils.constants import UTF_8
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()
COLUMN_NAME = "Column"
KEY_TYPE = "Key type"
DATA_TYPE = "Data type"
COL_DESCRIPTION = "Description"
TableKey = namedtuple("TableKey", ["schema", "table_name"])
class InvalidSampleDataException(Exception):
"""
Sample data is not valid to be ingested
"""
class ExtendedSampleDataSource(Source): # pylint: disable=too-many-instance-attributes
"""
Loads JSON data and prepares the required
python objects to be sent to the Sink.
"""
def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
import faker # pylint: disable=import-outside-toplevel
super().__init__()
self.fake = faker.Faker(["en-US", "zh_CN", "ja_JP", "th_TH"])
self.database_service_json = {}
self.dashboard_service_json = {}
self.config = config
self.service_connection = config.serviceConnection.__root__.config
self.metadata = metadata
self.list_policies = []
self.store_table_fqn = []
self.store_data_model_fqn = []
self.store_dashboard_fqn = []
sample_data_folder = self.service_connection.connectionOptions.__root__.get(
"sampleDataFolder"
)
if not sample_data_folder:
raise InvalidSampleDataException(
"Cannot get sampleDataFolder from connection options"
)
self.database_service_json = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/datasets/service.json",
"r",
encoding=UTF_8,
)
)
self.tables = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/datasets/tables.json",
"r",
encoding=UTF_8,
)
)
self.database_service_json = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/datasets/service.json",
"r",
encoding=UTF_8,
)
)
self.database_service = self.metadata.get_service_or_create(
entity=DatabaseService, config=WorkflowSource(**self.database_service_json)
)
self.dashboard_service_json = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/dashboards/service.json",
"r",
encoding=UTF_8,
)
)
self.data_models = json.load(
open( # pylint: disable=consider-using-with
sample_data_folder + "/dashboards/dashboardDataModels.json",
"r",
encoding=UTF_8,
)
)
self.dashboard_service = self.metadata.get_service_or_create(
entity=DashboardService,
config=WorkflowSource(**self.dashboard_service_json),
)
self.db_name = None
@classmethod
def create(cls, config_dict, metadata: OpenMetadata):
"""Create class instance"""
config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
connection: CustomDatabaseConnection = config.serviceConnection.__root__.config
if not isinstance(connection, CustomDatabaseConnection):
raise InvalidSourceException(
f"Expected CustomDatabaseConnection, but got {connection}"
)
return cls(config, metadata)
def prepare(self):
"""Nothing to prepare"""
def _iter(self, *_, **__) -> Iterable[Entity]:
yield from self.generate_sample_data()
def close(self):
"""Nothing to close"""
def test_connection(self) -> None:
"""Custom sources don't support testing connections"""
def generate_sample_data(self):
"""
Generate sample data for dashboard and database service,
with lineage between them, having long names, special characters and description
"""
for _ in range(5):
name = self.generate_name()
text = self.generate_text()
self.database_service_json["name"] = name
self.database_service_json["description"] = text
db = self.create_database_request(name, text)
yield Either(right=db)
for _ in range(2):
name = self.generate_name()
text = self.generate_text()
schema = self.create_database_schema_request(name, text, db)
yield Either(right=schema)
for table in self.tables["tables"]:
table_request = self.create_table_request(name, text, schema, table)
yield Either(right=table_request)
table_entity_fqn = fqn.build(
self.metadata,
entity_type=Table,
service_name=self.database_service.name.__root__,
database_name=db.name.__root__,
schema_name=schema.name.__root__,
table_name=table_request.name.__root__,
)
self.store_table_fqn.append(table_entity_fqn)
self.dashboard_service_json["name"] = name
self.dashboard_service_json["description"] = text
for data_model in self.data_models["datamodels"]:
name = self.generate_name()
text = self.generate_text()
data_model_request = self.create_dashboard_data_model_request(
name, text, data_model
)
yield Either(right=data_model_request)
data_model_entity_fqn = fqn.build(
self.metadata,
entity_type=DashboardDataModel,
service_name=self.dashboard_service.name.__root__,
data_model_name=data_model_request.name.__root__,
)
self.store_data_model_fqn.append(data_model_entity_fqn)
for table_fqn in self.store_table_fqn:
from_table = self.metadata.get_by_name(entity=Table, fqn=table_fqn)
for dashboard_datamodel_fqn in self.store_data_model_fqn:
to_datamodel = self.metadata.get_by_name(
entity=DashboardDataModel, fqn=dashboard_datamodel_fqn
)
yield Either(
right=AddLineageRequest(
edge=EntitiesEdge(
fromEntity=EntityReference(
id=from_table.id.__root__, type="table"
),
toEntity=EntityReference(
id=to_datamodel.id.__root__,
type="dashboardDataModel",
),
lineageDetails=LineageDetails(
source=LineageSource.DashboardLineage
),
)
)
)
def generate_name(self):
return f"Sample-@!3_(%t3st@)%_^{self.fake.name()}"
def generate_text(self):
return f"Sample-@!3_(%m@)%_^{self.fake.text()}"
def create_database_request(self, name, text):
db = CreateDatabaseRequest(
name=name,
description=text,
service=self.database_service.fullyQualifiedName.__root__,
)
return db
def create_database_schema_request(self, name, text, db):
self.db_name = db.name.__root__
db_fqn = fqn.build(
self.metadata,
entity_type=Database,
service_name=self.database_service.name.__root__,
database_name=db.name.__root__,
)
schema = CreateDatabaseSchemaRequest(
name=name,
description=text,
database=db_fqn,
)
return schema
def create_table_request(self, name, text, schema, table):
dbschema_fqn = fqn.build(
self.metadata,
entity_type=DatabaseSchema,
service_name=self.database_service.name.__root__,
database_name=self.db_name,
schema_name=schema.name.__root__,
)
table_request = CreateTableRequest(
name=name,
description=text,
columns=table["columns"],
databaseSchema=dbschema_fqn,
tableConstraints=table.get("tableConstraints"),
tableType=table["tableType"],
)
return table_request
def create_dashboard_data_model_request(self, name, text, data_model):
data_model_request = CreateDashboardDataModelRequest(
name=name,
description=text,
columns=data_model["columns"],
dataModelType=data_model["dataModelType"],
sql=data_model["sql"],
serviceType=data_model["serviceType"],
service=self.dashboard_service.fullyQualifiedName,
)
return data_model_request

2
ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
View File

@ -32,7 +32,6 @@ from metadata.generated.schema.entity.data.storedProcedure import (
)
from metadata.generated.schema.entity.data.table import (
ConstraintType,
EntityName,
IntervalType,
TableConstraint,
TablePartition,
@ -47,6 +46,7 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.basic import EntityName
from metadata.ingestion.api.models import Either
from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.ometa.ometa_api import OpenMetadata

3
ingestion/tests/unit/metadata/utils/test_entity_link.py
View File

@ -14,6 +14,7 @@ test entity link utils
"""
import pytest
from antlr4.error.Errors import ParseCancellationException
from metadata.utils.entity_link import get_decoded_column, get_table_or_column_fqn
@ -50,7 +51,7 @@ def test_get_table_or_column_fqn():
invalid_entity_link = (
"<#E::table::rds.dev.dbt_jaffle.customers::foo::number_of_orders>"
)
with pytest.raises(ValueError):
with pytest.raises(ParseCancellationException):
get_table_or_column_fqn(invalid_entity_link)
invalid_entity_link = "<#E::table::rds.dev.dbt_jaffle.customers>"

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -371,7 +371,7 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true);
assertSummary(result, ApiStatus.FAILURE, 2, 1, 1);
String[] expectedRows = {
resultsHeader, getFailedRecord(record, "[name must match \"(?U)^[\\w'\\- .&()%]+$\"]")
resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]")
};
assertRows(result, expectedRows);

3
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java
View File

@ -1096,8 +1096,9 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
CsvImportResult result = importCsv(team.getName(), csv, false);
assertSummary(result, ApiStatus.FAILURE, 2, 1, 1);
String[] expectedRows = {
resultsHeader, getFailedRecord(record, "[name must match \"(?U)^[\\w\\-.]+$\"]")
resultsHeader, getFailedRecord(record, "[name must match \"^((?!::).)*$\"]")
};
assertRows(result, expectedRows);
// Invalid team

2
openmetadata-service/src/test/java/org/openmetadata/service/util/ValidatorUtilTest.java
View File

@ -22,7 +22,7 @@ class ValidatorUtilTest {
// Invalid name
glossary.withName("invalid::Name").withDescription("description");
assertEquals("[name must match \"(?U)^[\\w'\\- .&()%]+$\"]", ValidatorUtil.validate(glossary));
assertEquals("[name must match \"^((?!::).)*$\"]", ValidatorUtil.validate(glossary));
// No error
glossary.withName("validName").withId(UUID.randomUUID()).withDescription("description");

65
openmetadata-spec/src/main/antlr4/org/openmetadata/schema/EntityLink.g4
View File

@ -1,54 +1,43 @@
grammar EntityLink;
entitylink
: '<#E' (RESERVED entity)+ '>' EOF
: RESERVED_START (separator entity_type separator name_or_fqn)+
(separator entity_field (separator name_or_fqn)*)* '>' EOF
;
entity
entity_type
: ENTITY_TYPE # entityType
| ENTITY_ATTRIBUTE # entityAttribute
| ENTITY_FQN # entityFqn
| ENTITY_FIELD # entityField
;
ENTITY_TYPE
: 'table'
| 'database'
| 'databaseSchema'
| 'metrics'
| 'dashboard'
| 'pipeline'
| 'chart'
| 'report'
| 'topic'
| 'mlmodel'
| 'bot'
| 'THREAD'
| 'location'
| 'glossary'
| 'glossaryTerm'
| 'tag'
| 'classification'
| 'type'
| 'testDefinition'
| 'testSuite'
| 'testCase'
| 'dashboardDataModel'
name_or_fqn
: NAME_OR_FQN # nameOrFQN
;
ENTITY_FIELD
: 'columns'
| 'description'
| 'tags'
| 'tasks'
entity_field
: ENTITY_FIELD # entityField
;
RESERVED
separator
: '::'
;
ENTITY_ATTRIBUTE
: [a-z]+
RESERVED_START
: '<#E'
;
ENTITY_FQN
: [\p{L}\p{N},. _\-'&()%"]+
ENTITY_TYPE
: 'table' | 'database' | 'databaseSchema' | 'metrics' | 'dashboard' | 'pipeline'
| 'chart' | 'report' | 'topic' | 'mlmodel' | 'bot' | 'THREAD' | 'location'
| 'glossary' | 'glossaryTerm' | 'tag' | 'classification' | 'type'
| 'testDefinition' | 'testSuite' | 'testCase' | 'dashboardDataModel'
;
ENTITY_FIELD
: 'columns' | 'description' | 'tags' | 'tasks'
;
NAME_OR_FQN
: ~(':')+ ('>')*? ~(':'|'>')+
;

2
openmetadata-spec/src/main/resources/json/schema/api/createBot.json
View File

@ -10,7 +10,7 @@
"properties": {
"name": {
"description": "Name of the bot.",
"$ref": "../entity/teams/user.json#/definitions/entityName"
"$ref": "../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Name used for display purposes. Example 'FirstName LastName'.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createContainer.json
View File

@ -10,7 +10,7 @@
"properties": {
"name": {
"description": "Name that identifies this Container model.",
"$ref": "../../entity/data/container.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this Container model.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createCustomProperty.json
View File

@ -24,7 +24,7 @@
"properties": {
"name": {
"description": "Name that identifies this Custom Property model.",
"$ref": "../../entity/data/container.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"description": {
"description": "Description of the Container instance.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createDatabase.json
View File

@ -10,7 +10,7 @@
"properties": {
"name": {
"description": "Name that identifies this database instance uniquely.",
"$ref": "../../entity/data/database.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this database.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createDatabaseSchema.json
View File

@ -11,7 +11,7 @@
"properties": {
"name": {
"description": "Name that identifies this database schema instance uniquely.",
"$ref": "../../entity/data/databaseSchema.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this database schema.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createGlossary.json
View File

@ -24,7 +24,7 @@
"description": "User references of the reviewers for this glossary.",
"type": "array",
"items": {
"$ref": "../../entity/teams/user.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
}
},
"owner": {

2
openmetadata-spec/src/main/resources/json/schema/api/data/createGlossaryTerm.json
View File

@ -56,7 +56,7 @@
"description": "User names of the reviewers for this glossary.",
"type" : "array",
"items" : {
"$ref" : "../../entity/teams/user.json#/definitions/entityName"
"$ref" : "../../type/basic.json#/definitions/entityName"
}
},
"owner": {

2
openmetadata-spec/src/main/resources/json/schema/api/data/createPipeline.json
View File

@ -10,7 +10,7 @@
"properties": {
"name": {
"description": "Name that identifies this pipeline instance uniquely.",
"$ref": "../../entity/data/pipeline.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this Pipeline. It could be title or label from the source services.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createStoredProcedure.json
View File

@ -11,7 +11,7 @@
"properties": {
"name": {
"description": "Name of a Stored Procedure.",
"$ref": "../../entity/data/storedProcedure.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this Stored Procedure.",

2
openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json
View File

@ -10,7 +10,7 @@
"properties": {
"name": {
"description": "Name that identifies the this entity instance uniquely. Same as id if when name is not unique",
"$ref": "../../entity/data/table.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this table.",

2
openmetadata-spec/src/main/resources/json/schema/api/teams/createRole.json
View File

@ -9,7 +9,7 @@
"properties": {
"name": {
"$ref": "../../entity/teams/role.json#/definitions/roleName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Optional name used for display purposes. Example 'Data Consumer'",

2
openmetadata-spec/src/main/resources/json/schema/api/teams/createUser.json
View File

@ -8,7 +8,7 @@
"javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
"properties": {
"name": {
"$ref": "../../entity/teams/user.json#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"description": {
"description": "Used for user biography.",

2
openmetadata-spec/src/main/resources/json/schema/entity/bot.json
View File

@ -13,7 +13,7 @@
},
"name": {
"description": "Name of the bot.",
"$ref": "../entity/teams/user.json#/definitions/entityName"
"$ref": "../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",

2
openmetadata-spec/src/main/resources/json/schema/entity/classification/tag.json
View File

@ -100,4 +100,4 @@
},
"required": ["name", "description"],
"additionalProperties": false
}
}

9
openmetadata-spec/src/main/resources/json/schema/entity/data/container.json
View File

@ -10,13 +10,6 @@
"org.openmetadata.schema.EntityInterface"
],
"definitions": {
"entityName": {
"description": "Name of a container. Expected to be unique in the same level containers.",
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^((?!::).)*$"
},
"containerDataModel": {
"description": "This captures information about how the container's data is modeled, if it has a schema. ",
"type": "object",
@ -65,7 +58,7 @@
},
"name": {
"description": "Name that identifies the container.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "Name that uniquely identifies a container in the format 'ServiceName.ContainerName'.",

9
openmetadata-spec/src/main/resources/json/schema/entity/data/database.json
View File

@ -8,13 +8,6 @@
"javaType": "org.openmetadata.schema.entity.data.Database",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"entityName": {
"description": "Name of a table. Expected to be unique within a database.",
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^((?!::).)*$"
}
},
"properties": {
"id": {
@ -23,7 +16,7 @@
},
"name": {
"description": "Name that identifies the database.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "Name that uniquely identifies a database in the format 'ServiceName.DatabaseName'.",

12
openmetadata-spec/src/main/resources/json/schema/entity/data/databaseSchema.json
View File

@ -7,14 +7,8 @@
"type": "object",
"javaType": "org.openmetadata.schema.entity.data.DatabaseSchema",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"entityName": {
"description": "Name of a table. Expected to be unique within a database.",
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^((?!::).)*$"
}
"definitions": {
},
"properties": {
"id": {
@ -23,7 +17,7 @@
},
"name": {
"description": "Name that identifies the schema.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "Name that uniquely identifies a schema in the format 'ServiceName.DatabaseName.SchemaName'.",

9
openmetadata-spec/src/main/resources/json/schema/entity/data/pipeline.json
View File

@ -8,13 +8,6 @@
"javaType": "org.openmetadata.schema.entity.data.Pipeline",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"entityName": {
"description": "Name of a pipeline. Expected to be unique within a pipeline service.",
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "^((?!::).)*$"
},
"statusType": {
"javaType": "org.openmetadata.schema.type.StatusType",
"description": "Enum defining the possible Status.",
@ -161,7 +154,7 @@
},
"name": {
"description": "Name that identifies this pipeline instance uniquely.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this Pipeline. It could be title or label from the source services.",

9
openmetadata-spec/src/main/resources/json/schema/entity/data/storedProcedure.json
View File

@ -8,13 +8,6 @@
"javaType": "org.openmetadata.schema.entity.data.StoredProcedure",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"entityName": {
"description": "Name of a Stored Procedure. Expected to be unique within a database schema.",
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^((?!::).)*$"
},
"storedProcedureCode": {
"properties": {
"language": {
@ -57,7 +50,7 @@
},
"name": {
"description": "Name of Stored Procedure.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "Fully qualified name of a Stored Procedure.",

9
openmetadata-spec/src/main/resources/json/schema/entity/data/table.json
View File

@ -10,13 +10,6 @@
"org.openmetadata.schema.EntityInterface", "org.openmetadata.schema.ColumnsEntityInterface"
],
"definitions": {
"entityName": {
"description": "Name of a table. Expected to be unique within a database.",
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^((?!::).)*$"
},
"profileSampleType": {
"description": "Type of Profile Sample (percentage or rows)",
"type": "string",
@ -898,7 +891,7 @@
},
"name": {
"description": "Name of a table. Expected to be unique within a database.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"displayName": {
"description": "Display Name that identifies this table. It could be title or label from the source services.",

9
openmetadata-spec/src/main/resources/json/schema/entity/teams/role.json
View File

@ -6,18 +6,15 @@
"javaType": "org.openmetadata.schema.entity.teams.Role",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"type": "object",
"definitions": {
"roleName": {
"description": "A unique name for the role.",
"$ref": "../../type/basic.json#/definitions/entityName"
}
"definitions": {
},
"properties": {
"id": {
"$ref": "../../type/basic.json#/definitions/uuid"
},
"name": {
"$ref": "#/definitions/roleName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",

9
openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json
View File

@ -7,13 +7,6 @@
"javaType": "org.openmetadata.schema.entity.teams.User",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"entityName": {
"description": "Login name of the user, typically the user ID from an identity provider. Example - uid from LDAP.",
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "(?U)^[\\w\\-.]+$"
},
"authenticationMechanism": {
"type": "object",
"description": "User/Bot Authentication Mechanism.",
@ -46,7 +39,7 @@
},
"name": {
"description": "A unique name of the user, typically the user ID from an identity provider. Example - uid from LDAP.",
"$ref": "#/definitions/entityName"
"$ref": "../../type/basic.json#/definitions/entityName"
},
"fullyQualifiedName": {
"description": "FullyQualifiedName same as `name`.",

4
openmetadata-spec/src/main/resources/json/schema/type/basic.json
View File

@ -99,8 +99,8 @@
"description": "Name that identifies an entity.",
"type": "string",
"minLength": 1,
"maxLength": 128,
"pattern": "(?U)^[\\w'\\- .&()%]+$"
"maxLength": 256,
"pattern": "^((?!::).)*$"
},
"fullyQualifiedEntityName": {
"description": "A unique name that identifies an entity. Example for table 'DatabaseService.Database.Schema.Table'.",

5
openmetadata-ui/src/main/resources/ui/cypress/common/common.js
View File

@ -26,6 +26,7 @@ import {
DATA_ASSETS,
DELETE_TERM,
EXPLORE_PAGE_TABS,
INVALID_NAMES,
NAME_VALIDATION_ERROR,
SEARCH_INDEX,
} from '../constants/constants';
@ -269,7 +270,9 @@ export const testServiceCreationAndIngestion = ({
cy.get('#name_help').should('be.visible').contains('Name is required');
// invalid name validation should work
cy.get('[data-testid="service-name"]').should('exist').type('!@#$%^&*()');
cy.get('[data-testid="service-name"]')
.should('exist')
.type(INVALID_NAMES.WITH_SPECIAL_CHARS);
cy.get('#name_help').should('be.visible').contains(NAME_VALIDATION_ERROR);
cy.get('[data-testid="service-name"]')

2
openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js
View File

@ -602,7 +602,7 @@ export const TAG_INVALID_NAMES = {
export const INVALID_NAMES = {
MAX_LENGTH:
'a87439625b1c2d3e4f5061728394a5b6c7d8e90a1b2c3d4e5f67890aba87439625b1c2d3e4f5061728394a5b6c7d8e90a1b2c3d4e5f67890abName can be a maximum of 128 characters',
WITH_SPECIAL_CHARS: '!@#$%^&*()',
WITH_SPECIAL_CHARS: '::normalName::',
};
export const NAME_VALIDATION_ERROR =

7
openmetadata-ui/src/main/resources/ui/src/antlr/EntityLinkSplitListener.js
View File

@ -25,12 +25,7 @@ export default class EntityLinkSplitListener extends EntityLinkListener {
}
// Enter a parse tree produced by EntityLinkParser#entityAttribute.
enterEntityAttribute(ctx) {
this.entityLinkParts.push(ctx.getText());
}
// Enter a parse tree produced by EntityLinkParser#entityFqn.
enterEntityFqn(ctx) {
enterNameOrFQN(ctx) {
this.entityLinkParts.push(ctx.getText());
}

4
openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx
View File

@ -170,11 +170,11 @@ export const AssetSelectionModal = ({
const fetchCurrentEntity = useCallback(async () => {
if (type === AssetsOfEntity.DOMAIN) {
const data = await getDomainByName(encodeURIComponent(entityFqn), '');
const data = await getDomainByName(getEncodedFqn(entityFqn), '');
setActiveEntity(data);
} else if (type === AssetsOfEntity.DATA_PRODUCT) {
const data = await getDataProductByName(
encodeURIComponent(entityFqn),
getEncodedFqn(entityFqn),
'domain,assets'
);
setActiveEntity(data);

3
openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx
View File

@ -80,6 +80,7 @@ import {
} from '../../../utils/RouterUtils';
import {
escapeESReservedCharacters,
getDecodedFqn,
getEncodedFqn,
} from '../../../utils/StringsUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
@ -105,7 +106,7 @@ const DataProductsDetailsPage = ({
tab: activeTab,
version,
} = useParams<{ fqn: string; tab: string; version: string }>();
const dataProductFqn = fqn ? decodeURIComponent(fqn) : '';
const dataProductFqn = fqn ? getDecodedFqn(fqn) : '';
const [dataProductPermission, setDataProductPermission] =
useState<OperationPermission>(DEFAULT_ENTITY_PERMISSION);
const [showActions, setShowActions] = useState(false);

5
openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsPage/DataProductsPage.component.tsx
View File

@ -33,6 +33,7 @@ import {
getDataProductVersionsPath,
getDomainPath,
} from '../../../utils/RouterUtils';
import { getDecodedFqn, getEncodedFqn } from '../../../utils/StringsUtils';
import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils';
import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder';
import EntityVersionTimeLine from '../../Entity/EntityVersionTimeLine/EntityVersionTimeLine';
@ -51,7 +52,7 @@ const DataProductsPage = () => {
);
const [selectedVersionData, setSelectedVersionData] = useState<DataProduct>();
const dataProductFqn = fqn ? decodeURIComponent(fqn) : '';
const dataProductFqn = fqn ? getDecodedFqn(fqn) : '';
const handleDataProductUpdate = async (updatedData: DataProduct) => {
if (dataProduct) {
@ -100,7 +101,7 @@ const DataProductsPage = () => {
setIsMainContentLoading(true);
try {
const data = await getDataProductByName(
encodeURIComponent(fqn),
getEncodedFqn(fqn),
'domain,owner,experts,assets'
);
setDataProduct(data);

5
openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuites/TestSuites.component.tsx
View File

@ -40,6 +40,7 @@ import {
} from '../../../rest/testAPI';
import { getEntityName } from '../../../utils/EntityUtils';
import { getTestSuitePath } from '../../../utils/RouterUtils';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder';
import FilterTablePlaceHolder from '../../common/ErrorWithPlaceholder/FilterTablePlaceHolder';
@ -83,7 +84,7 @@ export const TestSuites = ({ summaryPanel }: { summaryPanel: ReactNode }) => {
data-testid={name}
to={{
pathname: getTableTabPath(
encodeURIComponent(
getEncodedFqn(
record.executableEntityReference?.fullyQualifiedName ?? ''
),
EntityTabs.PROFILER
@ -99,7 +100,7 @@ export const TestSuites = ({ summaryPanel }: { summaryPanel: ReactNode }) => {
<Link
data-testid={name}
to={getTestSuitePath(
encodeURIComponent(record.fullyQualifiedName ?? record.name)
getEncodedFqn(record.fullyQualifiedName ?? record.name)
)}>
{getEntityName(record)}
</Link>

8
openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx
View File

@ -202,7 +202,7 @@ const DomainDetailsPage = ({
const res = await addDataProducts(data as CreateDataProduct);
history.push(
getDataProductsDetailsPath(
encodeURIComponent(res.fullyQualifiedName ?? '')
getEncodedFqn(res.fullyQualifiedName ?? '')
)
);
} catch (error) {
@ -229,7 +229,7 @@ const DomainDetailsPage = ({
const path = isVersionsView
? getDomainPath(domainFqn)
: getDomainVersionsPath(
encodeURIComponent(domainFqn),
getEncodedFqn(domainFqn),
toString(domain.version)
);
@ -300,9 +300,7 @@ const DomainDetailsPage = ({
fetchDomainAssets();
}
if (activeKey !== activeTab) {
history.push(
getDomainDetailsPath(encodeURIComponent(domainFqn), activeKey)
);
history.push(getDomainDetailsPath(getEncodedFqn(domainFqn), activeKey));
}
};

3
openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainLeftPanel/DomainLeftPanel.component.tsx
View File

@ -27,13 +27,14 @@ import { getEntityName } from '../../../utils/EntityUtils';
import Fqn from '../../../utils/Fqn';
import { checkPermission } from '../../../utils/PermissionsUtils';
import { getDomainPath } from '../../../utils/RouterUtils';
import { getDecodedFqn } from '../../../utils/StringsUtils';
import { DomainLeftPanelProps } from './DomainLeftPanel.interface';
const DomainsLeftPanel = ({ domains }: DomainLeftPanelProps) => {
const { t } = useTranslation();
const { permissions } = usePermissionProvider();
const { fqn } = useParams<{ fqn: string }>();
const domainFqn = fqn ? decodeURIComponent(fqn) : null;
const domainFqn = fqn ? getDecodedFqn(fqn) : null;
const history = useHistory();
const createDomainsPermission = useMemo(

5
openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainPage.component.tsx
View File

@ -29,6 +29,7 @@ import { Operation } from '../../generated/entity/policies/policy';
import { getDomainByName, patchDomains } from '../../rest/domainAPI';
import { checkPermission } from '../../utils/PermissionsUtils';
import { getDomainPath } from '../../utils/RouterUtils';
import { getDecodedFqn, getEncodedFqn } from '../../utils/StringsUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import './domain.less';
import DomainDetailsPage from './DomainDetailsPage/DomainDetailsPage.component';
@ -44,7 +45,7 @@ const DomainPage = () => {
useDomainProvider();
const [isMainContentLoading, setIsMainContentLoading] = useState(true);
const [activeDomain, setActiveDomain] = useState<Domain>();
const domainFqn = fqn ? decodeURIComponent(fqn) : null;
const domainFqn = fqn ? getDecodedFqn(fqn) : null;
const createDomainPermission = useMemo(
() => checkPermission(Operation.Create, ResourceEntity.DOMAIN, permissions),
@ -109,7 +110,7 @@ const DomainPage = () => {
setIsMainContentLoading(true);
try {
const data = await getDomainByName(
encodeURIComponent(fqn),
getEncodedFqn(fqn),
'children,owner,parent,experts'
);
setActiveDomain(data);

3
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx
View File

@ -70,6 +70,7 @@ import {
getGlossaryTermsVersionsPath,
getGlossaryVersionsPath,
} from '../../../utils/RouterUtils';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import SVGIcons, { Icons } from '../../../utils/SvgUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import { useAuthContext } from '../../Auth/AuthProviders/AuthProvider';
@ -182,7 +183,7 @@ const GlossaryHeader = ({
const handleGlossaryImport = () =>
history.push(
getGlossaryPathWithAction(
encodeURIComponent(selectedData.fullyQualifiedName ?? ''),
getEncodedFqn(selectedData.fullyQualifiedName ?? ''),
EntityAction.IMPORT
)
);

3
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx
View File

@ -38,6 +38,7 @@ import { getQueryFilterToExcludeTerm } from '../../../utils/GlossaryUtils';
import { getGlossaryTermsVersionsPath } from '../../../utils/RouterUtils';
import {
escapeESReservedCharacters,
getDecodedFqn,
getEncodedFqn,
} from '../../../utils/StringsUtils';
import { ActivityFeedTab } from '../../ActivityFeed/ActivityFeedTab/ActivityFeedTab.component';
@ -96,7 +97,7 @@ const GlossaryTermsV1 = ({
history.push({
pathname: version
? getGlossaryTermsVersionsPath(glossaryFqn, version, tab)
: getGlossaryTermDetailsPath(decodeURIComponent(glossaryFqn), tab),
: getGlossaryTermDetailsPath(getDecodedFqn(glossaryFqn), tab),
});
};

2
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx
View File

@ -294,7 +294,7 @@ const AssetsTabs = forwardRef(
const fetchCurrentEntity = useCallback(async () => {
let data;
const fqn = encodeURIComponent(entityFqn ?? '');
const fqn = getEncodedFqn(entityFqn ?? '');
switch (type) {
case AssetsOfEntity.DOMAIN:
data = await getDomainByName(fqn, '');

7
openmetadata-ui/src/main/resources/ui/src/components/Services/Services.tsx
View File

@ -50,6 +50,7 @@ import {
getResourceEntityFromServiceCategory,
getServiceTypesFromServiceCategory,
} from '../../utils/ServiceUtils';
import { getEncodedFqn } from '../../utils/StringsUtils';
import { FilterIcon } from '../../utils/TableUtils';
import { showErrorToast } from '../../utils/ToastUtils';
import ErrorPlaceHolder from '../common/ErrorWithPlaceholder/ErrorPlaceHolder';
@ -282,7 +283,7 @@ const Services = ({ serviceName }: ServicesProps) => {
className="max-two-lines"
data-testid={`service-name-${name}`}
to={getServiceDetailsPath(
encodeURIComponent(record.fullyQualifiedName ?? record.name),
getEncodedFqn(record.fullyQualifiedName ?? record.name),
serviceName
)}>
{getEntityName(record)}
@ -341,9 +342,7 @@ const Services = ({ serviceName }: ServicesProps) => {
<Link
className="no-underline"
to={getServiceDetailsPath(
encodeURIComponent(
service.fullyQualifiedName ?? service.name
),
getEncodedFqn(service.fullyQualifiedName ?? service.name),
serviceName
)}>
<Typography.Text

3
openmetadata-ui/src/main/resources/ui/src/components/Team/TeamDetails/TeamHierarchy.tsx
View File

@ -30,6 +30,7 @@ import { getTeamByName, updateTeam } from '../../../rest/teamsAPI';
import { Transi18next } from '../../../utils/CommonUtils';
import { getEntityName } from '../../../utils/EntityUtils';
import { getTeamsWithFqnPath } from '../../../utils/RouterUtils';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import { getTableExpandableConfig } from '../../../utils/TableUtils';
import { getMovedTeamData } from '../../../utils/TeamUtils';
import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils';
@ -59,7 +60,7 @@ const TeamHierarchy: FC<TeamHierarchyProps> = ({
<Link
className="link-hover"
to={getTeamsWithFqnPath(
encodeURIComponent(record.fullyQualifiedName || record.name)
getEncodedFqn(record.fullyQualifiedName || record.name)
)}>
{getEntityName(record)}
</Link>

63
openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.test.ts
View File

@ -10,7 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ENTITY_NAME_REGEX } from './regex.constants';
import { ENTITY_NAME_REGEX, TAG_NAME_REGEX } from './regex.constants';
describe('Test Regex', () => {
it('EntityName regex should pass for the valid entity name', () => {
@ -156,26 +156,57 @@ describe('Test Regex', () => {
});
it('EntityName regex should fail for the invalid entity name', () => {
// Contains letters, numbers, and # special characters.
expect(ENTITY_NAME_REGEX.test('HelloWorld123#')).toEqual(false);
// conatines :: in the name should fail
expect(ENTITY_NAME_REGEX.test('Hello::World')).toEqual(false);
});
// Contains letters, numbers, and $ special characters.
expect(ENTITY_NAME_REGEX.test('HelloWorld123$')).toEqual(false);
describe('TAG_NAME_REGEX', () => {
it('should match English letters', () => {
expect(TAG_NAME_REGEX.test('Hello')).toEqual(true);
});
// Contains letters, numbers, and ! special characters.
expect(ENTITY_NAME_REGEX.test('HelloWorld123!')).toEqual(false);
it('should match non-English letters', () => {
expect(TAG_NAME_REGEX.test('こんにちは')).toEqual(true);
});
// Contains letters, numbers, and @ special characters.
expect(ENTITY_NAME_REGEX.test('HelloWorld123@')).toEqual(false);
it('should match combined characters', () => {
expect(TAG_NAME_REGEX.test('é')).toEqual(true);
});
// Contains letters, numbers, and * special characters.
expect(ENTITY_NAME_REGEX.test('HelloWorld123*')).toEqual(false);
it('should match numbers', () => {
expect(TAG_NAME_REGEX.test('123')).toEqual(true);
});
// Contains letters, numbers, and special characters.
expect(ENTITY_NAME_REGEX.test('!@#$%^&*()')).toEqual(false);
it('should match underscores', () => {
expect(TAG_NAME_REGEX.test('_')).toEqual(true);
});
// Contains spanish characters and special characters.
expect(ENTITY_NAME_REGEX.test('¡Buenos días!')).toEqual(false);
expect(ENTITY_NAME_REGEX.test('¿Cómo estás?')).toEqual(false);
it('should match hyphens', () => {
expect(TAG_NAME_REGEX.test('-')).toEqual(true);
});
it('should match spaces', () => {
expect(TAG_NAME_REGEX.test(' ')).toEqual(true);
});
it('should match periods', () => {
expect(TAG_NAME_REGEX.test('.')).toEqual(true);
});
it('should match ampersands', () => {
expect(TAG_NAME_REGEX.test('&')).toEqual(true);
});
it('should match parentheses', () => {
expect(TAG_NAME_REGEX.test('()')).toEqual(true);
});
it('should not match other special characters', () => {
expect(TAG_NAME_REGEX.test('$')).toEqual(false);
});
it('should not match empty string', () => {
expect(TAG_NAME_REGEX.test('')).toEqual(false);
});
});
});

4
openmetadata-ui/src/main/resources/ui/src/constants/regex.constants.ts
View File

@ -24,7 +24,9 @@ export const FQN_REGEX = new RegExp(
* strings that contain a combination of letters, alphanumeric characters, hyphens,
* spaces, periods, single quotes, ampersands, and parentheses, with support for Unicode characters.
*/
export const ENTITY_NAME_REGEX = /^[\p{L}\p{M}\w\- .'&()%]+$/u;
export const ENTITY_NAME_REGEX = /^((?!::).)*$/;
export const TAG_NAME_REGEX = /^[\p{L}\p{M}\w\- .&()]+$/u;
export const delimiterRegex = /[\\[\]\\()\\;\\,\\|\\{}\\``\\/\\<>\\^]/g;
export const nameWithSpace = /\s/g;

3
openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryLeftPanel/GlossaryLeftPanel.component.tsx
View File

@ -28,13 +28,14 @@ import { getEntityName } from '../../../utils/EntityUtils';
import Fqn from '../../../utils/Fqn';
import { checkPermission } from '../../../utils/PermissionsUtils';
import { getGlossaryPath } from '../../../utils/RouterUtils';
import { getDecodedFqn } from '../../../utils/StringsUtils';
import { GlossaryLeftPanelProps } from './GlossaryLeftPanel.interface';
const GlossaryLeftPanel = ({ glossaries }: GlossaryLeftPanelProps) => {
const { t } = useTranslation();
const { permissions } = usePermissionProvider();
const { fqn: glossaryName } = useParams<{ fqn: string }>();
const glossaryFqn = glossaryName ? decodeURIComponent(glossaryName) : null;
const glossaryFqn = glossaryName ? getDecodedFqn(glossaryName) : null;
const history = useHistory();
const createGlossaryPermission = useMemo(

3
openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx
View File

@ -48,6 +48,7 @@ import {
import Fqn from '../../../utils/Fqn';
import { checkPermission } from '../../../utils/PermissionsUtils';
import { getGlossaryPath } from '../../../utils/RouterUtils';
import { getDecodedFqn } from '../../../utils/StringsUtils';
import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils';
import GlossaryLeftPanel from '../GlossaryLeftPanel/GlossaryLeftPanel.component';
@ -55,7 +56,7 @@ const GlossaryPage = () => {
const { t } = useTranslation();
const { permissions } = usePermissionProvider();
const { fqn: glossaryName } = useParams<{ fqn: string }>();
const glossaryFqn = decodeURIComponent(glossaryName);
const glossaryFqn = getDecodedFqn(glossaryName);
const history = useHistory();
const [glossaries, setGlossaries] = useState<Glossary[]>([]);
const [isLoading, setIsLoading] = useState(true);

5
openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/PoliciesListPage/PoliciesListPage.tsx
View File

@ -50,6 +50,7 @@ import {
getPolicyWithFqnPath,
getRoleWithFqnPath,
} from '../../../utils/RouterUtils';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import './policies-list.less';
@ -104,9 +105,7 @@ const PoliciesListPage = () => {
data-testid="policy-name"
to={
record.fullyQualifiedName
? getPolicyWithFqnPath(
encodeURIComponent(record.fullyQualifiedName)
)
? getPolicyWithFqnPath(getEncodedFqn(record.fullyQualifiedName))
: ''
}>
{getEntityName(record)}

3
openmetadata-ui/src/main/resources/ui/src/pages/RolesPage/RolesListPage/RolesListPage.tsx
View File

@ -51,6 +51,7 @@ import {
getPolicyWithFqnPath,
getRoleWithFqnPath,
} from '../../../utils/RouterUtils';
import { getEncodedFqn } from '../../../utils/StringsUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import './roles-list.less';
@ -105,7 +106,7 @@ const RolesListPage = () => {
className="link-hover"
data-testid="role-name"
to={getRoleWithFqnPath(
encodeURIComponent(record.fullyQualifiedName ?? '')
getEncodedFqn(record.fullyQualifiedName ?? '')
)}>
{getEntityName(record)}
</Link>

6
openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx
View File

@ -83,7 +83,7 @@ import {
import { defaultFields } from '../../utils/DatasetDetailsUtils';
import { getEntityName } from '../../utils/EntityUtils';
import { DEFAULT_ENTITY_PERMISSION } from '../../utils/PermissionsUtils';
import { getDecodedFqn } from '../../utils/StringsUtils';
import { getDecodedFqn, getEncodedFqn } from '../../utils/StringsUtils';
import { getTagsWithoutTier, getTierTags } from '../../utils/TableUtils';
import { createTagObject, updateTierTag } from '../../utils/TagsUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
@ -122,9 +122,9 @@ const TableDetailsPageV1 = () => {
const tableFqn = useMemo(
() =>
encodeURIComponent(
getEncodedFqn(
getPartialNameFromTableFQN(
decodeURIComponent(datasetFQN),
getDecodedFqn(datasetFQN),
[FqnPart.Service, FqnPart.Database, FqnPart.Schema, FqnPart.Table],
FQN_SEPARATOR_CHAR
)

4
openmetadata-ui/src/main/resources/ui/src/pages/TagsPage/TagsForm.tsx
View File

@ -16,8 +16,8 @@ import React, { useEffect, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { VALIDATION_MESSAGES } from '../../constants/constants';
import {
ENTITY_NAME_REGEX,
HEX_COLOR_CODE_REGEX,
TAG_NAME_REGEX,
} from '../../constants/regex.constants';
import { DEFAULT_FORM_VALUE } from '../../constants/Tags.constant';
import { FieldProp, FieldTypes } from '../../interface/FormUtils.interface';
@ -92,7 +92,7 @@ const TagsForm = ({
type: FieldTypes.TEXT,
rules: [
{
pattern: ENTITY_NAME_REGEX,
pattern: TAG_NAME_REGEX,
message: t('message.entity-name-validation'),
},
{

5
openmetadata-ui/src/main/resources/ui/src/pages/TeamsPage/TeamsPage.tsx
View File

@ -42,6 +42,7 @@ import {
import { updateUserDetail } from '../../rest/userAPI';
import { DEFAULT_ENTITY_PERMISSION } from '../../utils/PermissionsUtils';
import { getSettingPath, getTeamsWithFqnPath } from '../../utils/RouterUtils';
import { getDecodedFqn } from '../../utils/StringsUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import AddTeamForm from './AddTeamForm';
@ -126,7 +127,7 @@ const TeamsPage = () => {
const fetchAllTeamsBasicDetails = async (parentTeam?: string) => {
try {
const { data } = await getTeams(undefined, {
parentTeam: decodeURIComponent(parentTeam ?? '') ?? 'organization',
parentTeam: getDecodedFqn(parentTeam ?? '') ?? 'organization',
include: 'all',
});
@ -154,7 +155,7 @@ const TeamsPage = () => {
const { data } = await getTeams(
['userCount', 'childrenCount', 'owns', 'parents'],
{
parentTeam: decodeURIComponent(parentTeam ?? '') ?? 'organization',
parentTeam: getDecodedFqn(parentTeam ?? '') ?? 'organization',
include: 'all',
}
);

7
openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts
View File

@ -25,6 +25,7 @@ import { CSVImportResult } from '../generated/type/csvImportResult';
import { EntityHistory } from '../generated/type/entityHistory';
import { ListParams } from '../interface/API.interface';
import { getURLWithQueryFields } from '../utils/APIUtils';
import { getEncodedFqn } from '../utils/StringsUtils';
import APIClient from './index';
export type ListGlossaryTermsParams = ListParams & {
@ -130,7 +131,7 @@ export const getGlossaryTermByFQN = async (
arrQueryFields: string | string[] = ''
) => {
const url = getURLWithQueryFields(
`/glossaryTerms/name/${encodeURIComponent(glossaryTermFQN)}`,
`/glossaryTerms/name/${getEncodedFqn(glossaryTermFQN)}`,
arrQueryFields
);
@ -187,9 +188,7 @@ export const importGlossaryInCSVFormat = async (
headers: { 'Content-type': 'text/plain' },
};
const response = await APIClient.put<string, AxiosResponse<CSVImportResult>>(
`/glossaries/name/${encodeURIComponent(
glossaryName
)}/import?dryRun=${dryRun}`,
`/glossaries/name/${getEncodedFqn(glossaryName)}/import?dryRun=${dryRun}`,
data,
configOptions
);

3
openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts
View File

@ -27,6 +27,7 @@ import { EntityReference } from '../generated/type/entityReference';
import { Include } from '../generated/type/include';
import { Paging } from '../generated/type/paging';
import { getURLWithQueryFields } from '../utils/APIUtils';
import { getEncodedFqn } from '../utils/StringsUtils';
import APIClient from './index';
export type TableListParams = {
@ -234,7 +235,7 @@ export const getSampleDataByTableId = async (id: string) => {
};
export const getLatestTableProfileByFqn = async (fqn: string) => {
const encodedFQN = encodeURIComponent(fqn);
const encodedFQN = getEncodedFqn(fqn);
const response = await APIClient.get<Table>(
`${BASE_URL}/${encodedFQN}/tableProfile/latest`
);

6
openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts
View File

@ -132,7 +132,7 @@ export const getDomainPath = (fqn?: string) => {
let path = ROUTES.DOMAIN;
if (fqn) {
path = ROUTES.DOMAIN_DETAILS;
path = path.replace(PLACEHOLDER_ROUTE_FQN, encodeURIComponent(fqn));
path = path.replace(PLACEHOLDER_ROUTE_FQN, getEncodedFqn(fqn));
}
return path;
@ -166,7 +166,7 @@ export const getGlossaryPath = (fqn?: string) => {
let path = ROUTES.GLOSSARY;
if (fqn) {
path = ROUTES.GLOSSARY_DETAILS;
path = path.replace(PLACEHOLDER_ROUTE_FQN, encodeURIComponent(fqn));
path = path.replace(PLACEHOLDER_ROUTE_FQN, getEncodedFqn(fqn));
}
return path;
@ -514,7 +514,7 @@ export const getGlossaryTermsVersionsPath = (
? ROUTES.GLOSSARY_TERMS_VERSION_TAB
: ROUTES.GLOSSARY_TERMS_VERSION;
path = path
.replace(PLACEHOLDER_ROUTE_FQN, encodeURIComponent(glossaryTermsFQN))
.replace(PLACEHOLDER_ROUTE_FQN, getEncodedFqn(glossaryTermsFQN))
.replace(PLACEHOLDER_ROUTE_VERSION, version);
if (tab) {