yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2026-01-05 06:16:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): add ingestion source for SAP Analytics Cloud (#10958)

Browse Source 
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
Felix Lüdin 2024-08-26 20:29:15 +02:00 committed by GitHub
parent 99824b4972
commit ce99bc4f22
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 2587 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
View File

@ -83,7 +83,7 @@ import {
PROJECT_NAME,
} from './lookml';
import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, SAC, UNITY_CATALOG, VERTICA } from '../constants';
import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@ -171,6 +171,20 @@ import {
USER_ALLOW,
USER_DENY,
} from './azure';
import {
SAC_TENANT_URL,
SAC_TOKEN_URL,
SAC_CLIENT_ID,
SAC_CLIENT_SECRET,
INGEST_STORIES,
INGEST_APPLICATIONS,
RESOURCE_ID_ALLOW,
RESOURCE_ID_DENY,
RESOURCE_NAME_ALLOW,
RESOURCE_NAME_DENY,
FOLDER_ALLOW,
FOLDER_DENY,
} from './sac';
export enum RecipeSections {
Connection = 0,
@ -519,8 +533,29 @@ export const RECIPE_FIELDS: RecipeFields = {
filterFields: [GROUP_ALLOW, GROUP_DENY, USER_ALLOW, USER_DENY],
advancedFields: [AZURE_INGEST_USERS, AZURE_INGEST_GROUPS, STATEFUL_INGESTION_ENABLED, SKIP_USERS_WITHOUT_GROUP],
},
[SAC]: {
fields: [SAC_TENANT_URL, SAC_TOKEN_URL, SAC_CLIENT_ID, SAC_CLIENT_SECRET],
filterFields: [
INGEST_STORIES,
INGEST_APPLICATIONS,
RESOURCE_ID_ALLOW,
RESOURCE_ID_DENY,
RESOURCE_NAME_ALLOW,
RESOURCE_NAME_DENY,
FOLDER_ALLOW,
FOLDER_DENY,
],
advancedFields: [STATEFUL_INGESTION_ENABLED],
},
};
export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));
export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY, UNITY_CATALOG]);
export const CONNECTORS_WITH_TEST_CONNECTION = new Set([
SNOWFLAKE,
LOOKER,
BIGQUERY_BETA,
BIGQUERY,
UNITY_CATALOG,
SAC,
]);

161
datahub-web-react/src/app/ingest/source/builder/RecipeForm/sac.ts Normal file
View File

@ -0,0 +1,161 @@
import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
export const SAC_TENANT_URL: RecipeField = {
name: 'tenant_url',
label: 'Tenant URL',
tooltip: 'The URL of the SAP Analytics Cloud tenant.',
type: FieldType.TEXT,
fieldPath: 'source.config.tenant_url',
placeholder: 'https://company.eu10.sapanalytics.cloud',
required: true,
rules: null,
};
export const SAC_TOKEN_URL: RecipeField = {
name: 'token_url',
label: 'Token URL',
tooltip: 'The OAuth 2.0 Token Service URL.',
type: FieldType.TEXT,
fieldPath: 'source.config.token_url',
placeholder: 'https://company.eu10.hana.ondemand.com/oauth/token',
required: true,
rules: null,
};
export const SAC_CLIENT_ID: RecipeField = {
name: 'client_id',
label: 'Client ID',
tooltip: 'Client ID.',
type: FieldType.SECRET,
fieldPath: 'source.config.client_id',
placeholder: 'client_id',
required: true,
rules: null,
};
export const SAC_CLIENT_SECRET: RecipeField = {
name: 'client_secret',
label: 'Client Secret',
tooltip: 'Client Secret.',
type: FieldType.SECRET,
fieldPath: 'source.config.client_secret',
placeholder: 'client_secret',
required: true,
rules: null,
};
export const INGEST_STORIES: RecipeField = {
name: 'ingest_stories',
label: 'Ingest Stories',
tooltip: 'Whether stories should be ingested into DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_stories',
rules: null,
section: 'Stories and Applications',
};
export const INGEST_APPLICATIONS: RecipeField = {
name: 'ingest_applications',
label: 'Ingest Applications',
tooltip: 'Whether applications should be ingested into DataHub.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.ingest_applications',
rules: null,
section: 'Stories and Applications',
};
const resourceIdAllowFieldPath = 'source.config.resource_id_pattern.allow';
export const RESOURCE_ID_ALLOW: RecipeField = {
name: 'resource_id_pattern.allow',
label: 'Resource Id Allow Patterns',
tooltip:
'Only include specific Stories and Applications by providing the id of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: resourceIdAllowFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, resourceIdAllowFieldPath),
};
const resourceIdDenyFieldPath = 'source.config.resource_id_pattern.deny';
export const RESOURCE_ID_DENY: RecipeField = {
name: 'resource_id_pattern.deny',
label: 'Resource Id Deny Patterns',
tooltip:
'Exclude specific Stories and Applications by providing the id of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: resourceIdDenyFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, resourceIdDenyFieldPath),
};
const resourceNameAllowFieldPath = 'source.config.resource_id_pattern.allow';
export const RESOURCE_NAME_ALLOW: RecipeField = {
name: 'resource_name_pattern.allow',
label: 'Resource Name Allow Patterns',
tooltip:
'Only include specific Stories and Applications by providing the name of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: resourceNameAllowFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'Name of the story',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, resourceNameAllowFieldPath),
};
const resourceNameDenyFieldPath = 'source.config.resource_name_pattern.deny';
export const RESOURCE_NAME_DENY: RecipeField = {
name: 'resource_name_pattern.deny',
label: 'Resource Name Deny Patterns',
tooltip:
'Exclude specific Stories and Applications by providing the name of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: resourceNameDenyFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'Name of the story',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, resourceNameDenyFieldPath),
};
const folderAllowFieldPath = 'source.config.resource_id_pattern.allow';
export const FOLDER_ALLOW: RecipeField = {
name: 'folder_pattern.allow',
label: 'Folder Allow Patterns',
tooltip:
'Only include specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: folderAllowFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'Folder of the story',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, folderAllowFieldPath),
};
const folderDenyFieldPath = 'source.config.folder_pattern.deny';
export const FOLDER_DENY: RecipeField = {
name: 'folder_pattern.deny',
label: 'Folder Deny Patterns',
tooltip:
'Exclude specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: folderDenyFieldPath,
rules: null,
section: 'Stories and Applications',
placeholder: 'Folder of the story',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, folderDenyFieldPath),
};

12
datahub-web-react/src/app/ingest/source/builder/SelectTemplateStep.tsx
View File

@ -104,6 +104,18 @@ export const SelectTemplateStep = ({ state, updateState, goTo, cancel, ingestion
source.name.toLocaleLowerCase().includes(searchFilter.toLocaleLowerCase()),
);
filteredSources.sort((a, b) => {
if (a.name === 'custom') {
return 1;
}
if (b.name === 'custom') {
return -1;
}
return a.displayName.localeCompare(b.displayName);
});
return (
<Container>
<Section>

4
datahub-web-react/src/app/ingest/source/builder/constants.ts
View File

@ -34,6 +34,7 @@ import fivetranLogo from '../../../../images/fivetranlogo.png';
import csvLogo from '../../../../images/csv-logo.png';
import qlikLogo from '../../../../images/qliklogo.png';
import sigmaLogo from '../../../../images/sigmalogo.png';
import sacLogo from '../../../../images/saclogo.svg';
export const ATHENA = 'athena';
export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@ -122,6 +123,8 @@ export const QLIK_SENSE = 'qlik-sense';
export const QLIK_SENSE_URN = `urn:li:dataPlatform:${QLIK_SENSE}`;
export const SIGMA = 'sigma';
export const SIGMA_URN = `urn:li:dataPlatform:${SIGMA}`;
export const SAC = 'sac';
export const SAC_URN = `urn:li:dataPlatform:${SAC}`;
export const PLATFORM_URN_TO_LOGO = {
[ATHENA_URN]: athenaLogo,
@ -161,6 +164,7 @@ export const PLATFORM_URN_TO_LOGO = {
[CSV_URN]: csvLogo,
[QLIK_SENSE_URN]: qlikLogo,
[SIGMA_URN]: sigmaLogo,
[SAC_URN]: sacLogo,
};
export const SOURCE_TO_PLATFORM_URN = {

8
datahub-web-react/src/app/ingest/source/builder/sources.json
View File

@ -287,6 +287,14 @@
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'",
"recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH"
},
{
"urn": "urn:li:dataPlatform:sac",
"name": "sac",
"displayName": "SAP Analytics Cloud",
"description": "Import Stories, Applications and Models from SAP Analytics Cloud.",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/sac/",
"recipe": "source:\n type: sac\n config:\n tenant_url: # Your SAP Analytics Cloud tenant URL, e.g. https://company.eu10.sapanalytics.cloud or https://company.eu10.hcs.cloud.sap\n token_url: # The Token URL of your SAP Analytics Cloud tenant, e.g. https://company.eu10.hana.ondemand.com/oauth/token.\n\n # Add secret in Secrets Tab with relevant names for each variable\n client_id: \"${SAC_CLIENT_ID}\" # Your SAP Analytics Cloud client id\n client_secret: \"${SAC_CLIENT_SECRET}\" # Your SAP Analytics Cloud client secret"
},
{
"urn": "urn:li:dataPlatform:custom",
"name": "custom",

26
datahub-web-react/src/app/ingest/source/conf/sac/sac.ts Normal file
View File

@ -0,0 +1,26 @@
import { SourceConfig } from '../types';
import sacLogo from '../../../../../images/saclogo.svg';
const placeholderRecipe = `\
source:
type: sac
config:
tenant_url: # Your SAP Analytics Cloud tenant URL, e.g. https://company.eu10.sapanalytics.cloud or https://company.eu10.hcs.cloud.sap
token_url: # The Token URL of your SAP Analytics Cloud tenant, e.g. https://company.eu10.hana.ondemand.com/oauth/token.
# Add secret in Secrets Tab with relevant names for each variable
client_id: "\${SAC_CLIENT_ID}" # Your SAP Analytics Cloud client id
client_secret: "\${SAC_CLIENT_SECRET}" # Your SAP Analytics Cloud client secret
`;
export const SAC = 'sac';
const sacConfig: SourceConfig = {
type: SAC,
placeholderRecipe,
displayName: 'SAP Analytics Cloud',
docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/sac/',
logoUrl: sacLogo,
};
export default sacConfig;

2
datahub-web-react/src/app/ingest/source/conf/sources.tsx
View File

@ -17,6 +17,7 @@ import hiveConfig from './hive/hive';
import oracleConfig from './oracle/oracle';
import tableauConfig from './tableau/tableau';
import csvConfig from './csv/csv';
import sacConfig from './sac/sac';
const baseUrl = window.location.origin;
@ -48,6 +49,7 @@ export const SOURCE_TEMPLATE_CONFIGS: Array<SourceConfig> = [
oracleConfig,
hiveConfig,
csvConfig,
sacConfig,
{
type: 'custom',
placeholderRecipe: DEFAULT_PLACEHOLDER_RECIPE,

23
datahub-web-react/src/app/shared/getLogoFromPlatform.tsx
View File

@ -1,23 +0,0 @@
import lookerLogo from '../../images/lookerlogo.png';
import supersetLogo from '../../images/supersetlogo.png';
import airflowLogo from '../../images/airflowlogo.png';
import redashLogo from '../../images/redashlogo.png';
/**
* TODO: This is a temporary solution, until the backend can push logos for all data platform types.
*/
export function getLogoFromPlatform(platform: string) {
if (platform.toLowerCase() === 'looker') {
return lookerLogo;
}
if (platform.toLowerCase() === 'superset') {
return supersetLogo;
}
if (platform.toLowerCase() === 'airflow') {
return airflowLogo;
}
if (platform.toLowerCase() === 'redash') {
return redashLogo;
}
return undefined;
}

19
datahub-web-react/src/images/saclogo.svg Normal file
View File

@ -0,0 +1,19 @@
<svg width="256" height="256" viewBox="10 10 28 28" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip1_3169_9404)">
<path d="M19.3121 12.7918C20.9626 12.1015 22.7391 11.7867 24.5142 11.8619C24.537 11.8628 24.5476 11.8677 24.5539 11.8711C24.5621 11.8756 24.5762 11.8858 24.5919 11.9069C24.6253 11.9519 24.6528 12.0308 24.6363 12.1286L22.8975 22.4223C22.6578 23.8417 23.4472 25.2344 24.7881 25.7578L34.513 29.5538C34.6054 29.5899 34.6589 29.6541 34.6804 29.7058C34.6905 29.7301 34.692 29.7474 34.6916 29.7568C34.6914 29.764 34.6901 29.7756 34.6792 29.7956C33.8318 31.3572 32.6491 32.7197 31.2091 33.7811C29.3599 35.1441 27.1623 35.955 24.8711 36.1197C22.5798 36.2844 20.2888 35.7961 18.2637 34.7116C16.2387 33.6271 14.5626 31.9907 13.4299 29.9922C12.2971 27.9937 11.7541 25.7151 11.8639 23.4205C11.9736 21.126 12.7316 18.9096 14.0499 17.0283C15.3682 15.1471 17.1928 13.6781 19.3121 12.7918Z" stroke="url(#paint1_linear_3169_9404)" stroke-width="2.7"/>
<path fill-rule="evenodd" clip-rule="evenodd" d="M35.3878 25.3829C36.3778 25.7644 37.4764 25.1449 37.4976 24.0841C37.5056 23.6807 37.495 23.2764 37.4656 22.8723C37.322 20.899 36.7332 18.982 35.7421 17.2613C34.751 15.5405 33.3827 14.0592 31.7368 12.9254C31.3966 12.6911 31.0463 12.4728 30.6872 12.2711C29.7615 11.7512 28.6668 12.3792 28.489 13.4259L27.2378 20.7923C27.1034 21.5839 27.5454 22.3607 28.2947 22.6495L35.3878 25.3829Z" fill="url(#paint2_linear_3169_9404)"/>
</g>
<defs>
<linearGradient id="paint1_linear_3169_9404" x1="14.3288" y1="14.5387" x2="33.2053" y2="32.4019" gradientUnits="userSpaceOnUse">
<stop stop-color="#0195FF"/>
<stop offset="0.991028" stop-color="#1147E9"/>
</linearGradient>
<linearGradient id="paint2_linear_3169_9404" x1="28.7346" y1="14.058" x2="37.8097" y2="21.0485" gradientUnits="userSpaceOnUse">
<stop stop-color="#1348FF"/>
<stop offset="1" stop-color="#06238D"/>
</linearGradient>
<clipPath id="clip1_3169_9404">
<rect width="27" height="27" fill="white" transform="translate(10.5 10.501)"/>
</clipPath>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.0 KiB

45
metadata-ingestion/docs/sources/sac/sac_pre.md Normal file
View File

@ -0,0 +1,45 @@
## Configuration Notes
1. Refer to [Manage OAuth Clients](https://help.sap.com/docs/SAP_ANALYTICS_CLOUD/00f68c2e08b941f081002fd3691d86a7/4f43b54398fc4acaa5efa32badfe3df6.html) to create an OAuth client in SAP Analytics Cloud. The OAuth client is required to have the following properties:
- Purpose: API Access
- Access:
- Data Import Service
- Authorization Grant: Client Credentials
2. Maintain connection mappings (optional):
To map individual connections in SAP Analytics Cloud to platforms, platform instances and environments, the `connection_mapping` configuration can be used within the recipe:
```yaml
connection_mapping:
MY_BW_CONNECTION:
platform: bw
platform_instance: PROD_BW
env: PROD
MY_HANA_CONNECTION:
platform: hana
platform_instance: PROD_HANA
env: PROD
```
The key in the connection mapping dictionary represents the name of the connection created in SAP Analytics Cloud.
## Concept mapping
| SAP Analytics Cloud | DataHub |
|-----------------------|---------------------|
| `Story` | `Dashboard` |
| `Application` | `Dashboard` |
| `Live Data Model` | `Dataset` |
| `Import Data Model` | `Dataset` |
| `Model` | `Dataset` |
## Limitations
- Only models which are used in a Story or an Application will be ingested because there is no dedicated API to retrieve models (only for Stories and Applications).
- Browse Paths for models cannot be created because the folder where the models are saved is not returned by the API.
- Schema metadata is only ingested for Import Data Models because there is no possibility to get the schema metadata of the other model types.
- Lineages for Import Data Models cannot be ingested because the API is not providing any information about it.
- Currently, only SAP BW and SAP HANA are supported for ingesting the upstream lineages of Live Data Models - a warning is logged for all other connection types, please feel free to open an [issue on GitHub](https://github.com/datahub-project/datahub/issues/new/choose) with the warning message to have this fixed.
- For some models (e.g., builtin models) it cannot be detected whether the models are Live Data or Import Data Models. Therefore, these models will be ingested only with the `Story` subtype.

40
metadata-ingestion/docs/sources/sac/sac_recipe.yml Normal file
View File

@ -0,0 +1,40 @@
source:
type: sac
config:
stateful_ingestion:
enabled: true
tenant_url: # Your SAP Analytics Cloud tenant URL, e.g. https://company.eu10.sapanalytics.cloud or https://company.eu10.hcs.cloud.sap
token_url: # The Token URL of your SAP Analytics Cloud tenant, e.g. https://company.eu10.hana.ondemand.com/oauth/token.
# Add secret in Secrets Tab with relevant names for each variable
client_id: "${SAC_CLIENT_ID}" # Your SAP Analytics Cloud client id
client_secret: "${SAC_CLIENT_SECRET}" # Your SAP Analytics Cloud client secret
# ingest stories
ingest_stories: true
# ingest applications
ingest_applications: true
resource_id_pattern:
allow:
- .*
resource_name_pattern:
allow:
- .*
folder_pattern:
allow:
- .*
connection_mapping:
MY_BW_CONNECTION:
platform: bw
platform_instance: PROD_BW
env: PROD
MY_HANA_CONNECTION:
platform: hana
platform_instance: PROD_HANA
env: PROD

9
metadata-ingestion/setup.py
View File

@ -314,6 +314,12 @@ databricks = {
mysql = sql_common | {"pymysql>=1.0.2"}
sac = {
"requests",
"pyodata>=1.11.1",
"Authlib",
}
# Note: for all of these, framework_common will be added.
plugins: Dict[str, Set[str]] = {
# Sink plugins.
@ -480,6 +486,7 @@ plugins: Dict[str, Set[str]] = {
"fivetran": snowflake_common | bigquery_common | sqlglot_lib,
"qlik-sense": sqlglot_lib | {"requests", "websocket-client"},
"sigma": sqlglot_lib | {"requests"},
"sac": sac,
}
# This is mainly used to exclude plugins from the Docker image.
@ -620,6 +627,7 @@ base_dev_requirements = {
"kafka-connect",
"qlik-sense",
"sigma",
"sac",
]
if plugin
for dependency in plugins[plugin]
@ -735,6 +743,7 @@ entry_points = {
"fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource",
"qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource",
"sigma = datahub.ingestion.source.sigma.sigma:SigmaSource",
"sac = datahub.ingestion.source.sac.sac:SACSource",
],
"datahub.ingestion.transformer.plugins": [
"pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership",

7
metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
View File

@ -18,6 +18,9 @@ class DatasetSubTypes(str, Enum):
QLIK_DATASET = "Qlik Dataset"
BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot"
SIGMA_DATASET = "Sigma Dataset"
SAC_MODEL = "Model"
SAC_IMPORT_DATA_MODEL = "Import Data Model"
SAC_LIVE_DATA_MODEL = "Live Data Model"
# TODO: Create separate entity...
NOTEBOOK = "Notebook"
@ -71,3 +74,7 @@ class BIAssetSubTypes(str, Enum):
MODE_REPORT = "Report"
MODE_QUERY = "Query"
MODE_CHART = "Chart"
# SAP Analytics Cloud
SAC_STORY = "Story"
SAC_APPLICATION = "Application"

0
metadata-ingestion/src/datahub/ingestion/source/sac/__init__.py Normal file
View File

775
metadata-ingestion/src/datahub/ingestion/source/sac/sac.py Normal file
View File

@ -0,0 +1,775 @@
import json
import logging
from dataclasses import dataclass
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
import pyodata
import pyodata.v2.model
import pyodata.v2.service
from authlib.integrations.requests_client import OAuth2Session
from pydantic import Field, SecretStr, validator
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from datahub.configuration.common import AllowDenyPattern
from datahub.configuration.source_common import (
DEFAULT_ENV,
DatasetSourceConfigMixin,
EnvConfigMixin,
)
from datahub.emitter.mce_builder import (
dataset_urn_to_key,
make_dashboard_urn,
make_data_platform_urn,
make_dataplatform_instance_urn,
make_dataset_urn_with_platform_instance,
make_user_urn,
)
from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import (
SupportStatus,
capability,
config_class,
platform_name,
support_status,
)
from datahub.ingestion.api.incremental_lineage_helper import (
IncrementalLineageConfigMixin,
auto_incremental_lineage,
)
from datahub.ingestion.api.source import (
CapabilityReport,
MetadataWorkUnitProcessor,
SourceCapability,
TestableSource,
TestConnectionReport,
)
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import BIAssetSubTypes, DatasetSubTypes
from datahub.ingestion.source.sac.sac_common import (
ImportDataModelColumn,
Resource,
ResourceModel,
)
from datahub.ingestion.source.state.stale_entity_removal_handler import (
StaleEntityRemovalHandler,
StaleEntityRemovalSourceReport,
StatefulStaleMetadataRemovalConfig,
)
from datahub.ingestion.source.state.stateful_ingestion_base import (
StatefulIngestionConfigBase,
StatefulIngestionSourceBase,
)
from datahub.metadata.schema_classes import (
AuditStampClass,
BrowsePathEntryClass,
BrowsePathsClass,
BrowsePathsV2Class,
ChangeAuditStampsClass,
DashboardInfoClass,
DataPlatformInstanceClass,
DatasetLineageTypeClass,
DatasetPropertiesClass,
DateTypeClass,
NullTypeClass,
NumberTypeClass,
SchemaFieldClass,
SchemaFieldDataTypeClass,
SchemalessClass,
SchemaMetadataClass,
StatusClass,
StringTypeClass,
SubTypesClass,
UpstreamClass,
UpstreamLineageClass,
)
from datahub.utilities import config_clean
logger = logging.getLogger(__name__)
class ConnectionMappingConfig(EnvConfigMixin):
platform: Optional[str] = Field(
default=None, description="The platform that this connection mapping belongs to"
)
platform_instance: Optional[str] = Field(
default=None,
description="The instance of the platform that this connection mapping belongs to",
)
env: str = Field(
default=DEFAULT_ENV,
description="The environment that this connection mapping belongs to",
)
class SACSourceConfig(
StatefulIngestionConfigBase, DatasetSourceConfigMixin, IncrementalLineageConfigMixin
):
stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
default=None,
description="Stateful ingestion related configs",
)
tenant_url: str = Field(description="URL of the SAP Analytics Cloud tenant")
token_url: str = Field(
description="URL of the OAuth token endpoint of the SAP Analytics Cloud tenant"
)
client_id: str = Field(description="Client ID for the OAuth authentication")
client_secret: SecretStr = Field(
description="Client secret for the OAuth authentication"
)
ingest_stories: bool = Field(
default=True,
description="Controls whether Stories should be ingested",
)
ingest_applications: bool = Field(
default=True,
description="Controls whether Analytic Applications should be ingested",
)
ingest_import_data_model_schema_metadata: bool = Field(
default=True,
description="Controls whether schema metadata of Import Data Models should be ingested (ingesting schema metadata of Import Data Models significantly increases overall ingestion time)",
)
resource_id_pattern: AllowDenyPattern = Field(
AllowDenyPattern.allow_all(),
description="Patterns for selecting resource ids that are to be included",
)
resource_name_pattern: AllowDenyPattern = Field(
AllowDenyPattern.allow_all(),
description="Patterns for selecting resource names that are to be included",
)
folder_pattern: AllowDenyPattern = Field(
AllowDenyPattern.allow_all(),
description="Patterns for selecting folders that are to be included",
)
connection_mapping: Dict[str, ConnectionMappingConfig] = Field(
default={}, description="Custom mappings for connections"
)
query_name_template: Optional[str] = Field(
default="QUERY/{name}",
description="Template for generating dataset urns of consumed queries, the placeholder {query} can be used within the template for inserting the name of the query",
)
@validator("tenant_url", "token_url")
def remove_trailing_slash(cls, v):
return config_clean.remove_trailing_slashes(v)
@dataclass
class SACSourceReport(StaleEntityRemovalSourceReport):
pass
@platform_name("SAP Analytics Cloud", id="sac")
@config_class(SACSourceConfig)
@support_status(SupportStatus.TESTING)
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
@capability(
SourceCapability.LINEAGE_COARSE,
"Enabled by default (only for Live Data Models)",
)
@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
@capability(
SourceCapability.SCHEMA_METADATA,
"Enabled by default (only for Import Data Models)",
)
class SACSource(StatefulIngestionSourceBase, TestableSource):
config: SACSourceConfig
report: SACSourceReport
platform = "sac"
session: OAuth2Session
client: pyodata.Client
ingested_dataset_entities: Set[str] = set()
ingested_upstream_dataset_keys: Set[str] = set()
def __init__(self, config: SACSourceConfig, ctx: PipelineContext):
super().__init__(config, ctx)
self.config = config
self.report = SACSourceReport()
self.session, self.client = SACSource.get_sac_connection(self.config)
def close(self) -> None:
self.session.close()
super().close()
@classmethod
def create(cls, config_dict: dict, ctx: PipelineContext) -> "SACSource":
config = SACSourceConfig.parse_obj(config_dict)
return cls(config, ctx)
@staticmethod
def test_connection(config_dict: dict) -> TestConnectionReport:
test_report = TestConnectionReport()
try:
config = SACSourceConfig.parse_obj(config_dict)
# when creating the pyodata.Client, the metadata is automatically parsed and validated
session, _ = SACSource.get_sac_connection(config)
# test the Data Import Service separately here, because it requires specific properties when configuring the OAuth client
response = session.get(url=f"{config.tenant_url}/api/v1/dataimport/models")
response.raise_for_status()
session.close()
test_report.basic_connectivity = CapabilityReport(capable=True)
except Exception as e:
test_report.basic_connectivity = CapabilityReport(
capable=False, failure_reason=f"{e}"
)
return test_report
def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
return [
*super().get_workunit_processors(),
partial(
auto_incremental_lineage,
self.config.incremental_lineage,
),
StaleEntityRemovalHandler.create(
self, self.config, self.ctx
).workunit_processor,
]
def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
if self.config.ingest_stories or self.config.ingest_applications:
resources = self.get_resources()
for resource in resources:
datasets = []
for resource_model in resource.resource_models:
dataset_urn = make_dataset_urn_with_platform_instance(
platform=self.platform,
name=f"{resource_model.namespace}:{resource_model.model_id}",
platform_instance=self.config.platform_instance,
env=self.config.env,
)
if dataset_urn not in datasets:
datasets.append(dataset_urn)
if dataset_urn in self.ingested_dataset_entities:
continue
self.ingested_dataset_entities.add(dataset_urn)
yield from self.get_model_workunits(dataset_urn, resource_model)
yield from self.get_resource_workunits(resource, datasets)
def get_report(self) -> SACSourceReport:
return self.report
def get_resource_workunits(
self, resource: Resource, datasets: List[str]
) -> Iterable[MetadataWorkUnit]:
dashboard_urn = make_dashboard_urn(
platform=self.platform,
name=resource.resource_id,
platform_instance=self.config.platform_instance,
)
if resource.ancestor_path:
mcp = MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=BrowsePathsClass(
paths=[
f"/{self.platform}/{resource.ancestor_path}",
],
),
)
yield mcp.as_workunit()
mcp = MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=BrowsePathsV2Class(
path=[
BrowsePathEntryClass(id=folder_name)
for folder_name in resource.ancestor_path.split("/")
],
),
)
yield mcp.as_workunit()
if self.config.platform_instance is not None:
mcp = MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=DataPlatformInstanceClass(
platform=make_data_platform_urn(self.platform),
instance=make_dataplatform_instance_urn(
self.platform, self.config.platform_instance
),
),
)
yield mcp.as_workunit()
mcp = MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=DashboardInfoClass(
title=resource.name,
description=resource.description,
lastModified=ChangeAuditStampsClass(
created=AuditStampClass(
time=round(resource.created_time.timestamp() * 1000),
actor=make_user_urn(resource.created_by)
if resource.created_by
else "urn:li:corpuser:unknown",
),
lastModified=AuditStampClass(
time=round(resource.modified_time.timestamp() * 1000),
actor=make_user_urn(resource.modified_by)
if resource.modified_by
else "urn:li:corpuser:unknown",
),
),
customProperties={
"resourceType": resource.resource_type,
"resourceSubtype": resource.resource_subtype,
"storyId": resource.story_id,
"isMobile": str(resource.is_mobile),
},
datasets=sorted(datasets) if datasets else None,
externalUrl=f"{self.config.tenant_url}{resource.open_url}",
),
)
yield mcp.as_workunit()
type_name: Optional[str] = None
if resource.resource_subtype == "":
type_name = BIAssetSubTypes.SAC_STORY
elif resource.resource_subtype == "APPLICATION":
type_name = BIAssetSubTypes.SAC_APPLICATION
if type_name:
mcp = MetadataChangeProposalWrapper(
entityUrn=dashboard_urn,
aspect=SubTypesClass(
typeNames=[type_name],
),
)
yield mcp.as_workunit()
def get_model_workunits(
self, dataset_urn: str, model: ResourceModel
) -> Iterable[MetadataWorkUnit]:
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=DatasetPropertiesClass(
name=model.name,
description=model.description,
customProperties={
"namespace": model.namespace,
"modelId": model.model_id,
"isImport": "true" if model.is_import else "false",
},
externalUrl=f"{self.config.tenant_url}/sap/fpa/ui/tenants/3c44c#view_id=model;model_id={model.namespace}:{model.model_id}",
),
)
yield mcp.as_workunit()
if model.is_import and self.config.ingest_import_data_model_schema_metadata:
primary_fields: List[str] = []
schema_fields: List[SchemaFieldClass] = []
columns = self.get_import_data_model_columns(model_id=model.model_id)
for column in columns:
schema_field = SchemaFieldClass(
fieldPath=column.name,
type=self.get_schema_field_data_type(column),
nativeDataType=self.get_schema_field_native_data_type(column),
description=column.description,
isPartOfKey=column.is_key,
)
schema_fields.append(schema_field)
if column.is_key:
primary_fields.append(column.name)
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=SchemaMetadataClass(
schemaName=model.model_id,
platform=make_data_platform_urn(self.platform),
version=0,
hash="",
platformSchema=SchemalessClass(),
fields=schema_fields,
primaryKeys=primary_fields,
),
)
yield mcp.as_workunit()
if model.system_type in ("BW", "HANA") and model.external_id is not None:
upstream_dataset_name: Optional[str] = None
if model.system_type == "BW" and model.external_id.startswith(
"query:"
): # query:[][][query]
query = model.external_id[11:-1]
upstream_dataset_name = self.get_query_name(query)
elif model.system_type == "HANA" and model.external_id.startswith(
"view:"
): # view:[schema][schema.namespace][view]
schema, namespace_with_schema, view = model.external_id.split("][", 2)
schema = schema[6:]
namespace: Optional[str] = None
if len(schema) < len(namespace_with_schema):
namespace = namespace_with_schema[len(f"{schema}.") :]
view = view[:-1]
upstream_dataset_name = self.get_view_name(schema, namespace, view)
if upstream_dataset_name is not None:
if model.connection_id in self.config.connection_mapping:
connection = self.config.connection_mapping[model.connection_id]
platform = (
connection.platform
if connection.platform
else model.system_type.lower()
)
platform_instance = connection.platform_instance
env = connection.env
else:
platform = model.system_type.lower()
platform_instance = model.connection_id
env = DEFAULT_ENV
logger.info(
f"No connection mapping found for connection with id {model.connection_id}, connection id will be used as platform instance"
)
upstream_dataset_urn = make_dataset_urn_with_platform_instance(
platform=platform,
name=upstream_dataset_name,
platform_instance=platform_instance,
env=env,
)
if upstream_dataset_urn not in self.ingested_upstream_dataset_keys:
mcp = MetadataChangeProposalWrapper(
entityUrn=upstream_dataset_urn,
aspect=dataset_urn_to_key(upstream_dataset_urn),
)
yield mcp.as_workunit(is_primary_source=False)
self.ingested_upstream_dataset_keys.add(upstream_dataset_urn)
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=UpstreamLineageClass(
upstreams=[
UpstreamClass(
dataset=upstream_dataset_urn,
type=DatasetLineageTypeClass.COPY,
),
],
),
)
yield mcp.as_workunit()
else:
self.report.report_warning(
"unknown-upstream-dataset",
f"Unknown upstream dataset for model with id {model.namespace}:{model.model_id} and external id {model.external_id}",
)
elif model.system_type is not None:
self.report.report_warning(
"unknown-system-type",
f"Unknown system type {model.system_type} for model with id {model.namespace}:{model.model_id} and external id {model.external_id}",
)
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=StatusClass(
removed=False,
),
)
yield mcp.as_workunit()
if model.external_id and model.connection_id and model.system_type:
type_name = DatasetSubTypes.SAC_LIVE_DATA_MODEL
elif model.is_import:
type_name = DatasetSubTypes.SAC_IMPORT_DATA_MODEL
else:
type_name = DatasetSubTypes.SAC_MODEL
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=SubTypesClass(
typeNames=[type_name],
),
)
yield mcp.as_workunit()
mcp = MetadataChangeProposalWrapper(
entityUrn=dataset_urn,
aspect=DataPlatformInstanceClass(
platform=make_data_platform_urn(self.platform),
instance=self.config.platform_instance,
),
)
yield mcp.as_workunit()
@staticmethod
def get_sac_connection(
config: SACSourceConfig,
) -> Tuple[OAuth2Session, pyodata.Client]:
session = OAuth2Session(
client_id=config.client_id,
client_secret=config.client_secret.get_secret_value(),
token_endpoint=config.token_url,
token_endpoint_auth_method="client_secret_post",
grant_type="client_credentials",
)
retries = 3
backoff_factor = 10
status_forcelist = (500,)
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
session.register_compliance_hook(
"protected_request", _add_sap_sac_custom_auth_header
)
session.fetch_token()
client = pyodata.Client(
url=f"{config.tenant_url}/api/v1",
connection=session,
config=pyodata.v2.model.Config(retain_null=True),
)
return session, client
def get_resources(self) -> Iterable[Resource]:
import_data_model_ids = self.get_import_data_model_ids()
filter = "isTemplate eq 0 and isSample eq 0 and isPublic eq 1"
if self.config.ingest_stories and self.config.ingest_applications:
filter += " and ((resourceType eq 'STORY' and resourceSubtype eq '') or (resourceType eq 'STORY' and resourceSubtype eq 'APPLICATION'))"
elif self.config.ingest_stories and not self.config.ingest_applications:
filter += " and resourceType eq 'STORY' and resourceSubtype eq ''"
elif not self.config.ingest_stories and self.config.ingest_applications:
filter += (
" and resourceType eq 'STORY' and resourceSubtype eq 'APPLICATION'"
)
select = "resourceId,resourceType,resourceSubtype,storyId,name,description,createdTime,createdBy,modifiedBy,modifiedTime,openURL,ancestorPath,isMobile"
entities: pyodata.v2.service.ListWithTotalCount = (
self.client.entity_sets.Resources.get_entities()
.custom("$format", "json")
.filter(filter)
.select(select)
.execute()
)
entity: pyodata.v2.service.EntityProxy
for entity in entities:
resource_id: str = entity.resourceId
name: str = entity.name.strip()
if not self.config.resource_id_pattern.allowed(
resource_id
) or not self.config.resource_name_pattern.allowed(name):
continue
ancestor_path: Optional[str] = None
try:
ancestors = json.loads(entity.ancestorPath)
ancestor_path = "/".join(
ancestor.replace("/", "%2F") for ancestor in ancestors
)
except json.JSONDecodeError:
pass
if ancestor_path and not self.config.folder_pattern.allowed(ancestor_path):
continue
resource_models: Set[ResourceModel] = set()
select = "modelId,name,description,externalId,connectionId,systemType"
nav_entities: pyodata.v2.service.EntitySetProxy = (
entity.nav("resourceModels")
.get_entities()
.custom("$format", "json")
.select(select)
.execute()
)
nav_entity: pyodata.v2.service.EntityProxy
for nav_entity in nav_entities:
# the model id can have a different structure, commonly all model ids have a namespace (the part before the colon) and the model id itself
# t.4.sap.fpa.services.userFriendlyPerfLog:ACTIVITY_LOG is a builtin model without a possiblity to get more metadata about the model
# t.4.YV67EM4QBRU035A7TVKERZ786N:YV67EM4QBRU035A7TVKERZ786N is a model id where the model id itself also appears as part of the namespace
# t.4:C76tt2j402o1e69wnvrwfcl79c is a model id without the model id itself as part of the namespace
model_id: str = nav_entity.modelId
namespace, _, model_id = model_id.partition(":")
resource_models.add(
ResourceModel(
namespace=namespace,
model_id=model_id,
name=nav_entity.name.strip(),
description=nav_entity.description.strip(),
system_type=nav_entity.systemType, # BW or HANA
connection_id=nav_entity.connectionId,
external_id=nav_entity.externalId, # query:[][][query] or view:[schema][schema.namespace][view]
is_import=model_id in import_data_model_ids,
)
)
created_by: Optional[str] = entity.createdBy
if created_by in ("SYSTEM", "$DELETED_USER$"):
created_by = None
modified_by: Optional[str] = entity.modifiedBy
if modified_by in ("SYSTEM", "$DELETED_USER$"):
modified_by = None
yield Resource(
resource_id=resource_id,
resource_type=entity.resourceType,
resource_subtype=entity.resourceSubtype,
story_id=entity.storyId,
name=name,
description=entity.description.strip(),
created_time=entity.createdTime,
created_by=created_by,
modified_time=entity.modifiedTime,
modified_by=modified_by,
open_url=entity.openURL,
ancestor_path=ancestor_path,
is_mobile=entity.isMobile,
resource_models=frozenset(resource_models),
)
def get_import_data_model_ids(self) -> Set[str]:
response = self.session.get(
url=f"{self.config.tenant_url}/api/v1/dataimport/models"
)
response.raise_for_status()
import_data_model_ids = set(
model["modelID"] for model in response.json()["models"]
)
return import_data_model_ids
def get_import_data_model_columns(
self, model_id: str
) -> List[ImportDataModelColumn]:
response = self.session.get(
url=f"{self.config.tenant_url}/api/v1/dataimport/models/{model_id}/metadata"
)
response.raise_for_status()
model_metadata = response.json()
columns: List[ImportDataModelColumn] = []
for column in model_metadata["factData"]["columns"]:
columns.append(
ImportDataModelColumn(
name=column["columnName"].strip(),
description=column["descriptionName"].strip(),
property_type=column["propertyType"],
data_type=column["columnDataType"],
max_length=column.get("maxLength"),
precision=column.get("precision"),
scale=column.get("scale"),
is_key=column["isKey"],
)
)
return columns
def get_query_name(self, query: str) -> str:
if not self.config.query_name_template:
return query
query_name = self.config.query_name_template
query_name = query_name.replace("{name}", query)
return query_name
def get_view_name(self, schema: str, namespace: Optional[str], view: str) -> str:
if namespace:
return f"{schema}.{namespace}::{view}"
return f"{schema}.{view}"
def get_schema_field_data_type(
self, column: ImportDataModelColumn
) -> SchemaFieldDataTypeClass:
if column.property_type == "DATE":
return SchemaFieldDataTypeClass(type=DateTypeClass())
else:
if column.data_type == "string":
return SchemaFieldDataTypeClass(type=StringTypeClass())
elif column.data_type in ("decimal", "int32"):
return SchemaFieldDataTypeClass(type=NumberTypeClass())
else:
self.report.report_warning(
"unknown-data-type",
f"Unknown data type {column.data_type} found",
)
return SchemaFieldDataTypeClass(type=NullTypeClass())
def get_schema_field_native_data_type(self, column: ImportDataModelColumn) -> str:
native_data_type = column.data_type
if column.data_type == "decimal":
native_data_type = f"{column.data_type}({column.precision}, {column.scale})"
elif column.data_type == "int32":
native_data_type = f"{column.data_type}({column.precision})"
elif column.max_length is not None:
native_data_type = f"{column.data_type}({column.max_length})"
return native_data_type
def _add_sap_sac_custom_auth_header(
url: str, headers: Dict[str, str], body: Any
) -> Tuple[str, Dict[str, str], Any]:
headers["x-sap-sac-custom-auth"] = "true"
return url, headers, body

45
metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py Normal file
View File

@ -0,0 +1,45 @@
from dataclasses import dataclass
from datetime import datetime
from typing import FrozenSet, Optional
@dataclass(frozen=True)
class ResourceModel:
namespace: str
model_id: str
name: str
description: str
system_type: Optional[str]
connection_id: Optional[str]
external_id: Optional[str]
is_import: bool
@dataclass(frozen=True)
class Resource:
resource_id: str
resource_type: str
resource_subtype: str
story_id: str
name: str
description: str
created_time: datetime
created_by: Optional[str]
modified_time: datetime
modified_by: Optional[str]
open_url: str
ancestor_path: Optional[str]
is_mobile: bool
resource_models: FrozenSet[ResourceModel]
@dataclass(frozen=True)
class ImportDataModelColumn:
name: str
description: str
property_type: str
data_type: str
max_length: Optional[int]
precision: Optional[int]
scale: Optional[int]
is_key: bool

1
metadata-ingestion/src/datahub/utilities/logging_manager.py
View File

@ -283,3 +283,4 @@ logging.getLogger("urllib3.util.retry").setLevel(logging.WARNING)
logging.getLogger("snowflake").setLevel(level=logging.WARNING)
# logging.getLogger("botocore").setLevel(logging.INFO)
# logging.getLogger("google").setLevel(logging.INFO)
logging.getLogger("pyodata").setLevel(logging.WARNING)

0
metadata-ingestion/tests/integration/sac/__init__.py Normal file
View File

404
metadata-ingestion/tests/integration/sac/metadata.xml Normal file
View File

@ -0,0 +1,404 @@
<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
<edmx:Edmx Version="1.0" xmlns:edmx="http://schemas.microsoft.com/ado/2007/06/edmx">
<edmx:DataServices xmlns:m="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata" m:DataServiceVersion="2.0">
<Schema Namespace="sap.fpa.services.search.internal" xmlns:d="http://schemas.microsoft.com/ado/2007/08/dataservices" xmlns:m="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata" xmlns="http://schemas.microsoft.com/ado/2008/09/edm">
<EntityType Name="FilterRepositoriesType">
<Key>
<PropertyRef Name="id" />
</Key>
<Property Name="id" Type="Edm.String" Nullable="false" MaxLength="256" />
<Property Name="type" Type="Edm.String" MaxLength="256" />
<Property Name="subType" Type="Edm.String" MaxLength="256" />
<Property Name="name" Type="Edm.String" MaxLength="255" />
<Property Name="description" Type="Edm.String" MaxLength="1024" />
<Property Name="createdBy" Type="Edm.String" MaxLength="127" />
<Property Name="createdTime" Type="Edm.DateTime" />
<Property Name="modifiedBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedTime" Type="Edm.DateTime" />
<Property Name="isTemplate" Type="Edm.Int32" />
<Property Name="isSample" Type="Edm.Int32" />
<Property Name="isPublic" Type="Edm.Int32" />
<Property Name="openURL" Type="Edm.String" MaxLength="255" />
<Property Name="ancestorPath" Type="Edm.String" MaxLength="5000" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
<Property Name="isMobile" Type="Edm.Int32" />
<Property Name="parentId" Type="Edm.String" MaxLength="256" />
<NavigationProperty Name="createdByUser" Relationship="sap.fpa.services.search.internal.RepositoryCreatedByType" FromRole="FilterRepositoriesPrincipal" ToRole="createdByDependent" />
<NavigationProperty Name="modifiedByUser" Relationship="sap.fpa.services.search.internal.RepositoryModifiedByType" FromRole="FilterRepositoriesPrincipal" ToRole="modifiedByDependent" />
<NavigationProperty Name="permissions" Relationship="sap.fpa.services.search.internal.RepositoryPermissionsType" FromRole="FilterRepositoriesPrincipal" ToRole="resourcePermissionsDependent" />
</EntityType>
<EntityType Name="RepositoriesType">
<Key>
<PropertyRef Name="id" />
</Key>
<Property Name="name" Type="Edm.String" MaxLength="255" />
<Property Name="description" Type="Edm.String" MaxLength="1024" />
<Property Name="id" Type="Edm.String" Nullable="false" MaxLength="256" />
<Property Name="type" Type="Edm.String" MaxLength="256" />
<Property Name="subType" Type="Edm.String" MaxLength="256" />
<Property Name="createdTime" Type="Edm.DateTime" />
<Property Name="createdBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedTime" Type="Edm.DateTime" />
<Property Name="isTemplate" Type="Edm.Int32" />
<Property Name="isSample" Type="Edm.Int32" />
<Property Name="isPublic" Type="Edm.Int32" />
<Property Name="isMobile" Type="Edm.Int32" />
<Property Name="openURL" Type="Edm.String" MaxLength="255" />
<Property Name="ancestorPath" Type="Edm.String" MaxLength="5000" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
<Property Name="parentId" Type="Edm.String" MaxLength="256" />
<NavigationProperty Name="createdByUser" Relationship="sap.fpa.services.search.internal.FilterRepositoryCreatedByType" FromRole="RepositoriesPrincipal" ToRole="createdByDependent" />
<NavigationProperty Name="modifiedByUser" Relationship="sap.fpa.services.search.internal.FilterRepositoryModifiedByType" FromRole="RepositoriesPrincipal" ToRole="modifiedByDependent" />
<NavigationProperty Name="permissions" Relationship="sap.fpa.services.search.internal.FilterRepositoryPermissionsType" FromRole="RepositoriesPrincipal" ToRole="resourcePermissionsDependent" />
</EntityType>
<EntityType Name="FilterResourcesType">
<Key>
<PropertyRef Name="resourceId" />
</Key>
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="256" />
<Property Name="resourceType" Type="Edm.String" MaxLength="256" />
<Property Name="resourceSubtype" Type="Edm.String" MaxLength="256" />
<Property Name="storyId" Type="Edm.String" MaxLength="255" />
<Property Name="name" Type="Edm.String" MaxLength="255" />
<Property Name="description" Type="Edm.String" MaxLength="1024" />
<Property Name="createdBy" Type="Edm.String" MaxLength="127" />
<Property Name="createdTime" Type="Edm.DateTime" />
<Property Name="modifiedBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedTime" Type="Edm.DateTime" />
<Property Name="isTemplate" Type="Edm.Int32" />
<Property Name="isSample" Type="Edm.Int32" />
<Property Name="isPublic" Type="Edm.Int32" />
<Property Name="openURL" Type="Edm.String" MaxLength="255" />
<Property Name="ancestorPath" Type="Edm.String" MaxLength="5000" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
<Property Name="isMobile" Type="Edm.Int32" />
<NavigationProperty Name="resourceModels" Relationship="sap.fpa.services.search.internal.Resources_ModelsType" FromRole="FilterResourcesPrincipal" ToRole="ModelsDependent" />
<NavigationProperty Name="storyPages" Relationship="sap.fpa.services.search.internal.Resources_PagesType" FromRole="FilterResourcesPrincipal" ToRole="PagesDependent" />
<NavigationProperty Name="createdByUser" Relationship="sap.fpa.services.search.internal.Created_ByType" FromRole="FilterResourcesPrincipal" ToRole="createdByDependent" />
<NavigationProperty Name="modifiedByUser" Relationship="sap.fpa.services.search.internal.Modified_ByType" FromRole="FilterResourcesPrincipal" ToRole="modifiedByDependent" />
<NavigationProperty Name="resourcePermissions" Relationship="sap.fpa.services.search.internal.Resource_PermissionsType" FromRole="FilterResourcesPrincipal" ToRole="resourcePermissionsDependent" />
<NavigationProperty Name="resourceRemoteSystems" Relationship="sap.fpa.services.search.internal.Resource_Remote_SystemsType" FromRole="FilterResourcesPrincipal" ToRole="ModelsRemoteSystemsDependent" />
<NavigationProperty Name="resourceQueryValidationStatusInfos" Relationship="sap.fpa.services.search.internal.Resource_Query_Validation_StatusType" FromRole="FilterResourcesPrincipal" ToRole="resourceQueryValidationStatusInfosDependent" />
<NavigationProperty Name="isValid" Relationship="sap.fpa.services.search.internal.Resource_Is_ValidType" FromRole="FilterResourcesPrincipal" ToRole="isValidDependent" />
</EntityType>
<EntityType Name="ResourcesType">
<Key>
<PropertyRef Name="resourceId" />
</Key>
<Property Name="name" Type="Edm.String" MaxLength="255" />
<Property Name="description" Type="Edm.String" MaxLength="1024" />
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="256" />
<Property Name="resourceType" Type="Edm.String" MaxLength="256" />
<Property Name="resourceSubtype" Type="Edm.String" MaxLength="256" />
<Property Name="storyId" Type="Edm.String" MaxLength="255" />
<Property Name="createdTime" Type="Edm.DateTime" />
<Property Name="createdBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedBy" Type="Edm.String" MaxLength="127" />
<Property Name="modifiedTime" Type="Edm.DateTime" />
<Property Name="isTemplate" Type="Edm.Int32" />
<Property Name="isSample" Type="Edm.Int32" />
<Property Name="isPublic" Type="Edm.Int32" />
<Property Name="isMobile" Type="Edm.Int32" />
<Property Name="openURL" Type="Edm.String" MaxLength="255" />
<Property Name="ancestorPath" Type="Edm.String" MaxLength="5000" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
<NavigationProperty Name="resourceModels" Relationship="sap.fpa.services.search.internal.Filter_Resources_ModelsType" FromRole="ResourcesPrincipal" ToRole="ModelsDependent" />
<NavigationProperty Name="storyPages" Relationship="sap.fpa.services.search.internal.Filter_Resources_PagesType" FromRole="ResourcesPrincipal" ToRole="PagesDependent" />
<NavigationProperty Name="createdByUser" Relationship="sap.fpa.services.search.internal.Filter_Created_ByType" FromRole="ResourcesPrincipal" ToRole="createdByDependent" />
<NavigationProperty Name="modifiedByUser" Relationship="sap.fpa.services.search.internal.Filter_Modified_ByType" FromRole="ResourcesPrincipal" ToRole="modifiedByDependent" />
<NavigationProperty Name="resourcePermissions" Relationship="sap.fpa.services.search.internal.Filter_Resource_PermissionsType" FromRole="ResourcesPrincipal" ToRole="resourcePermissionsDependent" />
<NavigationProperty Name="resourceRemoteSystems" Relationship="sap.fpa.services.search.internal.Filter_Resource_Remote_SystemsType" FromRole="ResourcesPrincipal" ToRole="ModelsRemoteSystemsDependent" />
<NavigationProperty Name="resourceQueryValidationStatusInfos" Relationship="sap.fpa.services.search.internal.Filter_Resource_Query_Validation_StatusType" FromRole="ResourcesPrincipal" ToRole="resourceQueryValidationStatusInfosDependent" />
<NavigationProperty Name="isValid" Relationship="sap.fpa.services.search.internal.Filter_Resource_Is_ValidType" FromRole="ResourcesPrincipal" ToRole="isValidDependent" />
</EntityType>
<EntityType Name="ModelsType">
<Key>
<PropertyRef Name="resourceId" />
<PropertyRef Name="modelId" />
</Key>
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="255" />
<Property Name="modelId" Type="Edm.String" Nullable="false" MaxLength="400" />
<Property Name="name" Type="Edm.String" MaxLength="1024" />
<Property Name="description" Type="Edm.String" MaxLength="1024" />
<Property Name="externalId" Type="Edm.String" MaxLength="400" />
<Property Name="isPlanning" Type="Edm.Int32" DefaultValue="0" />
<Property Name="connectionId" Type="Edm.String" MaxLength="100" />
<Property Name="connectionName" Type="Edm.String" MaxLength="1024" />
<Property Name="connectionType" Type="Edm.String" MaxLength="10" />
<Property Name="systemType" Type="Edm.String" MaxLength="10" />
<Property Name="protocol" Type="Edm.String" MaxLength="5" />
<Property Name="client" Type="Edm.String" MaxLength="3" />
<Property Name="language" Type="Edm.String" MaxLength="10" />
<Property Name="hcpAccount" Type="Edm.String" MaxLength="256" />
<Property Name="pathPrefix" Type="Edm.String" MaxLength="2048" />
<Property Name="host" Type="Edm.String" MaxLength="2048" />
<Property Name="port" Type="Edm.String" MaxLength="5" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
</EntityType>
<EntityType Name="PagesType">
<Key>
<PropertyRef Name="storyId" />
<PropertyRef Name="sequenceNumber" />
</Key>
<Property Name="storyId" Type="Edm.String" Nullable="false" MaxLength="255" />
<Property Name="sequenceNumber" Type="Edm.Int32" Nullable="false" />
<Property Name="name" Type="Edm.String" MaxLength="1024" />
<Property Name="type" Type="Edm.String" MaxLength="15" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
</EntityType>
<EntityType Name="ModelsRemoteSystemsType">
<Key>
<PropertyRef Name="resourceId" />
</Key>
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="255" />
<Property Name="systemType" Type="Edm.String" MaxLength="5000" />
</EntityType>
<EntityType Name="isValidType">
<Key>
<PropertyRef Name="resourceId" />
</Key>
<Property Name="isValid" Type="Edm.Int32" />
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="256" />
</EntityType>
<EntityType Name="createdByType">
<Key>
<PropertyRef Name="userId" />
</Key>
<Property Name="userId" Type="Edm.String" Nullable="false" MaxLength="127" />
<Property Name="firstName" Type="Edm.String" MaxLength="5000" />
<Property Name="lastName" Type="Edm.String" MaxLength="5000" />
<Property Name="displayName" Type="Edm.String" MaxLength="5000" />
<Property Name="samlUsername" Type="Edm.String" MaxLength="256" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
</EntityType>
<EntityType Name="modifiedByType">
<Key>
<PropertyRef Name="userId" />
</Key>
<Property Name="userId" Type="Edm.String" Nullable="false" MaxLength="127" />
<Property Name="firstName" Type="Edm.String" MaxLength="5000" />
<Property Name="lastName" Type="Edm.String" MaxLength="5000" />
<Property Name="displayName" Type="Edm.String" MaxLength="5000" />
<Property Name="samlUsername" Type="Edm.String" MaxLength="256" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
</EntityType>
<EntityType Name="resourcePermissionsType">
<Key>
<PropertyRef Name="requestId" />
<PropertyRef Name="resourceId" />
</Key>
<Property Name="requestId" Type="Edm.String" Nullable="false" MaxLength="37" />
<Property Name="resourceId" Type="Edm.String" Nullable="false" MaxLength="256" />
<Property Name="isChangeable" Type="Edm.Int32" DefaultValue="0" />
<Property Name="isDeletable" Type="Edm.Int32" DefaultValue="0" />
<Property Name="isShared" Type="Edm.Int32" DefaultValue="0" />
<Property Name="isSharedToAny" Type="Edm.Int32" DefaultValue="0" />
<Property Name="isShareable" Type="Edm.Int32" DefaultValue="0" />
</EntityType>
<EntityType Name="resourceQueryValidationStatusInfosType">
<Key>
<PropertyRef Name="id" />
</Key>
<Property Name="id" Type="Edm.String" Nullable="false" MaxLength="37" />
<Property Name="resourceId" Type="Edm.String" MaxLength="256" />
<Property Name="code" Type="Edm.String" MaxLength="128" />
<Property Name="baseObject" Type="Edm.String" MaxLength="256" />
<Property Name="columnName" Type="Edm.String" MaxLength="256" />
<Property Name="objectDescription" Type="Edm.String" MaxLength="5000" />
<Property Name="objectType" Type="Edm.String" MaxLength="32" />
<Property Name="lastIndexedTime" Type="Edm.DateTime" />
</EntityType>
<Association Name="Resources_ModelsType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.ModelsType" Role="ModelsDependent" Multiplicity="*" />
</Association>
<Association Name="Resources_PagesType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.PagesType" Role="PagesDependent" Multiplicity="*" />
</Association>
<Association Name="Created_ByType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.createdByType" Role="createdByDependent" Multiplicity="1" />
</Association>
<Association Name="Modified_ByType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.modifiedByType" Role="modifiedByDependent" Multiplicity="1" />
</Association>
<Association Name="Resource_PermissionsType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourcePermissionsType" Role="resourcePermissionsDependent" Multiplicity="*" />
</Association>
<Association Name="Resource_Remote_SystemsType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.ModelsRemoteSystemsType" Role="ModelsRemoteSystemsDependent" Multiplicity="1" />
</Association>
<Association Name="Resource_Query_Validation_StatusType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourceQueryValidationStatusInfosType" Role="resourceQueryValidationStatusInfosDependent" Multiplicity="*" />
</Association>
<Association Name="Resource_Is_ValidType">
<End Type="sap.fpa.services.search.internal.FilterResourcesType" Role="FilterResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.isValidType" Role="isValidDependent" Multiplicity="1" />
</Association>
<Association Name="Filter_Resources_ModelsType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.ModelsType" Role="ModelsDependent" Multiplicity="*" />
</Association>
<Association Name="Filter_Resources_PagesType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.PagesType" Role="PagesDependent" Multiplicity="*" />
</Association>
<Association Name="Filter_Created_ByType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.createdByType" Role="createdByDependent" Multiplicity="1" />
</Association>
<Association Name="Filter_Modified_ByType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.modifiedByType" Role="modifiedByDependent" Multiplicity="1" />
</Association>
<Association Name="Filter_Resource_PermissionsType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourcePermissionsType" Role="resourcePermissionsDependent" Multiplicity="*" />
</Association>
<Association Name="Filter_Resource_Remote_SystemsType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.ModelsRemoteSystemsType" Role="ModelsRemoteSystemsDependent" Multiplicity="1" />
</Association>
<Association Name="Filter_Resource_Query_Validation_StatusType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourceQueryValidationStatusInfosType" Role="resourceQueryValidationStatusInfosDependent" Multiplicity="*" />
</Association>
<Association Name="Filter_Resource_Is_ValidType">
<End Type="sap.fpa.services.search.internal.ResourcesType" Role="ResourcesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.isValidType" Role="isValidDependent" Multiplicity="1" />
</Association>
<Association Name="RepositoryCreatedByType">
<End Type="sap.fpa.services.search.internal.FilterRepositoriesType" Role="FilterRepositoriesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.createdByType" Role="createdByDependent" Multiplicity="1" />
</Association>
<Association Name="RepositoryModifiedByType">
<End Type="sap.fpa.services.search.internal.FilterRepositoriesType" Role="FilterRepositoriesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.modifiedByType" Role="modifiedByDependent" Multiplicity="1" />
</Association>
<Association Name="RepositoryPermissionsType">
<End Type="sap.fpa.services.search.internal.FilterRepositoriesType" Role="FilterRepositoriesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourcePermissionsType" Role="resourcePermissionsDependent" Multiplicity="*" />
</Association>
<Association Name="FilterRepositoryCreatedByType">
<End Type="sap.fpa.services.search.internal.RepositoriesType" Role="RepositoriesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.createdByType" Role="createdByDependent" Multiplicity="1" />
</Association>
<Association Name="FilterRepositoryModifiedByType">
<End Type="sap.fpa.services.search.internal.RepositoriesType" Role="RepositoriesPrincipal" Multiplicity="*" />
<End Type="sap.fpa.services.search.internal.modifiedByType" Role="modifiedByDependent" Multiplicity="1" />
</Association>
<Association Name="FilterRepositoryPermissionsType">
<End Type="sap.fpa.services.search.internal.RepositoriesType" Role="RepositoriesPrincipal" Multiplicity="1" />
<End Type="sap.fpa.services.search.internal.resourcePermissionsType" Role="resourcePermissionsDependent" Multiplicity="*" />
</Association>
<EntityContainer Name="SearchMetadata" m:IsDefaultEntityContainer="true">
<EntitySet Name="FilterRepositories" EntityType="sap.fpa.services.search.internal.FilterRepositoriesType" />
<EntitySet Name="Repositories" EntityType="sap.fpa.services.search.internal.RepositoriesType" />
<EntitySet Name="FilterResources" EntityType="sap.fpa.services.search.internal.FilterResourcesType" />
<EntitySet Name="Resources" EntityType="sap.fpa.services.search.internal.ResourcesType" />
<EntitySet Name="Models" EntityType="sap.fpa.services.search.internal.ModelsType" />
<EntitySet Name="Pages" EntityType="sap.fpa.services.search.internal.PagesType" />
<EntitySet Name="ModelsRemoteSystems" EntityType="sap.fpa.services.search.internal.ModelsRemoteSystemsType" />
<EntitySet Name="isValid" EntityType="sap.fpa.services.search.internal.isValidType" />
<EntitySet Name="createdBy" EntityType="sap.fpa.services.search.internal.createdByType" />
<EntitySet Name="modifiedBy" EntityType="sap.fpa.services.search.internal.modifiedByType" />
<EntitySet Name="resourcePermissions" EntityType="sap.fpa.services.search.internal.resourcePermissionsType" />
<EntitySet Name="resourceQueryValidationStatusInfos" EntityType="sap.fpa.services.search.internal.resourceQueryValidationStatusInfosType" />
<AssociationSet Name="Resources_Models" Association="sap.fpa.services.search.internal.Resources_ModelsType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="ModelsDependent" EntitySet="Models" />
</AssociationSet>
<AssociationSet Name="Resources_Pages" Association="sap.fpa.services.search.internal.Resources_PagesType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="PagesDependent" EntitySet="Pages" />
</AssociationSet>
<AssociationSet Name="Created_By" Association="sap.fpa.services.search.internal.Created_ByType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="createdByDependent" EntitySet="createdBy" />
</AssociationSet>
<AssociationSet Name="Modified_By" Association="sap.fpa.services.search.internal.Modified_ByType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="modifiedByDependent" EntitySet="modifiedBy" />
</AssociationSet>
<AssociationSet Name="Resource_Permissions" Association="sap.fpa.services.search.internal.Resource_PermissionsType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="resourcePermissionsDependent" EntitySet="resourcePermissions" />
</AssociationSet>
<AssociationSet Name="Resource_Remote_Systems" Association="sap.fpa.services.search.internal.Resource_Remote_SystemsType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="ModelsRemoteSystemsDependent" EntitySet="ModelsRemoteSystems" />
</AssociationSet>
<AssociationSet Name="Resource_Query_Validation_Status" Association="sap.fpa.services.search.internal.Resource_Query_Validation_StatusType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="resourceQueryValidationStatusInfosDependent" EntitySet="resourceQueryValidationStatusInfos" />
</AssociationSet>
<AssociationSet Name="Resource_Is_Valid" Association="sap.fpa.services.search.internal.Resource_Is_ValidType">
<End Role="FilterResourcesPrincipal" EntitySet="FilterResources" />
<End Role="isValidDependent" EntitySet="isValid" />
</AssociationSet>
<AssociationSet Name="Filter_Resources_Models" Association="sap.fpa.services.search.internal.Filter_Resources_ModelsType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="ModelsDependent" EntitySet="Models" />
</AssociationSet>
<AssociationSet Name="Filter_Resources_Pages" Association="sap.fpa.services.search.internal.Filter_Resources_PagesType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="PagesDependent" EntitySet="Pages" />
</AssociationSet>
<AssociationSet Name="Filter_Created_By" Association="sap.fpa.services.search.internal.Filter_Created_ByType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="createdByDependent" EntitySet="createdBy" />
</AssociationSet>
<AssociationSet Name="Filter_Modified_By" Association="sap.fpa.services.search.internal.Filter_Modified_ByType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="modifiedByDependent" EntitySet="modifiedBy" />
</AssociationSet>
<AssociationSet Name="Filter_Resource_Permissions" Association="sap.fpa.services.search.internal.Filter_Resource_PermissionsType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="resourcePermissionsDependent" EntitySet="resourcePermissions" />
</AssociationSet>
<AssociationSet Name="Filter_Resource_Remote_Systems" Association="sap.fpa.services.search.internal.Filter_Resource_Remote_SystemsType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="ModelsRemoteSystemsDependent" EntitySet="ModelsRemoteSystems" />
</AssociationSet>
<AssociationSet Name="Filter_Resource_Query_Validation_Status" Association="sap.fpa.services.search.internal.Filter_Resource_Query_Validation_StatusType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="resourceQueryValidationStatusInfosDependent" EntitySet="resourceQueryValidationStatusInfos" />
</AssociationSet>
<AssociationSet Name="Filter_Resource_Is_Valid" Association="sap.fpa.services.search.internal.Filter_Resource_Is_ValidType">
<End Role="ResourcesPrincipal" EntitySet="Resources" />
<End Role="isValidDependent" EntitySet="isValid" />
</AssociationSet>
<AssociationSet Name="RepositoryCreatedBy" Association="sap.fpa.services.search.internal.RepositoryCreatedByType">
<End Role="FilterRepositoriesPrincipal" EntitySet="FilterRepositories" />
<End Role="createdByDependent" EntitySet="createdBy" />
</AssociationSet>
<AssociationSet Name="RepositoryModifiedBy" Association="sap.fpa.services.search.internal.RepositoryModifiedByType">
<End Role="FilterRepositoriesPrincipal" EntitySet="FilterRepositories" />
<End Role="modifiedByDependent" EntitySet="modifiedBy" />
</AssociationSet>
<AssociationSet Name="RepositoryPermissions" Association="sap.fpa.services.search.internal.RepositoryPermissionsType">
<End Role="FilterRepositoriesPrincipal" EntitySet="FilterRepositories" />
<End Role="resourcePermissionsDependent" EntitySet="resourcePermissions" />
</AssociationSet>
<AssociationSet Name="FilterRepositoryCreatedBy" Association="sap.fpa.services.search.internal.FilterRepositoryCreatedByType">
<End Role="RepositoriesPrincipal" EntitySet="Repositories" />
<End Role="createdByDependent" EntitySet="createdBy" />
</AssociationSet>
<AssociationSet Name="FilterRepositoryModifiedBy" Association="sap.fpa.services.search.internal.FilterRepositoryModifiedByType">
<End Role="RepositoriesPrincipal" EntitySet="Repositories" />
<End Role="modifiedByDependent" EntitySet="modifiedBy" />
</AssociationSet>
<AssociationSet Name="FilterRepositoryPermissions" Association="sap.fpa.services.search.internal.FilterRepositoryPermissionsType">
<End Role="RepositoriesPrincipal" EntitySet="Repositories" />
<End Role="resourcePermissionsDependent" EntitySet="resourcePermissions" />
</AssociationSet>
</EntityContainer>
</Schema>
</edmx:DataServices>
</edmx:Edmx>

663
metadata-ingestion/tests/integration/sac/sac_mces_golden.json Normal file
View File

@ -0,0 +1,663 @@
[
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,LXTH4JCE36EOYLU41PIINLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "browsePaths",
"aspect": {
"json": {
"paths": [
"/sac/Public/Folder 1/Folder 2"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,LXTH4JCE36EOYLU41PIINLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "Public"
},
{
"id": "Folder 1"
},
{
"id": "Folder 2"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {
"namespace": "t.4.ANL8Q577BA2F73KU3VELDXGWZK",
"modelId": "ANL8Q577BA2F73KU3VELDXGWZK",
"isImport": "false"
},
"externalUrl": "http://tenant/sap/fpa/ui/tenants/3c44c#view_id=model;model_id=t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK",
"name": "Name of the first model (BW)",
"description": "Description of the first model which has a connection to a BW query",
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,HANA.SCHEMA.CE.SCHEMA::VIEW,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetKey",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:hana",
"name": "HANA.SCHEMA.CE.SCHEMA::VIEW",
"origin": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"changeType": "UPSERT",
"aspectName": "upstreamLineage",
"aspect": {
"json": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:bw,BW.QUERY/QUERY_TECHNICAL_NAME,PROD)",
"type": "COPY"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Live Data Model"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:sac"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {
"namespace": "t.4.K73U3VELDXGWZKANL8Q577BA2F",
"modelId": "K73U3VELDXGWZKANL8Q577BA2F",
"isImport": "false"
},
"externalUrl": "http://tenant/sap/fpa/ui/tenants/3c44c#view_id=model;model_id=t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F",
"name": "Name of the second model (HANA)",
"description": "Description of the second model which has a connection to a HANA view",
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "schemaMetadata",
"aspect": {
"json": {
"schemaName": "DXGWZKANLK73U3VEL8Q577BA2F",
"platform": "urn:li:dataPlatform:sac",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.schema.Schemaless": {}
},
"fields": [
{
"fieldPath": "Account",
"nullable": false,
"description": "Account",
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "string(256)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "FIELD1",
"nullable": false,
"description": "FIELD1",
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "string(256)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "FIELD2",
"nullable": false,
"description": "FIELD2",
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "string(256)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "FIELD3",
"nullable": false,
"description": "FIELD3",
"type": {
"type": {
"com.linkedin.schema.DateType": {}
}
},
"nativeDataType": "string(256)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "Version",
"nullable": false,
"description": "Version",
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "string(300)",
"recursive": false,
"isPartOfKey": true
},
{
"fieldPath": "SignedData",
"nullable": false,
"description": "SignedData",
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
"nativeDataType": "decimal(31, 7)",
"recursive": false,
"isPartOfKey": false
}
],
"primaryKeys": [
"Account",
"FIELD1",
"FIELD2",
"FIELD3",
"Version"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "upstreamLineage",
"aspect": {
"json": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:hana,HANA.SCHEMA.CE.SCHEMA::VIEW,PROD)",
"type": "COPY"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Live Data Model"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:sac"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {
"namespace": "t.4.DXGWZKANLK73U3VEL8Q577BA2F",
"modelId": "DXGWZKANLK73U3VEL8Q577BA2F",
"isImport": "true"
},
"externalUrl": "http://tenant/sap/fpa/ui/tenants/3c44c#view_id=model;model_id=t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F",
"name": "Name of the third model (Import)",
"description": "Description of the third model which was imported",
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bw,BW.QUERY/QUERY_TECHNICAL_NAME,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetKey",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:bw",
"name": "BW.QUERY/QUERY_TECHNICAL_NAME",
"origin": "PROD"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Import Data Model"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:sac"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,LXTH4JCE36EOYLU41PIINLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "dashboardInfo",
"aspect": {
"json": {
"customProperties": {
"resourceType": "STORY",
"resourceSubtype": "",
"storyId": "STORY:t.4:LXTH4JCE36EOYLU41PIINLYPU9XRYM26",
"isMobile": "0"
},
"externalUrl": "http://tenant/sap/fpa/ui/tenants/3c44c/bo/story/LXTH4JCE36EOYLU41PIINLYPU9XRYM26",
"title": "Name of the story",
"description": "Description of the story",
"charts": [],
"datasets": [
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)"
],
"lastModified": {
"created": {
"time": 1667544309783,
"actor": "urn:li:corpuser:JOHN_DOE"
},
"lastModified": {
"time": 1673067981272,
"actor": "urn:li:corpuser:JOHN_DOE"
}
}
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,LXTH4JCE36EOYLU41PIINLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,LXTH4JCE36EOYLU41PIINLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Story"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,EOYLU41PIILXTH4JCE36NLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "browsePaths",
"aspect": {
"json": {
"paths": [
"/sac/Public/Folder 1/Folder 2"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,EOYLU41PIILXTH4JCE36NLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "dashboardInfo",
"aspect": {
"json": {
"customProperties": {
"resourceType": "STORY",
"resourceSubtype": "APPLICATION",
"storyId": "STORY:t.4:EOYLU41PIILXTH4JCE36NLYPU9XRYM26",
"isMobile": "0"
},
"externalUrl": "http://tenant/sap/fpa/ui/tenants/3c44c/bo/story/EOYLU41PIILXTH4JCE36NLYPU9XRYM26",
"title": "Name of the application",
"description": "Description of the application",
"charts": [],
"datasets": [
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F,PROD)",
"urn:li:dataset:(urn:li:dataPlatform:sac,t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F,PROD)"
],
"lastModified": {
"created": {
"time": 1673279404272,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1673279414272,
"actor": "urn:li:corpuser:unknown"
}
}
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,EOYLU41PIILXTH4JCE36NLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,EOYLU41PIILXTH4JCE36NLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Application"
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(sac,EOYLU41PIILXTH4JCE36NLYPU9XRYM26)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "Public"
},
{
"id": "Folder 1"
},
{
"id": "Folder 2"
}
]
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "sac-integration-test",
"lastRunId": "no-run-id-provided"
}
}
]

316
metadata-ingestion/tests/integration/sac/test_sac.py Normal file
View File

@ -0,0 +1,316 @@
from functools import partial
from typing import Dict
from urllib.parse import parse_qs
import pytest
from datahub.ingestion.run.pipeline import Pipeline
from tests.test_helpers import mce_helpers
MOCK_TENANT_URL = "http://tenant"
MOCK_TOKEN_URL = "http://tenant.authentication/oauth/token"
MOCK_CLIENT_ID = "foo"
MOCK_CLIENT_SECRET = "bar"
MOCK_ACCESS_TOKEN = "foobaraccesstoken"
@pytest.mark.integration
def test_sac(
pytestconfig,
tmp_path,
requests_mock,
mock_time,
):
requests_mock.post(
MOCK_TOKEN_URL,
json=match_token_url,
)
test_resources_dir = pytestconfig.rootpath / "tests/integration/sac"
with open(f"{test_resources_dir}/metadata.xml", mode="rb") as f:
content = f.read()
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/$metadata",
content=partial(match_metadata, content=content),
)
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/Resources?$format=json&$filter=isTemplate eq 0 and isSample eq 0 and isPublic eq 1 and ((resourceType eq 'STORY' and resourceSubtype eq '') or (resourceType eq 'STORY' and resourceSubtype eq 'APPLICATION'))&$select=resourceId,resourceType,resourceSubtype,storyId,name,description,createdTime,createdBy,modifiedBy,modifiedTime,openURL,ancestorPath,isMobile",
json=match_resources,
)
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/Resources%28%27LXTH4JCE36EOYLU41PIINLYPU9XRYM26%27%29/resourceModels?$format=json&$select=modelId,name,description,externalId,connectionId,systemType",
json=partial(match_resource, resource_id="LXTH4JCE36EOYLU41PIINLYPU9XRYM26"),
)
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/Resources%28%27EOYLU41PIILXTH4JCE36NLYPU9XRYM26%27%29/resourceModels?$format=json&$select=modelId,name,description,externalId,connectionId,systemType",
json=partial(match_resource, resource_id="EOYLU41PIILXTH4JCE36NLYPU9XRYM26"),
)
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/dataimport/models",
json=match_models,
)
requests_mock.get(
f"{MOCK_TENANT_URL}/api/v1/dataimport/models/DXGWZKANLK73U3VEL8Q577BA2F/metadata",
json=match_model_metadata,
)
pipeline = Pipeline.create(
{
"run_id": "sac-integration-test",
"source": {
"type": "sac",
"config": {
"tenant_url": MOCK_TENANT_URL,
"token_url": MOCK_TOKEN_URL,
"client_id": MOCK_CLIENT_ID,
"client_secret": MOCK_CLIENT_SECRET,
},
},
"sink": {
"type": "file",
"config": {"filename": f"{tmp_path}/sac_mces.json"},
},
},
)
pipeline.run()
pipeline.raise_from_status()
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/sac_mces.json",
golden_path=test_resources_dir / "sac_mces_golden.json",
ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS,
)
def match_token_url(request, context):
form = parse_qs(request.text, strict_parsing=True)
assert "grant_type" in form
assert len(form["grant_type"]) == 1
assert form["grant_type"][0] == "client_credentials"
assert "client_id" in form
assert len(form["client_id"]) == 1
assert form["client_id"][0] == MOCK_CLIENT_ID
assert "client_secret" in form
assert len(form["client_secret"]) == 1
assert form["client_secret"][0] == MOCK_CLIENT_SECRET
json = {
"access_token": MOCK_ACCESS_TOKEN,
"expires_in": 3599,
}
return json
def check_authorization(headers: Dict[str, str]) -> None:
assert "Authorization" in headers
assert headers["Authorization"] == f"Bearer {MOCK_ACCESS_TOKEN}"
assert "x-sap-sac-custom-auth" in headers
assert headers["x-sap-sac-custom-auth"] == "true"
def match_metadata(request, context, content):
check_authorization(request.headers)
context.headers["content-type"] = "application/xml"
return content
def match_resources(request, context):
check_authorization(request.headers)
json = {
"d": {
"results": [
{
"__metadata": {
"type": "sap.fpa.services.search.internal.ResourcesType",
"uri": "/api/v1/Resources('LXTH4JCE36EOYLU41PIINLYPU9XRYM26')",
},
"name": "Name of the story",
"description": "Description of the story",
"resourceId": "LXTH4JCE36EOYLU41PIINLYPU9XRYM26",
"resourceType": "STORY",
"resourceSubtype": "",
"storyId": "STORY:t.4:LXTH4JCE36EOYLU41PIINLYPU9XRYM26",
"createdTime": "/Date(1667544309783)/",
"createdBy": "JOHN_DOE",
"modifiedBy": "JOHN_DOE",
"modifiedTime": "/Date(1673067981272)/",
"isMobile": 0,
"openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/LXTH4JCE36EOYLU41PIINLYPU9XRYM26",
"ancestorPath": '["Public","Folder 1","Folder 2"]',
},
{
"__metadata": {
"type": "sap.fpa.services.search.internal.ResourcesType",
"uri": "/api/v1/Resources('EOYLU41PIILXTH4JCE36NLYPU9XRYM26')",
},
"name": "Name of the application",
"description": "Description of the application",
"resourceId": "EOYLU41PIILXTH4JCE36NLYPU9XRYM26",
"resourceType": "STORY",
"resourceSubtype": "APPLICATION",
"storyId": "STORY:t.4:EOYLU41PIILXTH4JCE36NLYPU9XRYM26",
"createdTime": "/Date(1673279404272)/",
"createdBy": "SYSTEM",
"modifiedBy": "$DELETED_USER$",
"modifiedTime": "/Date(1673279414272)/",
"isMobile": 0,
"openURL": "/sap/fpa/ui/tenants/3c44c/bo/story/EOYLU41PIILXTH4JCE36NLYPU9XRYM26",
"ancestorPath": '["Public","Folder 1","Folder 2"]',
},
],
},
}
return json
def match_resource(request, context, resource_id):
check_authorization(request.headers)
json = {
"d": {
"results": [
{
"__metadata": {
"type": "sap.fpa.services.search.internal.ModelsType",
"uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.ANL8Q577BA2F73KU3VELDXGWZK%3AANL8Q577BA2F73KU3VELDXGWZK')",
},
"modelId": "t.4.ANL8Q577BA2F73KU3VELDXGWZK:ANL8Q577BA2F73KU3VELDXGWZK",
"name": "Name of the first model (BW)",
"description": "Description of the first model which has a connection to a BW query",
"externalId": "query:[][][QUERY_TECHNICAL_NAME]",
"connectionId": "BW",
"systemType": "BW",
},
{
"__metadata": {
"type": "sap.fpa.services.search.internal.ModelsType",
"uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.K73U3VELDXGWZKANL8Q577BA2F%3AK73U3VELDXGWZKANL8Q577BA2F')",
},
"modelId": "t.4.K73U3VELDXGWZKANL8Q577BA2F:K73U3VELDXGWZKANL8Q577BA2F",
"name": "Name of the second model (HANA)",
"description": "Description of the second model which has a connection to a HANA view",
"externalId": "view:[SCHEMA][NAMESPACE.SCHEMA][VIEW]",
"connectionId": "HANA",
"systemType": "HANA",
},
{
"__metadata": {
"type": "sap.fpa.services.search.internal.ModelsType",
"uri": f"/api/v1/Models(resourceId='{resource_id}',modelId='t.4.DXGWZKANLK73U3VEL8Q577BA2F%3ADXGWZKANLK73U3VEL8Q577BA2F')",
},
"modelId": "t.4.DXGWZKANLK73U3VEL8Q577BA2F:DXGWZKANLK73U3VEL8Q577BA2F",
"name": "Name of the third model (Import)",
"description": "Description of the third model which was imported",
"externalId": "",
"connectionId": "",
"systemType": None,
},
],
},
}
return json
def match_models(request, context):
check_authorization(request.headers)
json = {
"models": [
{
"modelID": "DXGWZKANLK73U3VEL8Q577BA2F",
"modelName": "Name of the third model (Import)",
"modelDescription": "Description of the third model which was imported",
"modelURL": f"{MOCK_TENANT_URL}/api/v1/dataimport/models/DXGWZKANLK73U3VEL8Q577BA2F",
},
],
}
return json
def match_model_metadata(request, context):
check_authorization(request.headers)
json = {
"factData": {
"keys": [
"Account",
"FIELD1",
"FIELD2",
"FIELD3",
"Version",
],
"columns": [
{
"columnName": "Account",
"columnDataType": "string",
"maxLength": 256,
"isKey": True,
"propertyType": "PROPERTY",
"descriptionName": "Account",
},
{
"columnName": "FIELD1",
"columnDataType": "string",
"maxLength": 256,
"isKey": True,
"propertyType": "PROPERTY",
"descriptionName": "FIELD1",
},
{
"columnName": "FIELD2",
"columnDataType": "string",
"maxLength": 256,
"isKey": True,
"propertyType": "PROPERTY",
"descriptionName": "FIELD2",
},
{
"columnName": "FIELD3",
"columnDataType": "string",
"maxLength": 256,
"isKey": True,
"propertyType": "DATE",
"descriptionName": "FIELD3",
},
{
"columnName": "Version",
"columnDataType": "string",
"maxLength": 300,
"isKey": True,
"propertyType": "PROPERTY",
"descriptionName": "Version",
},
{
"columnName": "SignedData",
"columnDataType": "decimal",
"maxLength": 32,
"precision": 31,
"scale": 7,
"isKey": False,
"propertyType": "PROPERTY",
"descriptionName": "SignedData",
},
],
},
}
return json

10
metadata-service/war/src/main/resources/boot/data_platforms.json
View File

@ -664,5 +664,15 @@
"type": "OTHERS",
"logoUrl": "/assets/platforms/sigmalogo.png"
}
},
{
"urn": "urn:li:dataPlatform:sac",
"aspect": {
"datasetNameDelimiter": ".",
"name": "sac",
"displayName": "SAP Analytics Cloud",
"type": "OTHERS",
"logoUrl": "/assets/platforms/saclogo.svg"
}
}
]

2
smoke-test/tests/cypress/cypress/e2e/mutations/ingestion_source.js
View File

@ -13,7 +13,7 @@ describe("ingestion source creation flow", () => {
cy.goToIngestionPage();
cy.clickOptionWithId('[data-node-key="Sources"]');
cy.clickOptionWithTestId("create-ingestion-source-button");
cy.clickOptionWithText("Snowflake");
cy.clickOptionWithTextToScrollintoView("Snowflake");
cy.waitTextVisible("Snowflake Details");
cy.get("#account_id").type(accound_id);
cy.get("#warehouse").type(warehouse_id);

4
smoke-test/tests/cypress/cypress/e2e/mutations/managing_secrets.js
View File

@ -30,7 +30,7 @@ describe("managing secrets for ingestion creation", () => {
cy.goToIngestionPage();
cy.clickOptionWithId('[data-node-key="Sources"]');
cy.get("#ingestion-create-source").click();
cy.clickOptionWithText("Snowflake");
cy.clickOptionWithTextToScrollintoView("Snowflake");
cy.waitTextVisible("Snowflake Details");
cy.get("#account_id").type(accound_id);
cy.get("#warehouse").type(warehouse_id);
@ -69,7 +69,7 @@ describe("managing secrets for ingestion creation", () => {
// Verify secret is not present during ingestion source creation for password dropdown
cy.clickOptionWithText("Create new source");
cy.clickOptionWithText("Snowflake");
cy.clickOptionWithTextToScrollintoView("Snowflake");
cy.waitTextVisible("Snowflake Details");
cy.get("#account_id").type(accound_id);
cy.get("#warehouse").type(warehouse_id);