yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 05:58:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #587: Ingestion: Add standalone report process to generate datasets, usage & profile and serve from standalone server (#588)

Browse Source 
* Fix #587: Ingestion: Add standalone report process to generate datasets, usage & profile and serve from standalone server

* add localhost
This commit is contained in:
Sriharsha Chintalapani 2021-09-27 08:43:38 -07:00 committed by GitHub
parent a670ebb86f
commit eb2717b0e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 14779 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ingestion/examples/sample_data/dashboards/charts.json
View File

@ -4,7 +4,7 @@
"id": "2841fdb1-e378-4a2c-94f8-27c9f5d6ef8e",
"name": "101",
"displayName": "# of Games That Hit 100k in Sales By Release Year",
"fullyQualifiedName": "local_superset.# of Games That Hit 100k in Sales By Release Year",
"fullyQualifiedName": "local_superset.101",
"description": "",
"chartId": "114",
"chartType": "Area",
@ -14,7 +14,7 @@
"id": "3bcba490-9e5c-4946-a0e3-41e8ff8f4aa4",
"name":"110",
"displayName": "% Rural",
"fullyQualifiedName": "local_superset.% Rural",
"fullyQualifiedName": "local_superset.110",
"description": "",
"chartId": "166",
"chartType": "Other",
@ -24,6 +24,7 @@
"id": "22b95748-4a7b-48ad-859e-cf7c66a7f343",
"name": "92",
"displayName": "✈️ Relocation ability",
"fullyQualifiedName": "local_superset.92",
"description": "",
"chartId": "92",
"chartType": "Other",
@ -31,9 +32,9 @@
"href": "http://localhost:8585/api/v1/charts/22b95748-4a7b-48ad-859e-cf7c66a7f343"
}, {
"id": "62b31dcc-4619-46a0-99b1-0fa7cd6f93da",
"name": "11y",
"name": "11",
"displayName": "Age distribution of respondents",
"fullyQualifiedName": "local_superset.Age distribution of respondents",
"fullyQualifiedName": "local_superset.11",
"description": "",
"chartId": "117",
"chartType": "Histogram",
@ -42,7 +43,7 @@
}, {
"id": "57944482-e187-439a-aaae-0e8aabd2f455",
"displayName": "Arcs",
"fullyQualifiedName": "local_superset.Arcs",
"fullyQualifiedName": "local_superset.197",
"description": "",
"name": "197",
"chartType": "Other",
@ -51,7 +52,7 @@
}, {
"id": "d88e2056-c74a-410d-829e-eb31b040c132",
"displayName": "Are you an ethnic minority in your city?",
"fullyQualifiedName": "local_superset.Are you an ethnic minority in your city?",
"fullyQualifiedName": "local_superset.127",
"description": "",
"name": "127",
"chartType": "Other",
@ -60,7 +61,7 @@
}, {
"id": "c1d3e156-4628-414e-8d6e-a6bdd486128f",
"displayName": "Average and Sum Trends",
"fullyQualifiedName": "local_superset.Average and Sum Trends",
"fullyQualifiedName": "local_superset.183",
"description": "",
"name": "183",
"chartType": "Line",
@ -78,7 +79,7 @@
}, {
"id": "bf2eeac4-7226-46c6-bbef-918569c137a0",
"displayName": "Box plot",
"fullyQualifiedName": "local_superset.Box plot",
"fullyQualifiedName": "local_superset.170",
"description": "",
"name": "170",
"chartType": "Bar",
@ -87,7 +88,7 @@
}, {
"id": "167fd63b-42f1-4d7e-a37d-893fd8173b44",
"displayName": "Boy Name Cloud",
"fullyQualifiedName": "local_superset.Boy Name Cloud",
"fullyQualifiedName": "local_superset.180",
"description": "",
"name": "180",
"chartType": "Other",

22
ingestion/examples/sample_data/dashboards/dashboards.json
View File

@ -3,8 +3,8 @@
{
"id": "d4dc7baf-1b17-45f8-acd5-a15b78cc7c5f",
"name": "8",
"displayName": "[ untitled dashboard ]",
"fullyQualifiedName": "local_superset.[ untitled dashboard ]",
"displayName": "Orders dashboard",
"fullyQualifiedName": "local_superset.8",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/1/",
"charts": [183, 170, 197],
@ -14,7 +14,7 @@
"id": "063cd787-8630-4809-9702-34d3992c7248",
"name": "9",
"displayName": "COVID Vaccine Dashboard",
"fullyQualifiedName": "local_superset.COVID Vaccine Dashboard",
"fullyQualifiedName": "local_superset.9",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/8/",
"charts": [117, 197],
@ -24,7 +24,7 @@
"id": "df6c698e-066a-4440-be0a-121025573b73",
"name": "10",
"displayName": "deck.gl Demo",
"fullyQualifiedName": "local_superset.deck.gl Demo",
"fullyQualifiedName": "local_superset.10",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/deck/",
"charts": [127, 166, 114],
@ -34,7 +34,7 @@
"id": "98b38a49-b5c6-431b-b61f-690e39f8ead2",
"name": "11",
"displayName": "FCC New Coder Survey 2018",
"fullyQualifiedName": "local_superset.FCC New Coder Survey 2018",
"fullyQualifiedName": "local_superset.11",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/7/",
"charts": [183, 197, 170, 180],
@ -44,7 +44,7 @@
"id": "dffcf9b2-4f43-4881-a5f5-10109655bf50",
"name": "12",
"displayName": "Misc Charts",
"fullyQualifiedName": "local_superset.Misc Charts",
"fullyQualifiedName": "local_superset.12",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/misc_charts/",
"charts": [127, 197],
@ -54,7 +54,7 @@
"id": "2583737d-6236-421e-ba0f-cd0b79adb216",
"name": "31",
"displayName": "Sales Dashboard",
"fullyQualifiedName": "local_superset.Sales Dashboard",
"fullyQualifiedName": "local_superset.31",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/6/",
"charts": [92,117,166],
@ -64,7 +64,7 @@
"id": "6bf9bfcb-4e80-4af0-9f0c-13e47bbc27a2",
"name": "33",
"displayName": "Slack Dashboard",
"fullyQualifiedName": "local_superset.Slack Dashboard",
"fullyQualifiedName": "local_superset.33",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/10/",
"charts": [114, 92, 127],
@ -74,7 +74,7 @@
"id": "1f02caf2-c5e5-442d-bda3-b8ce3e757b45",
"name": "34",
"displayName": "Unicode Test",
"fullyQualifiedName": "local_superset.Unicode Test",
"fullyQualifiedName": "local_superset.34",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/unicode-test/",
"charts": [161, 170, 180],
@ -84,7 +84,7 @@
"id": "a3ace318-ee37-4da1-974a-62eddbd77d20",
"name": "45",
"displayName": "USA Births Names",
"fullyQualifiedName": "local_superset.USA Births Names",
"fullyQualifiedName": "local_superset.45",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/births/",
"charts": [180],
@ -94,7 +94,7 @@
"id": "e6e21717-1164-403f-8807-d12be277aec6",
"name": "51",
"displayName": "Video Game Sales",
"fullyQualifiedName": "local_superset.Video Game Sales",
"fullyQualifiedName": "local_superset.51",
"description": "",
"dashboardUrl": "http://localhost:808/superset/dashboard/11/",
"charts": [127, 183],

12
ingestion/examples/sample_data/datasets/tables.json
View File

@ -5,6 +5,7 @@
"name": "dim_address",
"tableType": "Regular",
"description": "This dimension table contains the billing and shipping addresses of customers. You can join this table with the sales table to generate lists of the billing and shipping addresses. Customers can enter their addresses more than once, so the same address can appear in more than one row in this table. This table contains one row per customer address.",
"fullyQualifiedName": "bigquery.shopify.dim_address",
"columns": [
{
"name": "address_id",
@ -104,6 +105,7 @@
"name": "dim_api_client",
"tableType": "Regular",
"description": "This dimension table contains a row for each channel or app that your customers use to create orders. Some examples of these include Facebook and Online Store. You can join this table with the sales table to measure channel performance.",
"fullyQualifiedName": "bigquery.shopify.dim_api_client",
"columns": [
{
"name": "api_client_id",
@ -133,6 +135,7 @@
"name": "dim_customer",
"tableType": "Regular",
"description": "The dimension table contains data about your customers. The customers table contains one row per customer. It includes historical metrics (such as the total amount that each customer has spent in your store) as well as forward-looking metrics (such as the predicted number of days between future orders and the expected order value in the next 30 days). This table also includes columns that segment customers into various categories (such as new, returning, promising, at risk, dormant, and loyal), which you can use to target marketing activities.",
"fullyQualifiedName": "bigquery.shopify.dim_customer",
"columns": [
{
"name": "customer_id",
@ -330,6 +333,7 @@
"name": "dim_location",
"tableType": "Regular",
"description": "The dimension table contains metrics about your Shopify POS. This table contains one row per Shopify POS location. You can use this table to generate a list of the Shopify POS locations or you can join the table with the sales table to measure sales performance.",
"fullyQualifiedName": "bigquery.shopify.dim_location",
"columns": [
{
"name": "location_id",
@ -365,6 +369,7 @@
"name": "dim_product",
"tableType": "Regular",
"description": "This dimension table contains information about each of the products in your store. This table contains one row per product. This table reflects the current state of products in your Shopify admin.",
"fullyQualifiedName": "bigquery.shopify.dim_product",
"columns": [
{
"name": "product_id",
@ -422,6 +427,7 @@
"name": "dim_product_variant",
"tableType": "Regular",
"description": "This dimension table contains current information about each of the product variants in your store. This table contains one row per product variant.",
"fullyQualifiedName": "bigquery.shopify.dim_product_variant",
"columns": [
{
"name": "product_variant_id",
@ -507,6 +513,7 @@
"name": "dim_shop",
"tableType": "Regular",
"description": "This dimension table contains online shop information. This table contains one shop per row.",
"fullyQualifiedName": "bigquery.shopify.dim_shop",
"columns": [
{
"name": "shop_id",
@ -543,6 +550,7 @@
"name": "dim_staff",
"tableType": "Regular",
"description": "This dimension table contains information about the staff accounts in the store. It contains one row per staff account. Use this table to generate a list of your staff accounts, or join it with the sales, API clients and locations tables to analyze staff performance at Shopify POS locations.",
"fullyQualifiedName": "bigquery.shopify.dim_staff",
"columns": [
{
"name": "user_id",
@ -593,6 +601,7 @@
"name": "fact_line_item",
"tableType": "Regular",
"description": "The fact table contains information about the line items in orders. Each row in the table is a line item in an order. It contains product and product variant details as they were at the time of the order. This table does not include information about returns. Join this table with the TODO fact_sales table to get the details of the product on the day it was sold. This data will match what appears on the order in your Shopify admin as well as the in the Sales reports.",
"fullyQualifiedName": "bigquery.shopify.fact_line_item",
"columns": [
{
"name": "line_item_id",
@ -725,6 +734,7 @@
"name": "fact_order",
"tableType": "Regular",
"description": "The orders table contains information about each order in your store. Although this table is good for generating order lists and joining with the dim_customer, use the sales table instead for computing financial or other metrics.",
"fullyQualifiedName": "bigquery.shopify.fact_order",
"columns": [
{
"name": "order_id",
@ -842,6 +852,7 @@
"name": "fact_sale",
"tableType": "Regular",
"description": "The fact table captures the value of products sold or returned, as well as the values of other charges such as taxes and shipping costs. The sales table contains one row per order line item, one row per returned line item, and one row per shipping charge. Use this table when you need financial metrics.",
"fullyQualifiedName": "bigquery.shopify.fact_sale",
"columns": [
{
"name": "sale_id",
@ -1046,6 +1057,7 @@
"name": "fact_session",
"tableType": "Regular",
"description": "This fact table contains information about the visitors to your online store. This table has one row per session, where one session can contain many page views. If you use Urchin Traffic Module (UTM) parameters in marketing campaigns, then you can use this table to track how many customers they direct to your store.",
"fullyQualifiedName": "bigquery.shopify.fact_session",
"columns": [
{
"name": "derived_session_token",

6
ingestion/examples/sample_data/topics/topics.json
View File

@ -2,6 +2,7 @@
"topics": [{
"name": "customer_events",
"description": "Kafka topic to capture the customer events such as location updates or profile updates",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 56,
"retentionSize": 322122382273,
"replicationFactor":2,
@ -13,6 +14,7 @@
{
"name": "product_events",
"description": "Kafka topic to capture the product events. This topic will get updates on products decription, price etc.",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 128,
"retentionSize": 322122382273,
"replicationFactor":4,
@ -24,6 +26,7 @@
{
"name": "shop_updates",
"description": "Kafka topic to get any shop updates such as new products, location",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 16,
"retentionSize": 322122382273,
"replicationFactor":3,
@ -35,6 +38,7 @@
{
"name": "shop_products",
"description": "Kafka topic to get products in a shop. This is constantly updating",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 128,
"retentionSize": 3222122382273,
"replicationFactor":2,
@ -46,6 +50,7 @@
{
"name": "orders",
"description": "All the order events on our online store",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 128,
"retentionSize": 3222122382273,
"replicationFactor":3,
@ -57,6 +62,7 @@
{
"name": "sales",
"description": "All sales related events gets captured in this topic",
"fullyQualifiedName": "sample_kafka.customer_events",
"partitions": 128,
"retentionSize": 3222122382273,
"replicationFactor":4,

4
ingestion/examples/workflows/redshift.json
View File

@ -12,10 +12,6 @@
}
}
},
"processor": {
"type": "pii",
"config": {}
},
"sink": {
"type": "metadata-rest",
"config": {}

10
ingestion/setup.py
View File

@ -70,6 +70,13 @@ pii_requirements = {
"spacy==3.0.5"
}
report_requirements = {
"asgiref==3.4.1",
"Django==3.2.7",
"pytz==2021.1",
"sqlparse==0.4.2"
}
base_plugins = {
"query-parser",
"metadata-usage",
@ -102,7 +109,8 @@ plugins: Dict[str, Set[str]] = {
"sample-data": {"faker~=8.1.1", "pandas~=1.3.1"},
"superset": {},
"tableau": {"tableau-api-lib==0.1.22"},
"vertica": {"sqlalchemy-vertica[vertica-python]>=0.0.5"}
"vertica": {"sqlalchemy-vertica[vertica-python]>=0.0.5"},
"report-server": report_requirements
}
build_options = {"includes": ["_cffi_backend"]}

43
ingestion/src/metadata/cmd.py
View File

@ -78,5 +78,48 @@ def ingest(config: str) -> None:
ret = workflow.print_status()
sys.exit(ret)
@metadata.command()
@click.option(
"-c",
"--config",
type=click.Path(exists=True, dir_okay=False),
help="Workflow config",
required=True,
)
def report(config: str) -> None:
"""Report command to generate static pages with metadata"""
config_file = pathlib.Path(config)
workflow_config = load_config_file(config_file)
file_sink = {'type': 'file', 'config': {'filename': '/tmp/datasets.json'}}
try:
logger.info(f"Using config: {workflow_config}")
if workflow_config.get('cron'):
del workflow_config['cron']
if workflow_config.get('sink'):
del workflow_config['sink']
workflow_config['sink'] = file_sink
### add json generation as the sink
workflow = Workflow.create(workflow_config)
except ValidationError as e:
click.echo(e, err=True)
sys.exit(1)
workflow.execute()
workflow.stop()
ret = workflow.print_status()
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metadata_server.openmetadata.settings')
try:
from django.core.management import call_command
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
call_command('runserver', 'localhost:8000')
except ImportError as exc:
logger.error(exc)
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
metadata.add_command(check)

2
ingestion/src/metadata/ingestion/sink/file.py
View File

@ -57,7 +57,7 @@ class FileSink(Sink):
if self.wrote_something:
self.file.write(",\n")
self.file.write(record.to_json())
self.file.write(record.json())
self.wrote_something = True
self.report.records_written(record)

5
ingestion/src/metadata/ingestion/source/sql_source.py
View File

@ -257,13 +257,14 @@ class SQLSource(Source):
self.status.scanned('{}.{}'.format(self.config.get_service_name(), table_name))
description = _get_table_description(schema, table_name, inspector)
fqn = f"{self.config.service_name}.{self.config.database}.{schema}.{table_name}"
table_columns = self._get_columns(schema, table_name, inspector)
table_entity = Table(
id=uuid.uuid4(),
name=table_name,
tableType='Regular',
description=description if description is not None else ' ',
fullyQualifiedName=fqn,
columns=table_columns
)
if self.sql_config.generate_sample_data:
@ -314,11 +315,13 @@ class SQLSource(Source):
description = _get_table_description(schema, view_name, inspector)
table_columns = self._get_columns(schema, view_name, inspector)
fqn = f"{self.config.service_name}.{self.config.database}.{schema}.{view_name}"
table = Table(
id=uuid.uuid4(),
name=view_name,
tableType='View',
description=description if description is not None else ' ',
fullyQualifiedName=fqn,
columns=table_columns,
viewDefinition=view_definition
)

0
ingestion/src/metadata_server/__init__.py Normal file
View File

0
ingestion/src/metadata_server/db.sqlite3 Normal file
View File

22
ingestion/src/metadata_server/manage.py Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
"""OpenMetadata Server for generated reports."""
import os
import sys
def main():
"""Run OpenMetadata report server."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openmetadata.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

0
ingestion/src/metadata_server/openmetadata/__init__.py Normal file
View File

16
ingestion/src/metadata_server/openmetadata/asgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
ASGI config for openmetadata project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metadata_server.openmetadata.settings')
application = get_asgi_application()

120
ingestion/src/metadata_server/openmetadata/settings.py Normal file
View File

@ -0,0 +1,120 @@
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-d+ngmo&w9xlfb)=x0)q-93*bb66r2hkhz$-@^=+8*30553@z@e'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ['*']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'metadata_server.openmetadata.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [BASE_DIR / "templates"],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'metadata_server.openmetadata.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
# Password validation
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [BASE_DIR / 'static', ]
MEDIA_URL = '/media/'
MEDIA_ROOT = BASE_DIR / 'media'
# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

10
ingestion/src/metadata_server/openmetadata/urls.py Normal file
View File

@ -0,0 +1,10 @@
from django.contrib import admin
from django.urls import path
from .views import list_page, detail_page
urlpatterns = [
path('admin/', admin.site.urls),
path('', list_page),
path('dataset/<str:fqn>/', detail_page, name="dataset")
]
handler404 = "metadata_server.openmetadata.views.page_not_found_view"

37
ingestion/src/metadata_server/openmetadata/views.py Normal file
View File

@ -0,0 +1,37 @@
import logging
from django.shortcuts import render
import json
logger: logging.Logger = logging.getLogger(__name__)
with open("/tmp/datasets.json", "r") as d:
data = json.load(d)
datasets = []
for dataset in data:
if 'table' in dataset:
datasets.append(dataset['table'])
elif 'dashboard' in dataset:
datasets.append(dataset['dashboard'])
elif 'topic' in dataset:
datasets.append(dataset['topic'])
else:
logger.info("unrecognized element {}".format(dataset))
def list_page(request):
template_name = "list.html"
return render(request, template_name, {"datasets": datasets})
def detail_page(request, fqn):
template_name = "dataset.html"
dataset = list(filter(lambda data: data["fullyQualifiedName"] == fqn, datasets))
if len(dataset) > 0:
return render(request, template_name, {"dataset": dataset[0]})
else:
return render(request, template_name, {"error": "No Data Found"})
def page_not_found_view(request, exception):
return render(request, '404.html', status=404)

16
ingestion/src/metadata_server/openmetadata/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for openmetadata project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'metadata_server.openmetadata.settings')
application = get_wsgi_application()

14095
ingestion/src/metadata_server/static/css/style.css Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
ingestion/src/metadata_server/static/img/bigquery.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 9.3 KiB

BIN
ingestion/src/metadata_server/static/img/favicon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 KiB

12
ingestion/src/metadata_server/static/img/logo.svg Normal file
View File

@ -0,0 +1,12 @@
<svg width="250" height="300" viewBox="0 0 250 300" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M250 63H0V237.5V238H0.00391658C0.541243 272.288 56.298 300 125 300C193.702 300 249.459 272.288 249.996 238H250V237.5V63Z" fill="url(#paint0_linear)"/>
<ellipse cx="125" cy="62.5" rx="125" ry="62.5" fill="#8D6AF1"/>
<ellipse cx="125.5" cy="63.25" rx="62.5" ry="31.25" fill="white"/>
<path d="M85 214.793V254.384C85 258.223 80.8526 260.63 77.5193 258.725L45.5193 240.44C43.9614 239.549 43 237.893 43 236.098V141.887C43 137.988 47.2635 135.589 50.5956 137.613L121.936 180.942C123.514 181.901 125.492 181.912 127.081 180.97L200.45 137.476C203.783 135.5 208 137.902 208 141.777V237.975C208 239.835 206.968 241.541 205.32 242.404L173.32 259.166C169.991 260.91 166 258.495 166 254.737V214.577C166 210.747 161.869 208.339 158.536 210.226L127.026 228.07C125.464 228.954 123.548 228.934 122.006 228.016L92.5565 210.496C89.2236 208.513 85 210.915 85 214.793Z" fill="white"/>
<defs>
<linearGradient id="paint0_linear" x1="125" y1="90" x2="125" y2="300" gradientUnits="userSpaceOnUse">
<stop stop-color="#35198E"/>
<stop offset="1" stop-color="#8D6AF1"/>
</linearGradient>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.2 KiB

43
ingestion/src/metadata_server/static/img/no-data-found.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 20 KiB

9
ingestion/src/metadata_server/templates/404.html Normal file
View File

@ -0,0 +1,9 @@
{% extends 'layout/base.html' %}
{% load static %}
{% block content %}
<div class="tw-flex tw-flex-col tw-place-items-center tw-mt-24" data-testid="no-page-found">
<div class="tw-flex tw-flex-col tw-mt-24 tw-place-items-center" data-testid="error"> <img data-testid="no-data-found" src="{% static 'img/no-data-found.svg' %}" width="500"></div>
<div class="tw-flex tw-mt-3 tw-justify-around" data-testid="route-links"><a class="tw-mr-2" href="/"><button class="tw-align-bottom tw-inline-flex tw-items-center tw-justify-center tw-cursor-pointer
tw-leading-5 tw-transition-colors tw-duration-150 tw-font-medium focus:tw-outline-none tw-px-4 tw-py-2 tw-rounded-lg tw-text-sm tw-text-white tw-bg-primary tw-border tw-border-transparent active:tw-bg-primary-active hover:tw-bg-primary-hover focus:tw-bg-primary-active focus:tw-ring focus:tw-ring-purple-300 " type="button">Go to Home</button></a></div>
</div>
{% endblock content %}

157
ingestion/src/metadata_server/templates/dataset.html Normal file
View File

File diff suppressed because one or more lines are too long

21
ingestion/src/metadata_server/templates/layout/base.html Normal file
View File

@ -0,0 +1,21 @@
{% load static %}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="shortcut icon" href="{% static 'img/favicon.png' %}" type="image/png" />
<link rel="stylesheet" href="{% static 'css/style.css' %}">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.3/css/all.css">
<title>{% block title %}{% endblock title %}</title>
</head>
<body>
{% include "layout/header.html" %}
<div class="main-container">
<div class="content-wrapper">
{% block content %}{% endblock content %}
</div>
</div>
</body>
</html>

54
ingestion/src/metadata_server/templates/layout/header.html Normal file
View File

@ -0,0 +1,54 @@
{% load static %}
<div class="tw-h-14 tw-py-2 tw-px-5 tw-border-b-2 tw-border-separator">
<div class="tw-flex tw-items-center tw-flex-row tw-justify-between tw-flex-nowrap">
<div class="tw-flex tw-items-center tw-flex-row tw-justify-between tw-flex-nowrap tw-mr-auto">
<a aria-current="page" class="active" href="/"><img alt="OpenMetadata Logo" class="svg-icon " src="{% static 'img/logo.svg' %}" width="30"></a>
<div class="tw-flex-none tw-relative tw-pl-5 " data-testid="appbar-item"><span class="fa fa-search tw-absolute tw-block tw-z-10 tw-w-9 tw-mt-2 tw-h-8 tw-leading-8 tw-text-center tw-pointer-events-none tw-text-gray-400"></span><input class="tw-relative search-grey tw-rounded tw-border tw-border-main tw-bg-body-main focus:tw-outline-none tw-pl-8 tw-py-1" type="text" value=""></div>
<div class="tw-ml-9">
<a class="tw-nav focus:tw-no-underline" data-testid="appbar-item" href="/explore" style="color: rgb(55, 53, 47);">Explore</a>
<div class="tw-relative tw-inline-block tw-text-left false" data-testid="dropdown-item">
<div>
<button aria-expanded="true" aria-haspopup="true" class="tw-inline-flex tw-px-4 tw-py-2 focus:tw-outline-none tw-justify-center tw-nav " data-testid="menu-button" id="menu-button" type="button">
<p class="hover:tw-underline" style="color: rgb(55, 53, 47);">Settings</p>
<svg aria-hidden="true" class="tw-inline-block tw-h-5 tw-w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg" style="margin-top: 1px; color: rgb(55, 53, 47);">
<path clip-rule="evenodd" d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1
0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z" fill-rule="evenodd"></path>
</svg>
</button>
</div>
</div>
</div>
</div>
<button class="tw-nav focus:tw-no-underline hover:tw-underline"><img alt="Doc icon" class="svg-icon tw-align-middle tw--mt-0.5 tw-mr-1" src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZlcnNpb249IjEuMSIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHhtbG5zOnN2Z2pzPSJodHRwOi8vc3ZnanMuY29tL3N2Z2pzIiB3aWR0aD0iNTEyIiBoZWlnaHQ9IjUxMiIgeD0iMCIgeT0iMCIgdmlld0JveD0iMCAwIDUxMiA1MTIiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDUxMiA1MTIiIHhtbDpzcGFjZT0icHJlc2VydmUiIGZpbGw9IiMzNzM1MmYiPgoJPGc+PGcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KCQk8cGF0aCBkPSJtNDUxLjA4MSAzMDEuMTcxYy01Ljg1OC01Ljg1Ny0xNS4zNTUtNS44NTctMjEuMjEzIDBzLTUuODU4IDE1LjM1NSAwIDIxLjIxM2wzMC4yNTkgMzAuMjU5YzIuOTI5IDIuOTI5IDYuNzY3IDQuMzkzIDEwLjYwNiA0LjM5M3M3LjY3OC0xLjQ2NSAxMC42MDctNC4zOTNjNS44NTgtNS44NTcgNS44NTgtMTUuMzU1IDAtMjEuMjEzeiIgZGF0YS1vcmlnaW5hbD0iIzAwMDAwMCIgc3R5bGU9IiI+PC9wYXRoPgoJCTxwYXRoIGQ9Im00NDAuNDc0IDE3Ny4wMzdjMy44MzggMCA3LjY3OC0xLjQ2NSAxMC42MDctNC4zOTNsMzAuMjU5LTMwLjI1OWM1Ljg1OC01Ljg1NyA1Ljg1OC0xNS4zNTUgMC0yMS4yMTNzLTE1LjM1NS01Ljg1OC0yMS4yMTMgMGwtMzAuMjU5IDMwLjI1OWMtNS44NTggNS44NTctNS44NTggMTUuMzU1IDAgMjEuMjEzIDIuOTI5IDIuOTI5IDYuNzY3IDQuMzkzIDEwLjYwNiA0LjM5M3oiIGRhdGEtb3JpZ2luYWw9IiMwMDAwMDAiIHN0eWxlPSIiPjwvcGF0aD4KCQk8cGF0aCBkPSJtNDk3IDIyMS45MDdoLTQyLjc5M2MtOC4yODQgMC0xNSA2LjcxNi0xNSAxNXM2LjcxNiAxNSAxNSAxNWg0Mi43OTNjOC4yODQgMCAxNS02LjcxNiAxNS0xNXMtNi43MTYtMTUtMTUtMTV6IiBkYXRhLW9yaWdpbmFsPSIjMDAwMDAwIiBzdHlsZT0iIj48L3BhdGg+CgkJPHBhdGggZD0ibTM5OS4yNDcgMjMuNTg4Yy00LjczNi0yLjY1My0xMC41MzYtMi41MzktMTUuMTY1LjI5NWwtMTcxLjc4NSAxMDUuMjA1aC03Ni4yNjFjLTE2LjI4Ny0uODYxLTMxLjc4MSA0LjgxNS00My42NzUgMTYuMDExLTQuNDI3IDQuMTY2LTguMTU5IDguOTc1LTExLjEzMyAxNC4yMTVoLTM3Ljc0MWMtMjMuOTc5IDAtNDMuNDg3IDE5LjUwOC00My40ODcgNDMuNDg3djY4LjIxMmMwIDIzLjk3OSAxOS41MDggNDMuNDg3IDQzLjQ4NyA0My40ODdoMzcuNjY2YzYuMDQgMTAuNzk4IDE1LjE1OCAxOS43NDkgMjYuMjQzIDI1LjY3M2wtNTIuMjYxIDkwLjUyYy0yLjAwNSAzLjQ3My0yLjUzNiA3LjYwNC0xLjQ3NSAxMS40NzIgMS4wNjIgMy44NjcgMy42MjggNy4xNDggNy4xMjUgOS4xMWw2Ni4xODIgMzcuMTI5YzIuMzIgMS4zMDIgNC44NCAxLjkyMSA3LjMyNiAxLjkyMSA1LjE4NyAwIDEwLjIyOC0yLjY5NSAxMy4wMDMtNy41MDNsNzQuNjI2LTEyOS4yNTggMTYyLjMzIDk2LjQ2OWMyLjM2IDEuNDAzIDUuMDExIDIuMTA1IDcuNjYzIDIuMTA1IDIuNTU4IDAgNS4xMTctLjY1MyA3LjQxOC0xLjk2MyA0LjY4Ny0yLjY2NyA3LjU4Mi03LjY0NSA3LjU4Mi0xMy4wMzd2LTQwMC40NjNjMC01LjQyOC0yLjkzMy0xMC40MzMtNy42NjgtMTMuMDg3em0tMzU1Ljc2IDI2MC45MTJjLTcuNDM3IDAtMTMuNDg3LTYuMDUtMTMuNDg3LTEzLjQ4N3YtNjguMjEyYzAtNy40MzcgNi4wNS0xMy40ODcgMTMuNDg3LTEzLjQ4N2gyOS42NTRjLS4wMDIuMjI0LS4wMTcuNDQ3LS4wMTcuNjcxdjkzLjg0NGMwIC4yMjUuMDE0LjQ0Ny4wMTcuNjcxem01OS42MzctOTQuNTE1YzAtOC43ODIgMy41NzEtMTcuMTggOS43OTctMjMuMDQgNS45MjYtNS41NzcgMTMuNjU4LTguMzgyIDIxLjc4My03Ljg4NC4zMDMuMDE4IDY2LjgyMy4wMjcgNjYuODIzLjAyN3YxNTguODJoLTY1LjQyMmMtMTguNTI5LTEuMzMxLTMyLjk4MS0xNi4yMjQtMzIuOTgxLTM0LjA4em0yNS42IDI2NS4wMDYtNDAuMDE2LTIyLjQ0OSA0Ny45NzktODMuMTAzYy4yODktLjUuNTM4LTEuMDEzLjc2My0xLjUzaDUzLjA5N3ptMjQ4LjE5MS00NC4yMTYtMTQ1LjM4OC04Ni40MDF2LTE3MS44ODNsMTQ1LjM4OC04OS4wNHoiIGRhdGEtb3JpZ2luYWw9IiMwMDAwMDAiIHN0eWxlPSIiPjwvcGF0aD4KCTwvZz48L2c+Cjwvc3ZnPg==" width="16"><span>What's new</span></button>
<div>
<div class="tw-relative tw-inline-block tw-text-left false" data-testid="dropdown-item">
<div>
<button aria-expanded="true" aria-haspopup="true" class="tw-inline-flex tw-px-4 tw-py-2 focus:tw-outline-none tw-justify-center tw-nav " data-testid="menu-button" id="menu-button" type="button">
<img alt="Doc icon" class="svg-icon tw-align-middle tw-mt-0.5 tw-mr-1" src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTEyIiBoZWlnaHQ9IjUxMiIgdmlld0JveD0iMCAwIDUxMiA1MTIiIGZpbGw9IiMzNzM1MmYiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxwYXRoIGQ9Ik0yNTYgMEMxMTQuNTA5IDAgMCAxMTQuNDk2IDAgMjU2QzAgMzk3LjQ4OSAxMTQuNDk2IDUxMiAyNTYgNTEyQzM5Ny40OTEgNTEyIDUxMiAzOTcuNTA0IDUxMiAyNTZDNTEyIDExNC41MDkgMzk3LjUwNCAwIDI1NiAwWk0yNTYgNDc2LjI3OUMxMzQuNTM4IDQ3Ni4yNzkgMzUuNzIxIDM3Ny40NjMgMzUuNzIxIDI1NkMzNS43MjEgMTM0LjUzNyAxMzQuNTM4IDM1LjcyMSAyNTYgMzUuNzIxQzM3Ny40NjMgMzUuNzIxIDQ3Ni4yNzkgMTM0LjUzNyA0NzYuMjc5IDI1NkM0NzYuMjc5IDM3Ny40NjMgMzc3LjQ2MyA0NzYuMjc5IDI1NiA0NzYuMjc5WiIgZmlsbD0iIzM3MzUyZiIvPgo8cGF0aCBkPSJNMjQ4LjQyNSAzMjMuOTI0QzIzNC4yNzIgMzIzLjkyNCAyMjIuODE1IDMzNS43MTggMjIyLjgxNSAzNDkuODdDMjIyLjgxNSAzNjMuNjg3IDIzMy45MzUgMzc1LjgxOCAyNDguNDI1IDM3NS44MThDMjYyLjkxNSAzNzUuODE4IDI3NC4zNzEgMzYzLjY4NyAyNzQuMzcxIDM0OS44N0MyNzQuMzcxIDMzNS43MTggMjYyLjU3NyAzMjMuOTI0IDI0OC40MjUgMzIzLjkyNFoiIGZpbGw9IiMzNzM1MmYiLz4KPHBhdGggZD0iTTI1Mi44MDUgMTI3LjQ2OUMyMDcuMzEzIDEyNy40NjkgMTg2LjQyMSAxNTQuNDI4IDE4Ni40MjEgMTcyLjYyNEMxODYuNDIxIDE4NS43NjYgMTk3LjU0MSAxOTEuODMyIDIwNi42MzkgMTkxLjgzMkMyMjQuODM2IDE5MS44MzIgMjE3LjQyMyAxNjUuODg0IDI1MS43OTQgMTY1Ljg4NEMyNjguNjQyIDE2NS44ODQgMjgyLjEyMiAxNzMuMjk4IDI4Mi4xMjIgMTg4Ljc5OUMyODIuMTIyIDIwNi45OTUgMjYzLjI1MSAyMTcuNDQxIDI1Mi4xMzEgMjI2Ljg3NkMyNDIuMzU4IDIzNS4yOTkgMjI5LjU1NCAyNDkuMTE2IDIyOS41NTQgMjc4LjA5NkMyMjkuNTU0IDI5NS42MTggMjM0LjI3MiAzMDAuNjczIDI0OC4wODcgMzAwLjY3M0MyNjQuNTk4IDMwMC42NzMgMjY3Ljk2OCAyOTMuMjYgMjY3Ljk2OCAyODYuODU2QzI2Ny45NjggMjY5LjMzNCAyNjguMzA1IDI1OS4yMjUgMjg2LjgzOSAyNDQuNzM1QzI5NS45MzcgMjM3LjY1OSAzMjQuNTc5IDIxNC43NDQgMzI0LjU3OSAxODMuMDY5QzMyNC41NzkgMTUxLjM5NCAyOTUuOTM3IDEyNy40NjkgMjUyLjgwNSAxMjcuNDY5WiIgZmlsbD0iIzM3MzUyZiIvPgo8L3N2Zz4K" width="16">
<p class="hover:tw-underline" style="color: rgb(55, 53, 47);">Need Help</p>
<svg aria-hidden="true" class="tw-inline-block tw-h-5 tw-w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg" style="margin-top: 1px; color: rgb(55, 53, 47);">
<path clip-rule="evenodd" d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1
0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z" fill-rule="evenodd"></path>
</svg>
</button>
</div>
</div>
</div>
<div data-testid="dropdown-profile">
<div class="tw-relative tw-inline-block tw-text-left false" data-testid="dropdown-item">
<div>
<button aria-expanded="true" aria-haspopup="true" class="tw-inline-flex tw-px-4 tw-py-2 focus:tw-outline-none tw-justify-center tw-nav " data-testid="menu-button" id="menu-button" type="button">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="tw-mr-1" style="height: 22px; width: 22px; border-radius: 50%;">
<path d="M511.676 498.752l-12.8-51.2a74.345 74.345 0 00-48.747-52.885l-93.867-31.275c-22.891-9.536-33.365-46.4-35.627-60.395a88.703 88.703 0 0031.36-57.664 17.391 17.391 0 012.581-10.987 10.665 10.665 0 007.317-6.4 159.447 159.447 0 0011.435-46.613 10.254 10.254 0 00-.32-2.581 27.434 27.434 0 00-10.347-15.531v-56.555c0-34.368-10.496-48.469-21.547-56.64C339.004 33.472 321.276 0 255.996 0c-57.917 2.332-104.335 48.75-106.667 106.667v56.555a27.434 27.434 0 00-10.347 15.531 10.26 10.26 0 00-.32 2.581 159.454 159.454 0 0011.435 46.635 9.067 9.067 0 006.4 6.123c1.195.597 3.435 3.691 3.435 11.243a88.683 88.683 0 0031.531 57.771c-2.24 13.973-12.651 50.816-34.901 60.117l-94.699 31.445a74.346 74.346 0 00-48.725 52.843l-12.8 51.2c-1.449 5.71 2.005 11.514 7.715 12.963.853.217 1.73.327 2.61.328H501.33c5.891-.002 10.665-4.779 10.664-10.67a10.722 10.722 0 00-.318-2.58z" fill="#455a64"></path>
</svg>
<p class="hover:tw-underline" style="color: rgb(55, 53, 47);"><span>Welcome, <span class="tw-font-medium">Aaron</span></span></p>
<svg aria-hidden="true" class="tw-inline-block tw-h-5 tw-w-5" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg" style="margin-top: 1px; color: rgb(55, 53, 47);">
<path clip-rule="evenodd" d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1
0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z" fill-rule="evenodd"></path>
</svg>
</button>
</div>
</div>
</div>
</div>
</div>

71
ingestion/src/metadata_server/templates/list.html Normal file
View File

File diff suppressed because one or more lines are too long