yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-17 13:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(neo4j): Allow datahub to connect to specific neo4j database (#9179)

Browse Source 
Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com>
This commit is contained in:
deepgarg-visa 2023-11-08 22:32:13 +05:30 committed by GitHub
parent e73e926999
commit 399e032dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 73 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker/docker-compose.override.yml
View File

@ -7,8 +7,12 @@ services:
environment: environment:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
- GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
volumes: volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins - ${HOME}/.datahub/plugins:/etc/datahub/plugins
datahub-upgrade:
environment:
- GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
mysql-setup: mysql-setup:
container_name: mysql-setup container_name: mysql-setup
hostname: mysql-setup hostname: mysql-setup

62
docker/quickstart/docker-compose-m1.quickstart.yml
View File

@ -81,32 +81,32 @@ services:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
- EBEAN_DATASOURCE_USERNAME=datahub
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - EBEAN_DATASOURCE_HOST=mysql:3306
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
- EBEAN_DATASOURCE_USERNAME=datahub
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- NEO4J_HOST=http://neo4j:7474 - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- NEO4J_URI=bolt://neo4j - ELASTICSEARCH_PORT=9200
- NEO4J_USERNAME=neo4j
- NEO4J_PASSWORD=datahub
- JAVA_OPTS=-Xms1g -Xmx1g
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=neo4j
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- ENTITY_SERVICE_ENABLE_RETENTION=true - ENTITY_SERVICE_ENABLE_RETENTION=true
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
- JAVA_OPTS=-Xms1g -Xmx1g
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- MAE_CONSUMER_ENABLED=true - MAE_CONSUMER_ENABLED=true
- MCE_CONSUMER_ENABLED=true - MCE_CONSUMER_ENABLED=true
- METADATA_SERVICE_AUTH_ENABLED=false
- NEO4J_HOST=http://neo4j:7474
- NEO4J_PASSWORD=datahub
- NEO4J_URI=bolt://neo4j
- NEO4J_USERNAME=neo4j
- PE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true
- UI_INGESTION_ENABLED=true - UI_INGESTION_ENABLED=true
- METADATA_SERVICE_AUTH_ENABLED=false
healthcheck: healthcheck:
interval: 1s interval: 1s
retries: 3 retries: 3
@ -134,23 +134,23 @@ services:
neo4j: neo4j:
condition: service_healthy condition: service_healthy
environment: environment:
- EBEAN_DATASOURCE_USERNAME=datahub - BACKFILL_BROWSE_PATHS_V2=true
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- GRAPH_SERVICE_IMPL=elasticsearch
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_USERNAME=datahub
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_PORT=9200
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- BACKFILL_BROWSE_PATHS_V2=true - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- REPROCESS_DEFAULT_BROWSE_PATHS_V2=false - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
hostname: datahub-upgrade hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}

62
docker/quickstart/docker-compose.quickstart.yml
View File

@ -81,32 +81,32 @@ services:
- DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
- DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
- DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms
- EBEAN_DATASOURCE_USERNAME=datahub
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092 - EBEAN_DATASOURCE_HOST=mysql:3306
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2
- EBEAN_DATASOURCE_USERNAME=datahub
- ELASTICSEARCH_HOST=elasticsearch - ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- NEO4J_HOST=http://neo4j:7474 - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- NEO4J_URI=bolt://neo4j - ELASTICSEARCH_PORT=9200
- NEO4J_USERNAME=neo4j
- NEO4J_PASSWORD=datahub
- JAVA_OPTS=-Xms1g -Xmx1g
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=neo4j
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- ENTITY_SERVICE_ENABLE_RETENTION=true - ENTITY_SERVICE_ENABLE_RETENTION=true
- ES_BULK_REFRESH_POLICY=WAIT_UNTIL
- GRAPH_SERVICE_DIFF_MODE_ENABLED=true
- GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
- JAVA_OPTS=-Xms1g -Xmx1g
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- MAE_CONSUMER_ENABLED=true - MAE_CONSUMER_ENABLED=true
- MCE_CONSUMER_ENABLED=true - MCE_CONSUMER_ENABLED=true
- METADATA_SERVICE_AUTH_ENABLED=false
- NEO4J_HOST=http://neo4j:7474
- NEO4J_PASSWORD=datahub
- NEO4J_URI=bolt://neo4j
- NEO4J_USERNAME=neo4j
- PE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true
- UI_INGESTION_ENABLED=true - UI_INGESTION_ENABLED=true
- METADATA_SERVICE_AUTH_ENABLED=false
healthcheck: healthcheck:
interval: 1s interval: 1s
retries: 3 retries: 3
@ -134,23 +134,23 @@ services:
neo4j: neo4j:
condition: service_healthy condition: service_healthy
environment: environment:
- EBEAN_DATASOURCE_USERNAME=datahub - BACKFILL_BROWSE_PATHS_V2=true
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_PORT=9200
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- GRAPH_SERVICE_IMPL=elasticsearch
- DATAHUB_GMS_HOST=datahub-gms - DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080 - DATAHUB_GMS_PORT=8080
- EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
- EBEAN_DATASOURCE_HOST=mysql:3306
- EBEAN_DATASOURCE_PASSWORD=datahub
- EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
- EBEAN_DATASOURCE_USERNAME=datahub
- ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false
- ELASTICSEARCH_HOST=elasticsearch
- ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true
- ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true
- ELASTICSEARCH_PORT=9200
- ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
- BACKFILL_BROWSE_PATHS_V2=true - GRAPH_SERVICE_IMPL=${GRAPH_SERVICE_IMPL:-elasticsearch}
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
- REPROCESS_DEFAULT_BROWSE_PATHS_V2=false - REPROCESS_DEFAULT_BROWSE_PATHS_V2=false
hostname: datahub-upgrade hostname: datahub-upgrade
image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head}

1
metadata-service/configuration/src/main/resources/application.yml
View File

@ -251,6 +251,7 @@ neo4j:
username: ${NEO4J_USERNAME:neo4j} username: ${NEO4J_USERNAME:neo4j}
password: ${NEO4J_PASSWORD:datahub} password: ${NEO4J_PASSWORD:datahub}
uri: ${NEO4J_URI:bolt://localhost} uri: ${NEO4J_URI:bolt://localhost}
database: ${NEO4J_DATABASE:graph.db}
maxConnectionPoolSize: ${NEO4J_MAX_CONNECTION_POOL_SIZE:100} maxConnectionPoolSize: ${NEO4J_MAX_CONNECTION_POOL_SIZE:100}
maxConnectionAcquisitionTimeout: ${NEO4J_MAX_CONNECTION_ACQUISITION_TIMEOUT_IN_SECONDS:60} maxConnectionAcquisitionTimeout: ${NEO4J_MAX_CONNECTION_ACQUISITION_TIMEOUT_IN_SECONDS:60}
maxConnectionLifetimeInSeconds: ${NEO4j_MAX_CONNECTION_LIFETIME_IN_SECONDS:3600} maxConnectionLifetimeInSeconds: ${NEO4j_MAX_CONNECTION_LIFETIME_IN_SECONDS:3600}

7
metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/Neo4jGraphServiceFactory.java
View File

@ -6,8 +6,10 @@ import com.linkedin.metadata.graph.neo4j.Neo4jGraphService;
import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import org.neo4j.driver.Driver; import org.neo4j.driver.Driver;
import org.neo4j.driver.SessionConfig;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
@ -24,10 +26,13 @@ public class Neo4jGraphServiceFactory {
@Qualifier("entityRegistry") @Qualifier("entityRegistry")
private EntityRegistry entityRegistry; private EntityRegistry entityRegistry;
@Value("${neo4j.database}")
private String neo4jDatabase;
@Bean(name = "neo4jGraphService") @Bean(name = "neo4jGraphService")
@Nonnull @Nonnull
protected Neo4jGraphService getInstance() { protected Neo4jGraphService getInstance() {
LineageRegistry lineageRegistry = new LineageRegistry(entityRegistry); LineageRegistry lineageRegistry = new LineageRegistry(entityRegistry);
return new Neo4jGraphService(lineageRegistry, neo4jDriver); return new Neo4jGraphService(lineageRegistry, neo4jDriver, SessionConfig.forDatabase(neo4jDatabase));
} }
} }