diff --git a/docker/local-metadata/Dockerfile b/docker/local-metadata/Dockerfile index cbac6c53dc3..0472d0f4849 100644 --- a/docker/local-metadata/Dockerfile +++ b/docker/local-metadata/Dockerfile @@ -16,8 +16,8 @@ EXPOSE 8585 RUN apt-get update -y && \ apt-get install --no-install-recommends wget -y && \ rm -rf /var/lib/apt/lists/* -COPY docker/local-metadata/openmetadata-start.sh / -COPY docker/local-metadata/openmetadata.yaml / +COPY docker/metadata/openmetadata-start.sh / +COPY docker/metadata/openmetadata.yaml / COPY openmetadata-dist/target/openmetadata-*.tar.gz / RUN chmod 777 openmetadata-start.sh && tar zxvf openmetadata-*.tar.gz CMD ["./openmetadata-start.sh"] \ No newline at end of file diff --git a/docker/local-metadata/openmetadata-start.sh b/docker/local-metadata/openmetadata-start.sh deleted file mode 100644 index a246dc983f6..00000000000 --- a/docker/local-metadata/openmetadata-start.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done -mv /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml -cd /openmetadata-*/ -./bootstrap/bootstrap_storage.sh migrate -./bin/openmetadata-server-start.sh conf/openmetadata.yaml \ No newline at end of file diff --git a/docker/local-metadata/openmetadata.yaml b/docker/local-metadata/openmetadata.yaml deleted file mode 100644 index f63a6e0bf7d..00000000000 --- a/docker/local-metadata/openmetadata.yaml +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -swagger: - resourcePackage: org.openmetadata.catalog.resources - - -server: - rootPath: '/api/*' - applicationConnectors: - - type: http - port: 8585 - adminConnectors: - - type: http - port: 8586 - - # Above configuration for running http is fine for dev and testing. - # For production setup, where UI app will hit apis through DPS it - # is strongly recommended to run https instead. Note that only - # keyStorePath and keyStorePassword are mandatory properties. Values - # for other properties are defaults - #server: - #applicationConnectors: - # - type: https - # port: 8585 - # keyStorePath: ./conf/keystore.jks - # keyStorePassword: changeit - # keyStoreType: JKS - # keyStoreProvider: - # trustStorePath: /path/to/file - # trustStorePassword: changeit - # trustStoreType: JKS - # trustStoreProvider: - # keyManagerPassword: changeit - # needClientAuth: false - # wantClientAuth: - # certAlias: - # crlPath: /path/to/file - # enableCRLDP: false - # enableOCSP: false - # maxCertPathLength: (unlimited) - # ocspResponderUrl: (none) - # jceProvider: (none) - # validateCerts: true - # validatePeers: true - # supportedProtocols: SSLv3 - # supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 - # allowRenegotiation: true - # endpointIdentificationAlgorithm: (none) - - #adminConnectors: - # - type: https - # port: 8586 - # keyStorePath: ./conf/keystore.jks - # keyStorePassword: changeit - # keyStoreType: JKS - # keyStoreProvider: - # trustStorePath: /path/to/file - # trustStorePassword: changeit - # trustStoreType: JKS - # trustStoreProvider: - # keyManagerPassword: changeit - # needClientAuth: false - # wantClientAuth: - # certAlias: - # crlPath: /path/to/file - # enableCRLDP: false - # enableOCSP: false - # maxCertPathLength: (unlimited) - # ocspResponderUrl: (none) - # jceProvider: (none) - # validateCerts: true - # validatePeers: true - # supportedProtocols: SSLv3 - # supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 - # allowRenegotiation: true - # endpointIdentificationAlgorithm: (none) - -# Logging settings. -# https://logback.qos.ch/manual/layouts.html#conversionWord -logging: - level: INFO - loggers: - org.openmetadata.catalog.common: DEBUG - io.swagger: ERROR - appenders: - - type: file - threshold: TRACE - logFormat: "%level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %msg %n" - currentLogFilename: ./logs/openmetadata.log - archivedLogFilenamePattern: ./logs/openmetadata-%d{yyyy-MM-dd}-%i.log.gz - archivedFileCount: 7 - timeZone: UTC - maxFileSize: 50MB - -database: - # the name of the JDBC driver, mysql in our case - driverClass: com.mysql.cj.jdbc.Driver - # the username and password - user: openmetadata_user - password: openmetadata_password - # the JDBC URL; the database is called openmetadata_db - url: jdbc:mysql://mysql:3306/openmetadata_db?allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=UTC - - -elasticsearch: - host: elasticsearch - port: 9200 - -eventHandlerConfiguration: - eventHandlerClassNames: - - "org.openmetadata.catalog.events.AuditEventHandler" - - "org.openmetadata.catalog.events.ElasticSearchEventHandler" - -airflowConfiguration: - apiEndpoint: "http://ingestion:8080" - username: "admin" - password: "admin" - metadataApiEndpoint: "http://openmetadata-server:8585/api" - authProvider: "no-auth" - -health: - delayedShutdownHandlerEnabled: true - shutdownWaitPeriod: 1s - healthCheckUrlPaths: ["/api/v1/health-check"] - healthChecks: - - name: UserDatabaseCheck - critical: true - schedule: - checkInterval: 2500ms - downtimeInterval: 10s - failureAttempts: 2 - successAttempts: 1 diff --git a/docker/metadata/Dockerfile b/docker/metadata/Dockerfile index 33d55688a9a..7efd2e966ab 100644 --- a/docker/metadata/Dockerfile +++ b/docker/metadata/Dockerfile @@ -1,3 +1,14 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM openjdk:11 EXPOSE 8585 diff --git a/docker/metadata/openmetadata-start.sh b/docker/metadata/openmetadata-start.sh index fed3f9689a8..66ca998ad83 100644 --- a/docker/metadata/openmetadata-start.sh +++ b/docker/metadata/openmetadata-start.sh @@ -13,5 +13,5 @@ while ! wget -O /dev/null -o /dev/null mysql:3306; do sleep 5; done cp /openmetadata.yaml /openmetadata-*/conf/openmetadata.yaml cd /openmetadata-*/ -./bootstrap/bootstrap_storage.sh migrate +./bootstrap/bootstrap_storage.sh drop-create-all ./bin/openmetadata-server-start.sh conf/openmetadata.yaml diff --git a/docker/metadata/openmetadata.yaml b/docker/metadata/openmetadata.yaml index 259e52e1cd8..91bc2138f1c 100644 --- a/docker/metadata/openmetadata.yaml +++ b/docker/metadata/openmetadata.yaml @@ -114,11 +114,20 @@ database: elasticsearch: host: elasticsearch port: 9200 + scheme: "http" eventHandlerConfiguration: eventHandlerClassNames: - "org.openmetadata.catalog.events.AuditEventHandler" - - "org.openmetadata.catalog.events.ElasticSearchEventHandler" + - "org.openmetadata.catalog.elasticsearch.ElasticSearchEventHandler" + - "org.openmetadata.catalog.events.ChangeEventHandler" + +airflowConfiguration: + apiEndpoint: "http://ingestion:8080" + username: "admin" + password: "admin" + metadataApiEndpoint: "http://openmetadata-server:8585/api" + authProvider: "no-auth" health: delayedShutdownHandlerEnabled: true diff --git a/docs/install/setup-ingestion.md b/docs/install/setup-ingestion.md index 3c13573016e..09862a22a39 100644 --- a/docs/install/setup-ingestion.md +++ b/docs/install/setup-ingestion.md @@ -61,13 +61,6 @@ metadata ingest -c ./examples/workflows/redshift.json source env/bin/activate metadata ingest -c ./pipelines/sample_usage.json ``` -#### Generate Sample Users - -```text - source env/bin/activate - metadata ingest -c ./pipelines/sample_users.json -``` - #### Ingest MySQL data to Metadata APIs ```text diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index bc37062394d..e12bee249ec 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -238,7 +238,7 @@ def docker(start, stop, clean, type, path) -> None: """\nHead to http://localhost:8585 to play around with OpenMetadata UI. \nTo checkout Ingestion via Airflow, go to http://localhost:8080 \n(username: admin, password: admin) """, - fg="bright_white", + fg="bright_blue", ) click.secho( "Need support? Get in touch on Slack: https://slack.open-metadata.org/",