From 0e3efabd2c19e24bcfb81602f897802be1cb1d06 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 10 Nov 2023 16:00:20 -0800 Subject: [PATCH] fix(build): set `@cliMajorVersion@` correctly (#9228) --- .github/workflows/build-and-test.yml | 6 +-- .github/workflows/check-datahub-jars.yml | 4 +- .github/workflows/code-checks.yml | 4 +- .github/workflows/docker-ingestion-smoke.yml | 4 +- .github/workflows/docker-postgres-setup.yml | 4 +- .github/workflows/docker-unified.yml | 52 +++++-------------- .github/workflows/publish-datahub-jars.yml | 4 +- .github/workflows/spark-smoke-test.yml | 5 +- metadata-service/configuration/build.gradle | 7 ++- .../src/main/resources/application.yml | 9 ++-- metadata-service/factories/build.gradle | 5 -- .../tests/read_only/test_services_up.py | 8 +++ 12 files changed, 39 insertions(+), 73 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 25f3957e8f..10c137a206 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -27,7 +27,7 @@ jobs: command: [ # metadata-ingestion and airflow-plugin each have dedicated build jobs "except_metadata_ingestion", - "frontend" + "frontend", ] timezone: ["UTC", "America/New_York"] runs-on: ubuntu-latest @@ -36,9 +36,7 @@ jobs: - uses: szenius/set-timezone@v1.0 with: timezoneLinux: ${{ matrix.timezone }} - - uses: actions/checkout@v3 - with: - fetch-depth: 800 + - uses: hsheth2/sane-checkout-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index 41f9ea91a9..8e507ea40f 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -27,9 +27,7 @@ jobs: command: ["datahub-client", "datahub-protobuf", "spark-lineage"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 800 + - uses: hsheth2/sane-checkout-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index e12971b8a6..38f0946678 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -31,9 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml index 82b57d2360..803ddc6fce 100644 --- a/.github/workflows/docker-ingestion-smoke.yml +++ b/.github/workflows/docker-ingestion-smoke.yml @@ -50,9 +50,7 @@ jobs: if: ${{ needs.setup.outputs.publish == 'true' }} steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml index fda4349f90..e4d6cfc106 100644 --- a/.github/workflows/docker-postgres-setup.yml +++ b/.github/workflows/docker-postgres-setup.yml @@ -46,9 +46,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 5f5a62de62..18cb946b95 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -74,9 +74,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-service:war:build -x test --parallel @@ -132,9 +130,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel @@ -190,9 +186,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Pre-build artifacts for docker image run: | ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel @@ -248,9 +242,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Pre-build artifacts for docker image run: | ./gradlew :datahub-upgrade:build -x test --parallel @@ -306,9 +298,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Pre-build artifacts for docker image run: | ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel @@ -366,9 +356,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: @@ -388,9 +376,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: @@ -410,9 +396,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - name: Build and push uses: ./.github/actions/docker-custom-build-and-push with: @@ -434,9 +418,7 @@ jobs: needs: setup steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: dorny/paths-filter@v2 id: filter with: @@ -468,9 +450,7 @@ jobs: needs: [setup, datahub_ingestion_base_build] steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: dorny/paths-filter@v2 id: filter with: @@ -510,9 +490,7 @@ jobs: needs: [setup, datahub_ingestion_base_build] steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: dorny/paths-filter@v2 id: filter with: @@ -554,9 +532,7 @@ jobs: needs: [setup, datahub_ingestion_base_slim_build] steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: dorny/paths-filter@v2 id: filter with: @@ -637,9 +613,7 @@ jobs: needs: [setup, datahub_ingestion_base_full_build] steps: - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 800 + uses: hsheth2/sane-checkout-action@v1 - uses: dorny/paths-filter@v2 id: filter with: diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 7cd07b130d..ec7985ef3b 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -48,9 +48,7 @@ jobs: needs: ["check-secret", "setup"] if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }} steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 800 + - uses: hsheth2/sane-checkout-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml index 541b2019b9..70b66d6452 100644 --- a/.github/workflows/spark-smoke-test.yml +++ b/.github/workflows/spark-smoke-test.yml @@ -29,10 +29,7 @@ jobs: spark-smoke-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 800 - fetch-tags: true + - uses: hsheth2/sane-checkout-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle index bf79469633..80cf654126 100644 --- a/metadata-service/configuration/build.gradle +++ b/metadata-service/configuration/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java' } +apply from: "../../gradle/versioning/versioning.gradle" dependencies { implementation externalDependency.jacksonDataBind @@ -12,4 +13,8 @@ dependencies { compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok -} \ No newline at end of file +} + +processResources.configure { + finalizedBy printVersionDetails // always print version details +} diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml index 46aa02d985..40674e13e6 100644 --- a/metadata-service/configuration/src/main/resources/application.yml +++ b/metadata-service/configuration/src/main/resources/application.yml @@ -44,6 +44,7 @@ authorization: restApiAuthorization: ${REST_API_AUTHORIZATION_ENABLED:false} ingestion: + # The value of cliMajorVersion is substituted in by the processResources Gradle task. enabled: ${UI_INGESTION_ENABLED:true} defaultCliVersion: "${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}" maxSerializedStringLength: "${INGESTION_MAX_SERIALIZED_STRING_LENGTH:16000000}" # Indicates the maximum allowed JSON String length Jackson will handle, impacts the maximum size of ingested aspects @@ -74,7 +75,7 @@ datahub: plugin: pluginSecurityMode: ${PLUGIN_SECURITY_MODE:RESTRICTED} # Possible value RESTRICTED or LENIENT, default to RESTRICTED entityRegistry: - path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models} + path: ${ENTITY_REGISTRY_PLUGIN_PATH:/etc/datahub/plugins/models} retention: path: ${RETENTION_PLUGIN_PATH:/etc/datahub/plugins/retention} auth: @@ -280,14 +281,13 @@ updateIndices: ingestionScheduler: enabled: ${ENABLE_INGESTION_SCHEDULER_HOOK:true} # enable to execute ingestion scheduling - bootstrap: upgradeDefaultBrowsePaths: enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones backfillBrowsePathsV2: enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate reprocessDefaultBrowsePathsV2: - enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]} + enabled: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:false} # reprocess V2 browse paths which were set to the default: {"path":[{"id":"Default"}]} policies: file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json} # eg for local file @@ -295,7 +295,6 @@ bootstrap: servlets: waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize - systemUpdate: initialBackOffMs: ${BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS:5000} maxBackOffs: ${BOOTSTRAP_SYSTEM_UPDATE_MAX_BACK_OFFS:50} @@ -371,4 +370,4 @@ cache: corpUserCredentials: 20 corpUserSettings: 20 -springdoc.api-docs.groups.enabled: true \ No newline at end of file +springdoc.api-docs.groups.enabled: true diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index f848a5e339..2e99def17c 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -1,5 +1,4 @@ apply plugin: 'java-library' -apply from: "../../gradle/versioning/versioning.gradle" dependencies { api project(':metadata-io') @@ -65,7 +64,3 @@ configurations.all{ exclude group: "commons-io", module:"commons-io" exclude group: "jline", module:"jline" } - -processResources.configure { - finalizedBy printVersionDetails // always print version details -} diff --git a/smoke-test/tests/read_only/test_services_up.py b/smoke-test/tests/read_only/test_services_up.py index e48df52bb9..cbe92625f4 100644 --- a/smoke-test/tests/read_only/test_services_up.py +++ b/smoke-test/tests/read_only/test_services_up.py @@ -23,3 +23,11 @@ def test_gms_config_accessible(): assert gms_config["versions"]["linkedin/datahub"]["version"] == DATAHUB_VERSION else: print("[WARN] TEST_DATAHUB_VERSION is not set") + + # Make sure that the default CLI version gets generated properly. + # While we don't want to hardcode the actual value, we can make + # sure it mostly looks like a version string. + default_cli_version: str = gms_config["managedIngestion"]["defaultCliVersion"] + print(f"Default CLI version: {default_cli_version}") + assert not default_cli_version.startswith("@") + assert "." in default_cli_version