From 48b46971b8f40fd68c83d0afe1a6ca5f005dfa7d Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sun, 2 Oct 2022 16:46:34 -0700 Subject: [PATCH] feat(ingest): remove hardcoded env variable default for cli version (#6075) --- .github/workflows/build-and-test.yml | 2 + .../ingest/source/builder/NameSourceStep.tsx | 2 +- .../datahub-gms/env/docker-without-neo4j.env | 3 +- docker/datahub-gms/env/docker.env | 1 - ...ocker-compose-without-neo4j.quickstart.yml | 1 - .../quickstart/docker-compose.quickstart.yml | 1 - gradle/versioning/versioning.gradle | 86 +++++++++++++++++++ metadata-service/factories/build.gradle | 5 ++ .../src/main/resources/application.yml | 2 +- 9 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 gradle/versioning/versioning.gradle diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 5f96dceb5b..3a4b889a60 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,6 +25,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up JDK 11 uses: actions/setup-java@v1 with: diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx index 138136c646..d7c1a2e4bd 100644 --- a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx @@ -98,7 +98,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps) Advanced: Provide a custom CLI version to use for ingestion. setVersion(event.target.value)} /> diff --git a/docker/datahub-gms/env/docker-without-neo4j.env b/docker/datahub-gms/env/docker-without-neo4j.env index 45b8e8c5ed..6a085266d5 100644 --- a/docker/datahub-gms/env/docker-without-neo4j.env +++ b/docker/datahub-gms/env/docker-without-neo4j.env @@ -16,7 +16,6 @@ MAE_CONSUMER_ENABLED=true MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true -UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 ENTITY_SERVICE_ENABLE_RETENTION=true # Uncomment to disable persistence of client-side analytics events @@ -46,4 +45,4 @@ ENTITY_SERVICE_ENABLE_RETENTION=true # Uncomment to run a one-time upgrade to migrate legacy default browse path format to latest format # More details can be found at https://datahubproject.io/docs/advanced/browse-paths-upgrade -# UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED=true \ No newline at end of file +# UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED=true diff --git a/docker/datahub-gms/env/docker.env b/docker/datahub-gms/env/docker.env index 1b859aa59b..7c0297a3cd 100644 --- a/docker/datahub-gms/env/docker.env +++ b/docker/datahub-gms/env/docker.env @@ -20,7 +20,6 @@ MAE_CONSUMER_ENABLED=true MCE_CONSUMER_ENABLED=true PE_CONSUMER_ENABLED=true UI_INGESTION_ENABLED=true -UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 # Uncomment to enable Metadata Service Authentication # METADATA_SERVICE_AUTH_ENABLED=true diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index 05ca657678..b007b59d52 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -82,7 +82,6 @@ services: - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - UI_INGESTION_ENABLED=true - - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 - ENTITY_SERVICE_ENABLE_RETENTION=true hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index dc35e0c935..a7018a176f 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -89,7 +89,6 @@ services: - MCE_CONSUMER_ENABLED=true - PE_CONSUMER_ENABLED=true - UI_INGESTION_ENABLED=true - - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42 hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: diff --git a/gradle/versioning/versioning.gradle b/gradle/versioning/versioning.gradle new file mode 100644 index 0000000000..50fd3e844b --- /dev/null +++ b/gradle/versioning/versioning.gradle @@ -0,0 +1,86 @@ +/** + Applies a consistent versioning scheme to all projects using this script + +Uses git tags to mint versions by default. +git tags can be of a few forms: +- short sha (typical for a PR or a commit) (e.g. 38960ae) +- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4) + +Produces the following variables and supports token replacement +- version: server version amenable for creating jars +- fullVersion: full version string +- cliMajorVersion: cli version amenable for binding to server as a default + 0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories) + + All inference can be overridden by passing in the releaseVersion property + e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4 + + **/ + + +import org.apache.tools.ant.filters.ReplaceTokens + +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def cliMajorVersion = "0.8.42" // base default cli major version +def snapshotVersion = false +if (project.hasProperty("releaseVersion")) { + version = releaseVersion + detailedVersionString = releaseVersion +} else { + try { + // apply this plugin in a try-catch block so that we can handle cases without .git directory + apply plugin: "com.palantir.git-version" + def details = versionDetails() + detailedVersionString = gitVersion() + version = details.lastTag + version = version.startsWith("v")? version.substring(1): version + def suffix = details.isCleanTag? "": "-SNAPSHOT" + snapshotVersion = ! details.isCleanTag + } + catch (Exception e) { + e.printStackTrace() + // last fall back + version = detailedVersionString + } +} + +// trim version if it is of size 4 to size 3 +def versionParts = version.tokenize(".") +if (versionParts.size() > 3) { + // at-least 4 part version + // we check if the 4th part is a .0 in which case we want to create a release + if ((versionParts.size() == 4) && (versionParts[3] == '0')) { + versionParts = versionParts[0..2] + } + version = versionParts[0..2].join('.') + if (versionParts.size() > 3) { + version = version + "-" + versionParts[3..versionParts.size()-1].join('-') + } + cliMajorVersion = version +} else if (versionParts.size() == 3) { + cliMajorVersion = version +} + +if (snapshotVersion) { + if (versionParts[versionParts.size()-1].isInteger()) { + def base_version = versionParts[0..versionParts.size()-2].join('.') + version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1] + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + cliMajorVersion = versionParts[0..versionParts.size()-1].join('.') + } +} + + +processResources { + filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) + filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion]) +} + +task printVersionDetails() { + println("fullVersion=" + detailedVersionString) + println("cliMajorVersion=" + cliMajorVersion) + println("version=" + version) +} diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 453e8727c4..6ad9fc6b89 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'java' +apply from: "../../gradle/versioning/versioning.gradle" dependencies { compile project(':metadata-io') @@ -33,3 +34,7 @@ dependencies { testCompile externalDependency.testng } + +processResources.configure { + finalizedBy printVersionDetails // always print version details +} diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index e136a18602..a54a16d59d 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -44,7 +44,7 @@ authorization: ingestion: enabled: ${UI_INGESTION_ENABLED:true} - defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:0.8.42}' + defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}' telemetry: enabledCli: ${CLI_TELEMETRY_ENABLED:true}