feat(ingest): remove hardcoded env variable default for cli version (#6075)

.github/workflows/build-and-test.yml

@@ -25,6 +25,8 @@ jobs:
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:

datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx

@@ -98,7 +98,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 Advanced: Provide a custom CLI version to use for ingestion.
                             </Typography.Paragraph>
                             <Input
-                                placeholder="0.8.42"
+                                placeholder="(e.g. 0.8.42)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
                             />

docker/datahub-gms/env/docker-without-neo4j.env

@@ -16,7 +16,6 @@ MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
 PE_CONSUMER_ENABLED=true
 UI_INGESTION_ENABLED=true
-UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42
 ENTITY_SERVICE_ENABLE_RETENTION=true
 
 # Uncomment to disable persistence of client-side analytics events

docker/datahub-gms/env/docker.env

@@ -20,7 +20,6 @@ MAE_CONSUMER_ENABLED=true
 MCE_CONSUMER_ENABLED=true
 PE_CONSUMER_ENABLED=true
 UI_INGESTION_ENABLED=true
-UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42
 
 # Uncomment to enable Metadata Service Authentication
 # METADATA_SERVICE_AUTH_ENABLED=true

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -82,7 +82,6 @@ services:
     - MCE_CONSUMER_ENABLED=true
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42
     - ENTITY_SERVICE_ENABLE_RETENTION=true
     hostname: datahub-gms
     image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}

docker/quickstart/docker-compose.quickstart.yml

@@ -89,7 +89,6 @@ services:
     - MCE_CONSUMER_ENABLED=true
     - PE_CONSUMER_ENABLED=true
     - UI_INGESTION_ENABLED=true
-    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.42
     hostname: datahub-gms
     image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head}
     ports:

gradle/versioning/versioning.gradle

@@ -0,0 +1,86 @@
+/**
+ Applies a consistent versioning scheme to all projects using this script
+
+Uses git tags to mint versions by default. 
+git tags can be of a few forms:
+- short sha (typical for a PR or a commit) (e.g. 38960ae)
+- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4)
+
+Produces the following variables and supports token replacement
+- version: server version amenable for creating jars
+- fullVersion: full version string 
+- cliMajorVersion: cli version amenable for binding to server as a default
+  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
+
+ All inference can be overridden by passing in the releaseVersion property
+ e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
+
+ **/
+
+
+import org.apache.tools.ant.filters.ReplaceTokens
+
+def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
+def cliMajorVersion = "0.8.42" // base default cli major version
+def snapshotVersion = false
+if (project.hasProperty("releaseVersion")) {
+  version = releaseVersion
+  detailedVersionString = releaseVersion
+} else {
+  try {
+      // apply this plugin in a try-catch block so that we can handle cases without .git directory
+      apply plugin: "com.palantir.git-version"
+      def details = versionDetails()
+      detailedVersionString = gitVersion()
+      version = details.lastTag
+      version = version.startsWith("v")? version.substring(1): version
+      def suffix = details.isCleanTag? "": "-SNAPSHOT"
+      snapshotVersion = ! details.isCleanTag
+    }
+    catch (Exception e) {
+          e.printStackTrace()
+          // last fall back
+          version = detailedVersionString
+    }
+}
+
+// trim version if it is of size 4 to size 3
+def versionParts = version.tokenize(".")
+if (versionParts.size() > 3) {
+  // at-least 4 part version
+  // we check if the 4th part is a .0 in which case we want to create a release
+  if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
+    versionParts = versionParts[0..2]
+  }
+  version = versionParts[0..2].join('.')
+  if (versionParts.size() > 3) {
+      version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
+  }
+  cliMajorVersion = version
+} else if (versionParts.size() == 3) {
+  cliMajorVersion = version
+}
+
+if (snapshotVersion) {
+    if (versionParts[versionParts.size()-1].isInteger()) {
+      def base_version = versionParts[0..versionParts.size()-2].join('.')
+      version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+      cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1]
+    } else {
+      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+      version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+      cliMajorVersion = versionParts[0..versionParts.size()-1].join('.')
+  }
+}
+
+
+processResources {
+  filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
+  filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion])
+}
+
+task printVersionDetails() {
+  println("fullVersion=" + detailedVersionString)
+  println("cliMajorVersion=" + cliMajorVersion)
+  println("version=" +  version)
+}

metadata-service/factories/build.gradle

@@ -1,4 +1,5 @@
 apply plugin: 'java'
+apply from: "../../gradle/versioning/versioning.gradle"
 
 dependencies {
   compile project(':metadata-io')
@@ -33,3 +34,7 @@ dependencies {
   testCompile externalDependency.testng
 
 }
+
+processResources.configure {
+  finalizedBy printVersionDetails // always print version details
+}

metadata-service/factories/src/main/resources/application.yml

@@ -44,7 +44,7 @@ authorization:
 
 ingestion:
   enabled: ${UI_INGESTION_ENABLED:true}
-  defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:0.8.42}'
+  defaultCliVersion: '${UI_INGESTION_DEFAULT_CLI_VERSION:@cliMajorVersion@}'
 
 telemetry:
   enabledCli: ${CLI_TELEMETRY_ENABLED:true}