Creating make script for code freeze automation (#12976)

* Disk space test * Disk space test with other action * Disk space test with other action * Disk space test with first action * checking labeler * updating version of wait action * xvfb-run check * Update disk-space-test.yml * xvfb-run check using sudo * checking large packages * Update disk-space-test.yml * Update disk-space-test.yml * Update disk-space-test.yml * Update disk-space-test.yml * Checking the logic * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update update_version.py * Update Makefile * Update update_version.py * Update Makefile * permission issue check * Update Makefile * Update Makefile * Update Makefile * Update Makefile * Update Makefile * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update Makefile * Creating make script for code freeze automation * updating dockerfile version with main branch * Updating paths in makefile * Updating the python code with logger * Using argparse to pass the arguments in python code * feat:Updating script path * Delete update_version.py from root directory
2026-01-05 20:17:07 +00:00 · 2023-09-13 11:40:28 +02:00 · 2023-09-13 11:40:28 +02:00 · 9d649964fd
commit 9d649964fd
parent 4aab921d10
5 changed files with 207 additions and 7 deletions
--- a/81
+++ b/81
@ -246,4 +246,83 @@ generate-schema-docs:  ## Generates markdown files for documenting the JSON Sche
 	@echo "Generating Schema docs"
 	python -m pip install "jsonschema2md"
 	python scripts/generate_docs_schemas.py
-	
+
+#Upgrade release automation scripts below	
+.PHONY: update_all
+update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
+	@echo "The release version is: $(RELEASE_VERSION)" ; \
+	echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \
+	$(MAKE) update_maven ; \
+	$(MAKE) update_github_action_paths ; \
+	$(MAKE) update_python_release_paths ; \
+	$(MAKE) update_dockerfile_version ; \
+	$(MAKE) update_ingestion_dockerfile_version ; \
+
+#remove comment and use the below section when want to use this sub module "update_all" independently to update github actions
+#make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2
+
+.PHONY: update_maven
+update_maven: ## To update the common and pom.xml maven version
+	@echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \
+	mvn versions:set -DnewVersion=$(RELEASE_VERSION)
+#remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions
+#make update_maven RELEASE_VERSION=2.2.2
+
+
+.PHONY: update_github_action_paths
+update_github_action_paths: ## To update the github action ci docker files
+	@echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
+	file_paths="docker/docker-compose-quickstart/Dockerfile \
+	            .github/workflows/docker-openmetadata-db.yml \
+	            .github/workflows/docker-openmetadata-ingestion-base.yml \
+	            .github/workflows/docker-openmetadata-ingestion.yml \
+	            .github/workflows/docker-openmetadata-postgres.yml \
+	            .github/workflows/docker-openmetadata-server.yml"; \
+	for file_path in $$file_paths; do \
+	    python3 scripts/update_version.py 1 $$file_path -s $(RELEASE_VERSION) ; \
+	done; \
+	file_paths1="docker/docker-compose-quickstart/Dockerfile"; \
+	for file_path in $$file_paths1; do \
+	    python3 scripts/update_version.py 4 $$file_path -s $(RELEASE_VERSION) ; \
+	done
+
+#remove comment and use the below section when want to use this sub module "update_github_action_paths" independently to update github actions
+#make update_github_action_paths RELEASE_VERSION=2.2.2
+
+.PHONY: update_python_release_paths
+update_python_release_paths: ## To update the setup.py files
+	file_paths="ingestion/setup.py \
+				openmetadata-airflow-apis/setup.py"; \
+	echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \
+	for file_path in $$file_paths; do \
+	    python3 scripts/update_version.py 2 $$file_path -s $(PY_RELEASE_VERSION) ; \
+	done
+# Commented section for independent usage of the module update_python_release_paths independently to update github actions
+#make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2
+
+.PHONY: update_dockerfile_version
+update_dockerfile_version: ## To update the dockerfiles version
+	@file_paths="docker/docker-compose-ingestion/docker-compose-ingestion-postgres.yml \
+		     docker/docker-compose-ingestion/docker-compose-ingestion.yml \
+		     docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \
+		     docker/docker-compose-quickstart/docker-compose-postgres.yml \
+		     docker/docker-compose-quickstart/docker-compose.yml"; \
+	echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \
+	for file_path in $$file_paths; do \
+	    python3 scripts/update_version.py 3 $$file_path -s $(RELEASE_VERSION) ; \
+	done
+#remove comment and use the below section when want to use this sub module "update_dockerfile_version" independently to update github actions
+#make update_dockerfile_version RELEASE_VERSION=2.2.2
+
+.PHONY: update_ingestion_dockerfile_version
+update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version
+	@file_paths="ingestion/Dockerfile \
+	             ingestion/operators/docker/Dockerfile"; \
+	echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \
+	for file_path in $$file_paths; do \
+	    python3 scripts/update_version.py 4 $$file_path -s $(PY_RELEASE_VERSION) ; \
+	done
+#remove comment and use the below section when want to use this sub module "update_ingestion_dockerfile_version" independently to update github actions
+#make update_ingestion_dockerfile_version PY_RELEASE_VERSION=2.2.2.2
+
+#Upgrade release automation scripts above
--- a/docker/docker-compose-quickstart/Dockerfile
+++ b/docker/docker-compose-quickstart/Dockerfile
@ -10,8 +10,10 @@
 #  limitations under the License.

 # Build stage
+ARG RI_VERSION="1.2.0"
 FROM alpine:3.15 AS build
-ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/openmetadata-1.2.0.tar.gz"
+ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz"
+
 RUN mkdir -p /opt/openmetadata && \
    wget ${RELEASE_URL} && \
    tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \
--- a/ingestion/Dockerfile
+++ b/ingestion/Dockerfile
@ -80,10 +80,10 @@ ARG INGESTION_DEPENDENCY="all"
 ENV PIP_NO_CACHE_DIR=1
 # Make pip silent
 ENV PIP_QUIET=1
-
+ARG RI_VERSION="1.1.0.4"
 RUN pip install --upgrade pip
-RUN pip install "openmetadata-managed-apis~=1.1.0.4" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.9.txt"
-RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=1.1.0.4"
+RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.9.txt"
+RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"

 # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593
 RUN echo "Image built for $(uname -m)"
--- a/ingestion/operators/docker/Dockerfile
+++ b/ingestion/operators/docker/Dockerfile
@ -86,9 +86,10 @@ ENV PIP_QUIET=1
 RUN pip install --upgrade pip

 ARG INGESTION_DEPENDENCY="all"
+ARG RI_VERSION="1.1.0.4"
 RUN pip install --upgrade pip
-RUN pip install "openmetadata-ingestion[airflow]~=1.1.0.4"
-RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=1.1.0.4"
+RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}"
+RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"

 # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593
 RUN echo "Image built for $(uname -m)"
--- a/scripts/update_version.py
+++ b/scripts/update_version.py
@ -0,0 +1,118 @@
+import sys
+import argparse
+import fileinput
+import os
+import re
+import logging
+
+# Configure the logger
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger()
+
+# Function to update the Github workflow with search pattern as "input=" or "DOCKER_RELEASE_TAG="
+def update_github_action(file_path, release_version):
+    logger.info(f"Updating Github workflow's Docker version in {file_path} to version {release_version}\n")
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+
+        # Update the input pattern
+        input_pattern = r'input=\d+(\.\d+)*(\.\d+)?'
+        input_replacement = f'input={release_version}'
+        updated_content = re.sub(input_pattern, input_replacement, content)
+
+        # Update the DOCKER_RELEASE_TAG pattern
+        docker_release_tag_pattern = r'DOCKER_RELEASE_TAG=\d+(\.\d+)*(\.\d+)?'
+        docker_release_tag_replacement = f'DOCKER_RELEASE_TAG={release_version}'
+        updated_content = re.sub(docker_release_tag_pattern, docker_release_tag_replacement, updated_content)
+
+        with open(file_path, 'w') as file:
+            file.write(updated_content)
+
+        logger.info(f"Patterns updated to {release_version} in {file_path}")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+
+# Function to update the Python files in ingestion with search pattern as "version="
+def update_python_files(file_path, release_version):
+    # Logic for updating Python files
+    logger.info(f"Updating version numbers in {file_path} to {release_version}\n")
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+
+        pattern = r'version\s*=\s*"([^"]+)"'
+        updated_content = re.sub(pattern, f'version="{release_version}"', content)
+
+        with open(file_path, 'w') as file:
+            file.write(updated_content)
+
+        logger.info(f"Version numbers updated to {release_version} in {file_path}")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+
+# Function to update the image version in Docker compose files with search pattern where image, docker, getcollate, and openmetadata are used.
+def update_dockerfile_version(file_path, release_version):
+    # Logic for updating Docker compose version
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+
+        # Update image versions using regular expression
+        updated_content = re.sub(
+            r'(image: docker\.getcollate\.io/openmetadata/.*?):[\d.]+',
+            rf'\1:{release_version}',
+            content
+        )
+
+        with open(file_path, 'w') as file:
+            file.write(updated_content)
+
+        logger.info(f"Updated image versions in {file_path}")
+    except Exception as e:
+        logger.error(f"An error occurred while updating {file_path}: {e}")
+
+# Function to update the DOCKERFILE used to create the images, search pattern used as "RI_VERSION"
+def update_ingestion_version(file_path, release_version):
+    logger.info(f"Updating ingestion version in {file_path} to version {release_version}\n")
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+
+        pattern = r'RI_VERSION="[\d\.]+"'
+        replacement = f'RI_VERSION="{release_version}"'
+        updated_content = re.sub(pattern, replacement, content)
+
+        with open(file_path, 'w') as file:
+            file.write(updated_content)
+
+        logger.info(f"RI_VERSION updated to {release_version} in {file_path}")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Update version information in files.")
+    parser.add_argument("action_type", type=int, choices=range(1, 5), help="Type of action to perform")
+    parser.add_argument("file_path", type=str, help="Path to the file to update")
+    parser.add_argument("-s", dest="release_version", required=True, help="Release version to set")
+
+    args = parser.parse_args()
+
+    action_type = args.action_type
+    file_path = args.file_path
+    release_version = args.release_version
+
+    if action_type == 1:
+        update_github_action(file_path, release_version)
+    elif action_type == 2:
+        update_python_files(file_path, release_version)
+    elif action_type == 3:
+        update_dockerfile_version(file_path, release_version)
+    elif action_type == 4:
+        update_ingestion_version(file_path, release_version)
+    else:
+        logger.error("Invalid action type")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()