From 9d649964fd717c721bc323d9f9f2fe082ac20ee8 Mon Sep 17 00:00:00 2001 From: Anuj359 Date: Wed, 13 Sep 2023 11:40:28 +0200 Subject: [PATCH] Creating make script for code freeze automation (#12976) * Disk space test * Disk space test with other action * Disk space test with other action * Disk space test with first action * checking labeler * updating version of wait action * xvfb-run check * Update disk-space-test.yml * xvfb-run check using sudo * checking large packages * Update disk-space-test.yml * Update disk-space-test.yml * Update disk-space-test.yml * Update disk-space-test.yml * Checking the logic * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update update_version.py * Update Makefile * Update update_version.py * Update Makefile * permission issue check * Update Makefile * Update Makefile * Update Makefile * Update Makefile * Update Makefile * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update release-code-freeze.yml * Update Makefile * Creating make script for code freeze automation * updating dockerfile version with main branch * Updating paths in makefile * Updating the python code with logger * Using argparse to pass the arguments in python code * feat:Updating script path * Delete update_version.py from root directory --- Makefile | 81 +++++++++++++- docker/docker-compose-quickstart/Dockerfile | 4 +- ingestion/Dockerfile | 6 +- ingestion/operators/docker/Dockerfile | 5 +- scripts/update_version.py | 118 ++++++++++++++++++++ 5 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 scripts/update_version.py diff --git a/Makefile b/Makefile index 0112b042e71..02f87a85835 100644 --- a/Makefile +++ b/Makefile @@ -246,4 +246,83 @@ generate-schema-docs: ## Generates markdown files for documenting the JSON Sche @echo "Generating Schema docs" python -m pip install "jsonschema2md" python scripts/generate_docs_schemas.py - \ No newline at end of file + +#Upgrade release automation scripts below +.PHONY: update_all +update_all: ## To update all the release related files run make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 + @echo "The release version is: $(RELEASE_VERSION)" ; \ + echo "The python metadata release version: $(PY_RELEASE_VERSION)" ; \ + $(MAKE) update_maven ; \ + $(MAKE) update_github_action_paths ; \ + $(MAKE) update_python_release_paths ; \ + $(MAKE) update_dockerfile_version ; \ + $(MAKE) update_ingestion_dockerfile_version ; \ + +#remove comment and use the below section when want to use this sub module "update_all" independently to update github actions +#make update_all RELEASE_VERSION=2.2.2 PY_RELEASE_VERSION=2.2.2.2 + +.PHONY: update_maven +update_maven: ## To update the common and pom.xml maven version + @echo "Updating Maven projects to version $(RELEASE_VERSION)..."; \ + mvn versions:set -DnewVersion=$(RELEASE_VERSION) +#remove comment and use the below section when want to use this sub module "update_maven" independently to update github actions +#make update_maven RELEASE_VERSION=2.2.2 + + +.PHONY: update_github_action_paths +update_github_action_paths: ## To update the github action ci docker files + @echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \ + file_paths="docker/docker-compose-quickstart/Dockerfile \ + .github/workflows/docker-openmetadata-db.yml \ + .github/workflows/docker-openmetadata-ingestion-base.yml \ + .github/workflows/docker-openmetadata-ingestion.yml \ + .github/workflows/docker-openmetadata-postgres.yml \ + .github/workflows/docker-openmetadata-server.yml"; \ + for file_path in $$file_paths; do \ + python3 scripts/update_version.py 1 $$file_path -s $(RELEASE_VERSION) ; \ + done; \ + file_paths1="docker/docker-compose-quickstart/Dockerfile"; \ + for file_path in $$file_paths1; do \ + python3 scripts/update_version.py 4 $$file_path -s $(RELEASE_VERSION) ; \ + done + +#remove comment and use the below section when want to use this sub module "update_github_action_paths" independently to update github actions +#make update_github_action_paths RELEASE_VERSION=2.2.2 + +.PHONY: update_python_release_paths +update_python_release_paths: ## To update the setup.py files + file_paths="ingestion/setup.py \ + openmetadata-airflow-apis/setup.py"; \ + echo "Updating Python setup file versions to $(PY_RELEASE_VERSION)... "; \ + for file_path in $$file_paths; do \ + python3 scripts/update_version.py 2 $$file_path -s $(PY_RELEASE_VERSION) ; \ + done +# Commented section for independent usage of the module update_python_release_paths independently to update github actions +#make update_python_release_paths PY_RELEASE_VERSION=2.2.2.2 + +.PHONY: update_dockerfile_version +update_dockerfile_version: ## To update the dockerfiles version + @file_paths="docker/docker-compose-ingestion/docker-compose-ingestion-postgres.yml \ + docker/docker-compose-ingestion/docker-compose-ingestion.yml \ + docker/docker-compose-openmetadata/docker-compose-openmetadata.yml \ + docker/docker-compose-quickstart/docker-compose-postgres.yml \ + docker/docker-compose-quickstart/docker-compose.yml"; \ + echo "Updating docker github action release version to $(RELEASE_VERSION)... "; \ + for file_path in $$file_paths; do \ + python3 scripts/update_version.py 3 $$file_path -s $(RELEASE_VERSION) ; \ + done +#remove comment and use the below section when want to use this sub module "update_dockerfile_version" independently to update github actions +#make update_dockerfile_version RELEASE_VERSION=2.2.2 + +.PHONY: update_ingestion_dockerfile_version +update_ingestion_dockerfile_version: ## To update the ingestion dockerfiles version + @file_paths="ingestion/Dockerfile \ + ingestion/operators/docker/Dockerfile"; \ + echo "Updating ingestion dockerfile release version to $(PY_RELEASE_VERSION)... "; \ + for file_path in $$file_paths; do \ + python3 scripts/update_version.py 4 $$file_path -s $(PY_RELEASE_VERSION) ; \ + done +#remove comment and use the below section when want to use this sub module "update_ingestion_dockerfile_version" independently to update github actions +#make update_ingestion_dockerfile_version PY_RELEASE_VERSION=2.2.2.2 + +#Upgrade release automation scripts above diff --git a/docker/docker-compose-quickstart/Dockerfile b/docker/docker-compose-quickstart/Dockerfile index e35fb3932fb..a3cb22aa84e 100644 --- a/docker/docker-compose-quickstart/Dockerfile +++ b/docker/docker-compose-quickstart/Dockerfile @@ -10,8 +10,10 @@ # limitations under the License. # Build stage +ARG RI_VERSION="1.2.0" FROM alpine:3.15 AS build -ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/1.2.0-release/openmetadata-1.2.0.tar.gz" +ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz" + RUN mkdir -p /opt/openmetadata && \ wget ${RELEASE_URL} && \ tar zxvf openmetadata-*.tar.gz -C /opt/openmetadata --strip-components 1 && \ diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index 379e2591bb6..f85eaa39007 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -80,10 +80,10 @@ ARG INGESTION_DEPENDENCY="all" ENV PIP_NO_CACHE_DIR=1 # Make pip silent ENV PIP_QUIET=1 - +ARG RI_VERSION="1.1.0.4" RUN pip install --upgrade pip -RUN pip install "openmetadata-managed-apis~=1.1.0.4" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.9.txt" -RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=1.1.0.4" +RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.9.txt" +RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 RUN echo "Image built for $(uname -m)" diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 1bdc6580702..001290f557c 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -86,9 +86,10 @@ ENV PIP_QUIET=1 RUN pip install --upgrade pip ARG INGESTION_DEPENDENCY="all" +ARG RI_VERSION="1.1.0.4" RUN pip install --upgrade pip -RUN pip install "openmetadata-ingestion[airflow]~=1.1.0.4" -RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=1.1.0.4" +RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}" +RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 RUN echo "Image built for $(uname -m)" diff --git a/scripts/update_version.py b/scripts/update_version.py new file mode 100644 index 00000000000..c8bf177bebe --- /dev/null +++ b/scripts/update_version.py @@ -0,0 +1,118 @@ +import sys +import argparse +import fileinput +import os +import re +import logging + +# Configure the logger +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger() + +# Function to update the Github workflow with search pattern as "input=" or "DOCKER_RELEASE_TAG=" +def update_github_action(file_path, release_version): + logger.info(f"Updating Github workflow's Docker version in {file_path} to version {release_version}\n") + try: + with open(file_path, 'r') as file: + content = file.read() + + # Update the input pattern + input_pattern = r'input=\d+(\.\d+)*(\.\d+)?' + input_replacement = f'input={release_version}' + updated_content = re.sub(input_pattern, input_replacement, content) + + # Update the DOCKER_RELEASE_TAG pattern + docker_release_tag_pattern = r'DOCKER_RELEASE_TAG=\d+(\.\d+)*(\.\d+)?' + docker_release_tag_replacement = f'DOCKER_RELEASE_TAG={release_version}' + updated_content = re.sub(docker_release_tag_pattern, docker_release_tag_replacement, updated_content) + + with open(file_path, 'w') as file: + file.write(updated_content) + + logger.info(f"Patterns updated to {release_version} in {file_path}") + except Exception as e: + logger.error(f"An error occurred: {e}") + +# Function to update the Python files in ingestion with search pattern as "version=" +def update_python_files(file_path, release_version): + # Logic for updating Python files + logger.info(f"Updating version numbers in {file_path} to {release_version}\n") + try: + with open(file_path, 'r') as file: + content = file.read() + + pattern = r'version\s*=\s*"([^"]+)"' + updated_content = re.sub(pattern, f'version="{release_version}"', content) + + with open(file_path, 'w') as file: + file.write(updated_content) + + logger.info(f"Version numbers updated to {release_version} in {file_path}") + except Exception as e: + logger.error(f"An error occurred: {e}") + +# Function to update the image version in Docker compose files with search pattern where image, docker, getcollate, and openmetadata are used. +def update_dockerfile_version(file_path, release_version): + # Logic for updating Docker compose version + try: + with open(file_path, 'r') as file: + content = file.read() + + # Update image versions using regular expression + updated_content = re.sub( + r'(image: docker\.getcollate\.io/openmetadata/.*?):[\d.]+', + rf'\1:{release_version}', + content + ) + + with open(file_path, 'w') as file: + file.write(updated_content) + + logger.info(f"Updated image versions in {file_path}") + except Exception as e: + logger.error(f"An error occurred while updating {file_path}: {e}") + +# Function to update the DOCKERFILE used to create the images, search pattern used as "RI_VERSION" +def update_ingestion_version(file_path, release_version): + logger.info(f"Updating ingestion version in {file_path} to version {release_version}\n") + try: + with open(file_path, 'r') as file: + content = file.read() + + pattern = r'RI_VERSION="[\d\.]+"' + replacement = f'RI_VERSION="{release_version}"' + updated_content = re.sub(pattern, replacement, content) + + with open(file_path, 'w') as file: + file.write(updated_content) + + logger.info(f"RI_VERSION updated to {release_version} in {file_path}") + except Exception as e: + logger.error(f"An error occurred: {e}") + +def main(): + parser = argparse.ArgumentParser(description="Update version information in files.") + parser.add_argument("action_type", type=int, choices=range(1, 5), help="Type of action to perform") + parser.add_argument("file_path", type=str, help="Path to the file to update") + parser.add_argument("-s", dest="release_version", required=True, help="Release version to set") + + args = parser.parse_args() + + action_type = args.action_type + file_path = args.file_path + release_version = args.release_version + + if action_type == 1: + update_github_action(file_path, release_version) + elif action_type == 2: + update_python_files(file_path, release_version) + elif action_type == 3: + update_dockerfile_version(file_path, release_version) + elif action_type == 4: + update_ingestion_version(file_path, release_version) + else: + logger.error("Invalid action type") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file