From 14e475cefeed074e5df7e2fbeecf2c5f5010512c Mon Sep 17 00:00:00 2001 From: IceS2 Date: Thu, 18 Jul 2024 11:52:56 +0200 Subject: [PATCH] MINOR: Add PyRight TypeCheck to our Python Project (#17060) * Add PyRight TypeCheck to our Python Project * Change pyright for basedpyright * Fix PyRight --- ingestion/Makefile | 5 +++++ ingestion/pyproject.toml | 35 +++++++++++++++++++++++++++++++++++ ingestion/setup.py | 1 + ingestion/src/metadata/cmd.py | 28 ++++++++++++++++------------ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/ingestion/Makefile b/ingestion/Makefile index 719784b9534..d83745ae1c5 100644 --- a/ingestion/Makefile +++ b/ingestion/Makefile @@ -33,6 +33,10 @@ install_apis: ## Install the REST APIs module to the current environment lint: ## Run pylint on the Python sources to analyze the codebase PYTHONPATH="${PYTHONPATH}:$(INGESTION_DIR)/plugins" find $(PY_SOURCE) -path $(PY_SOURCE)/metadata/generated -prune -false -o -type f -name "*.py" | xargs pylint --rcfile=$(INGESTION_DIR)/pyproject.toml +.PHONY: static-checks +static-checks: + basedpyright -p $(INGESTION_DIR)/pyproject.toml + .PHONY: precommit_install precommit_install: ## Install the project's precommit hooks from .pre-commit-config.yaml @echo "Installing pre-commit hooks" @@ -64,6 +68,7 @@ run_ometa_integration_tests: ## Run Python integration tests .PHONY: run_python_tests run_python_tests: ## Run all Python tests with coverage coverage erase + $(MAKE) static-checks $(MAKE) unit_ingestion $(MAKE) run_ometa_integration_tests coverage report --rcfile $(INGESTION_DIR)/pyproject.toml || true diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index 046e5ba3f8d..a7e3273384a 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -141,3 +141,38 @@ skip_glob = [ profile = "black" indent = " " multi_line_output = 3 + +[tool.basedpyright] +include = ["src"] +exclude = [ + "**/__pycache__", + "src/metadata/generated/*", + "src/metadata/__version__.py", +] + +# TODO: Remove the ignored paths little by little. +ignore = [ + "src/_openmetadata_testutils/*", + "src/airflow_provider_openmetadata/*", + "src/metadata/antlr/*", + "src/metadata/automations/*", + "src/metadata/cli/*", + "src/metadata/clients/*", + "src/metadata/config/*", + "src/metadata/data_insight/*", + "src/metadata/data_quality/*", + "src/metadata/examples/*", + "src/metadata/great_expectations/*", + "src/metadata/ingestion/*", + "src/metadata/mixins/*", + "src/metadata/parsers/*", + "src/metadata/pii/*", + "src/metadata/profiler/*", + "src/metadata/readers/*", + "src/metadata/timer/*", + "src/metadata/utils/*", + "src/metadata/workflow/*", +] + +reportDeprecated = false +reportMissingTypeStubs = false diff --git a/ingestion/setup.py b/ingestion/setup.py index 37601b4d9f4..b26edb00865 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -341,6 +341,7 @@ test = { # Install GE because it's not in the `all` plugin VERSIONS["great-expectations"], "moto~=5.0", + "basedpyright~=1.14", "pytest==7.0.0", "pytest-cov", "pytest-order", diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index 3ecd18c3fab..19a9c45709b 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -17,6 +17,9 @@ from enum import Enum from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path +# pyright: reportUnusedCallResult=false +from typing import List, Optional + from metadata.__version__ import get_metadata_version from metadata.cli.app import run_app from metadata.cli.dataquality import run_test @@ -88,7 +91,7 @@ def add_metadata_args(parser: argparse.ArgumentParser): ) -def get_parser(args=None): +def get_parser(args: Optional[List[str]] = None): """ Parser method that returns parsed_args """ @@ -141,24 +144,23 @@ def get_parser(args=None): return parser.parse_args(args) -def metadata(args=None): +def metadata(args: Optional[List[str]] = None): """ This method implements parsing of the arguments passed from CLI """ contains_args = vars(get_parser(args)) metadata_workflow = contains_args.get("command") - config_file = contains_args.get("config") + config_file: Optional[Path] = contains_args.get("config") path = None if config_file: - path = Path(config_file).expanduser() + path = config_file.expanduser() if contains_args.get("debug"): set_loggers_level(logging.DEBUG) - elif contains_args.get("log_level"): - set_loggers_level(contains_args.get("log_level")) else: - set_loggers_level(logging.INFO) + log_level: str = contains_args.get("log_level", logging.INFO) + set_loggers_level(log_level) - if metadata_workflow in RUN_PATH_METHODS: + if path and metadata_workflow and metadata_workflow in RUN_PATH_METHODS: RUN_PATH_METHODS[metadata_workflow](path) if metadata_workflow == MetadataCommands.WEBHOOK.value: @@ -171,17 +173,19 @@ def metadata(args=None): self.wfile.write(bytes("Hello, World! Here is a GET response", "utf8")) def do_POST(self): # pylint: disable=invalid-name - content_len = int(self.headers.get("Content-Length")) - post_body = self.rfile.read(content_len) + if self.headers.get("Content-Length"): + content_len = int(self.headers["Content-Length"]) + post_body = self.rfile.read(content_len) + logger.info(post_body) + self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() - logger.info(post_body) logger.info( f"Starting server at {contains_args.get('host')}:{contains_args.get('port')}" ) with HTTPServer( - (contains_args.get("host"), contains_args.get("port")), WebhookHandler + (contains_args["host"], contains_args["port"]), WebhookHandler ) as server: server.serve_forever()