aligned pre-commit with Makefile. added check json (#15295)

* aligned pre-commit with Makefile. added check json

* add check for json/yaml files

* add path checks for json/yaml validation

.github/workflows/validate-jsons-yamls.yml

@@ -0,0 +1,96 @@
+#  Copyright 2024 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+name: Validate JSON/YAML
+
+# read-write repo token
+# access to secrets
+on:
+  pull_request_target:
+    types: [labeled, opened, synchronize, reopened]
+    paths:
+      - "**.json"
+      - "**.yaml"
+      - "**.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  validate-json-yaml:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Wait for the labeler
+        uses: lewagon/wait-on-check-action@v1.3.3
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          check-name: Team Label
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 90
+
+      - name: Verify PR labels
+        uses: jesusvasquez333/verify-pr-label-action@v1.4.0
+        if: ${{ github.event_name == 'pull_request_target' }}
+        with:
+          github-token: '${{ secrets.GITHUB_TOKEN }}'
+          valid-labels: 'safe to test'
+          pull-request-number: '${{ github.event.pull_request.number }}'
+          disable-reviews: true  # To not auto approve changes
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install Ubuntu related dependencies
+        run: |
+          sudo apt-get update  && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
+
+      - name: Install Python & Openmetadata related dependencies
+        run: |
+          python3 -m venv env
+          source env/bin/activate
+          pip install pyyaml
+
+      # Add back linting once we have 10/10 on main
+      - name: Code style check
+        id: style
+        continue-on-error: true
+        run: |
+          source env/bin/activate
+          pip install pyyaml
+          ./scripts/validate_json_yaml.sh
+
+      - name: Create a comment in the PR with the instructions
+        if: steps.style.outcome != 'success'
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            **JSON/YAML Validations failed.**
+
+            Please run `./scripts/validate_json_yaml.sh` to find any bad json/yaml files and fix them.
+            You can also use [pre-commit](https://pre-commit.com/) to automate the Python code formatting.
+
+            You can install the pre-commit hooks with `make install_test precommit_install`.
+
+      - name: Python checkstyle failed, check the comment in the PR
+        if: steps.style.outcome != 'success'
+        run: |
+          exit 1

.pre-commit-config.yaml

@@ -1,22 +1,29 @@
 default_language_version:
   python: python3
 repos:
-  - repo: https://github.com/ambv/black
+  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 22.3.0
+    rev: v2.3.0
     hooks:
-      - id: black
+      - id: check-json
-        exclude: ingestion/src/metadata/generated
+        exclude: vscode
+  - repo: https://github.com/hadialqattan/pycln
+    rev: v2.4.0
+    hooks:
+      - id: pycln
+        files: ^(ingestion|openmetadata-airflow-apis)/
+        args: [ "--config", "ingestion/pyproject.toml" ]
   - repo: https://github.com/timothycrosley/isort
     rev: 5.12.0
     hooks:
       - id: isort
-        args: ["--profile", "black"]
+        files: ^(ingestion|openmetadata-airflow-apis)/
-        exclude: ingestion/src/metadata/generated
+        args: [ "--settings-file", "ingestion/pyproject.toml" ]
-  - repo: https://github.com/hadialqattan/pycln
+  - repo: https://github.com/ambv/black
-    rev: v1.2.4
+    rev: 22.3.0
     hooks:
-      - id: pycln
+      - id: black
-        exclude: ingestion/src/metadata/generated
+        files: ^(ingestion|openmetadata-airflow-apis)/
+        args: [ "--config", "ingestion/pyproject.toml" ]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.1
     hooks:

scripts/validate_json_yaml.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# bash strict mode
+set -eup pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+EXCLUDED_DIRS="vscode"
+echo "Validating JSON files..."
+git ls-files | grep "\.json$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do jq . "$file" >/dev/null 2>&1 || { echo "Invalid JSON in $file"; exit 1; }; done
+echo "Validating YAML files..."
+git ls-files | grep -E "\.ya?ml$" | grep -vE "^($EXCLUDED_DIRS)/" | while read file; do python ${SCRIPT_DIR}/validate_yaml.py "$file" >/dev/null 2>&1 || { echo "Invalid YAML in $file"; exit 1; }; done

scripts/validate_yaml.py

@@ -0,0 +1,9 @@
+import os
+import sys
+
+import yaml
+
+path = sys.argv[1]
+if os.path.islink(path):
+    exit()
+yaml.safe_load(open(path, "r"))