From a3490093df5d1392b92a6329d34d6630007ca72e Mon Sep 17 00:00:00 2001 From: Nahuel Date: Thu, 2 Feb 2023 17:10:35 +0100 Subject: [PATCH] Generate PDF for Snyk security report (#10086) * Generate PDF for Snyk security report * Add missing doc * Minor change --- .github/workflows/security-scan.yml | 30 ++++++++++++++++-- Makefile | 9 ++++-- scripts/datamodel_generation.py | 17 +++++++++++ scripts/html_to_pdf.py | 47 +++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 scripts/html_to_pdf.py diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 216a906bfd1..233b55f6c9c 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -1,4 +1,3 @@ - # Copyright 2021 Collate # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +12,7 @@ name: security-scan on: schedule: - - cron: '0 0 * * 1' + - cron: '0 0 */2 * *' workflow_dispatch: jobs: @@ -41,7 +40,7 @@ jobs: # stop relying on apt cache of GitHub runners sudo apt-get update sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \ - librdkafka-dev unixodbc-dev libevent-dev + librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf # Install and Authenticate to Snyk - name: Install Snyk & Authenticate @@ -60,10 +59,35 @@ jobs: run: mvn -DskipTests clean install - name: Run Scan + id: security-report run: | source env/bin/activate make snyk-report + - name: Slack on Failure + if: steps.security-report.outcome != 'success' + uses: slackapi/slack-github-action@v1.23.0 + with: + payload: | + { + "text": "🚨 Security report failed, please check it . 🚨" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + + - name: Slack on Success + if: steps.security-report.outcome == 'success' + uses: slackapi/slack-github-action@v1.23.0 + with: + payload: | + { + "text": "🟢 Security report generated, please check it ." + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + - name: Upload Snyk Report HTML files uses: actions/upload-artifact@v3 with: diff --git a/Makefile b/Makefile index 84d55eddee7..e92f9c2ecbc 100644 --- a/Makefile +++ b/Makefile @@ -251,13 +251,16 @@ snyk-report: ## Uses Snyk CLI to run a security scan of the different pieces of $(MAKE) snyk-airflow-apis-report $(MAKE) snyk-server-report $(MAKE) snyk-ui-report - $(MAKE) export-snyk-html-report + $(MAKE) export-snyk-pdf-report -.PHONY: export-snyk-html-report -export-snyk-html-report: ## export json file from security-report/ to HTML +.PHONY: export-snyk-pdf-report +export-snyk-pdf-report: ## export json file from security-report/ to HTML @echo "Reading all results" npm install snyk-to-html -g ls security-report | xargs -I % snyk-to-html -i security-report/% -o security-report/%.html + pip install pdfkit + pip install PyPDF2 + python scripts/html_to_pdf.py # Ingestion Operators .PHONY: build-ingestion-base-local diff --git a/scripts/datamodel_generation.py b/scripts/datamodel_generation.py index 08e67a4bf3e..41c29652da5 100644 --- a/scripts/datamodel_generation.py +++ b/scripts/datamodel_generation.py @@ -1,3 +1,20 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script generates the Python models from the JSON Schemas definition. Additionally, it replaces the `SecretStr` +pydantic class used for the password fields with the `CustomSecretStr` pydantic class which retrieves the secrets +from a configured secrets' manager. +""" + import datamodel_code_generator.model.pydantic from datamodel_code_generator.imports import Import diff --git a/scripts/html_to_pdf.py b/scripts/html_to_pdf.py new file mode 100644 index 00000000000..eba9913a175 --- /dev/null +++ b/scripts/html_to_pdf.py @@ -0,0 +1,47 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script generates a PDF from all HTML files contained in a INPUT_FOLDER into a OUTPUT_FOLDER with a PDF_FILE_NAME. +It removes all the html files during the generation of the PDF file. +""" +import glob +import os + +import pdfkit +from PyPDF2 import PdfMerger + +INPUT_FOLDER = "security-report" + +OUTPUT_FOLDER = "security-report" + +PDF_FILE_NAME = "security-report" + +merger = PdfMerger() + +for file in glob.glob(f"{INPUT_FOLDER}/*.html"): + file_name, _ = os.path.splitext(file) + pdf_file = f"{file_name}.pdf" + print(f"Generating PDF file '{pdf_file}'") + pdfkit.from_file(file, pdf_file) + merger.append(pdf_file) + try: + print(f"Removing file '{file}'") + os.remove(file) + print(f"Removing file '{file_name}'") + os.remove(file_name) + except OSError as err: + pass + +print("Generating PDF report...") +merger.write(f"{OUTPUT_FOLDER}/{PDF_FILE_NAME}.pdf") +merger.close() +print("Process done!")