feat(ingest): headers for codegen Python scripts (#2637)

metadata-ingestion/scripts/avro_codegen.py

@@ -1,18 +1,31 @@
 import json
+from pathlib import Path
+from typing import Union
 
 import click
 from avrogen import write_schema_files
 
+autogen_header = """# flake8: noqa
 
-def suppress_checks_in_file(filepath: str) -> None:
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
-    """Adds a couple lines to the top of a file to suppress flake8 and black"""
+# Do not modify manually!
+
+# fmt: off
+"""
+
+
+def suppress_checks_in_file(filepath: Union[str, Path]) -> None:
+    """
+    Adds a couple lines to the top of an autogenerated file:
+        - Comments to suppress flake8 and black.
+        - A note stating that the file was autogenerated.
+    """
 
     with open(filepath, "r+") as f:
         contents = f.read()
 
         f.seek(0, 0)
-        f.write("# flake8: noqa\n")
+        f.write(autogen_header)
-        f.write("# fmt: off\n")
         f.write(contents)
         f.write("# fmt: on\n")
 
@@ -21,7 +34,6 @@ def suppress_checks_in_file(filepath: str) -> None:
 @click.argument("schema_file", type=click.Path(exists=True))
 @click.argument("outdir", type=click.Path())
 def generate(schema_file: str, outdir: str) -> None:
-    # print(f'using {schema_file}')
     with open(schema_file) as f:
         raw_schema_text = f.read()
 
@@ -33,11 +45,15 @@ def generate(schema_file: str, outdir: str) -> None:
     redo_spaces = json.dumps(json.loads(schema_json), indent=2)
 
     write_schema_files(redo_spaces, outdir)
-    suppress_checks_in_file(f"{outdir}/schema_classes.py")
     with open(f"{outdir}/__init__.py", "w"):
         # Truncate this file.
         pass
 
+    # Add headers for all generated files
+    generated_files = Path(outdir).glob("**/*.py")
+    for file in generated_files:
+        suppress_checks_in_file(file)
+
 
 if __name__ == "__main__":
     generate()

metadata-ingestion/src/datahub/metadata/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/events/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ....schema_classes import KafkaAuditHeaderClass
 
 
 KafkaAuditHeader = KafkaAuditHeaderClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/chart/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import ChartInfoClass
 from .....schema_classes import ChartQueryClass
 from .....schema_classes import ChartQueryTypeClass
@@ -8,3 +14,4 @@ ChartInfo = ChartInfoClass
 ChartQuery = ChartQueryClass
 ChartQueryType = ChartQueryTypeClass
 ChartType = ChartTypeClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import AccessLevelClass
 from .....schema_classes import AuditStampClass
 from .....schema_classes import BrowsePathsClass
@@ -48,3 +54,4 @@ OwnershipType = OwnershipTypeClass
 Status = StatusClass
 TagAssociation = TagAssociationClass
 VersionTag = VersionTagClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/common/fieldtransformer/__init__.py

@@ -1,6 +1,13 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ......schema_classes import TransformationTypeClass
 from ......schema_classes import UDFTransformerClass
 
 
 TransformationType = TransformationTypeClass
 UDFTransformer = UDFTransformerClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/dashboard/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import DashboardInfoClass
 
 
 DashboardInfo = DashboardInfoClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/datajob/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import DataFlowInfoClass
 from .....schema_classes import DataJobInfoClass
 from .....schema_classes import DataJobInputOutputClass
@@ -6,3 +12,4 @@ from .....schema_classes import DataJobInputOutputClass
 DataFlowInfo = DataFlowInfoClass
 DataJobInfo = DataJobInfoClass
 DataJobInputOutput = DataJobInputOutputClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/datajob/azkaban/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ......schema_classes import AzkabanJobTypeClass
 
 
 AzkabanJobType = AzkabanJobTypeClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/dataplatform/__init__.py

@@ -1,6 +1,13 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import DataPlatformInfoClass
 from .....schema_classes import PlatformTypeClass
 
 
 DataPlatformInfo = DataPlatformInfoClass
 PlatformType = PlatformTypeClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/dataprocess/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import DataProcessInfoClass
 
 
 DataProcessInfo = DataProcessInfoClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/dataset/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import DatasetDeprecationClass
 from .....schema_classes import DatasetFieldMappingClass
 from .....schema_classes import DatasetLineageTypeClass
@@ -14,3 +20,4 @@ DatasetProperties = DatasetPropertiesClass
 DatasetUpstreamLineage = DatasetUpstreamLineageClass
 Upstream = UpstreamClass
 UpstreamLineage = UpstreamLineageClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/glossary/__init__.py

@@ -1,6 +1,13 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import GlossaryNodeInfoClass
 from .....schema_classes import GlossaryTermInfoClass
 
 
 GlossaryNodeInfo = GlossaryNodeInfoClass
 GlossaryTermInfo = GlossaryTermInfoClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import CorpGroupInfoClass
 from .....schema_classes import CorpUserEditableInfoClass
 from .....schema_classes import CorpUserInfoClass
@@ -6,3 +12,4 @@ from .....schema_classes import CorpUserInfoClass
 CorpGroupInfo = CorpGroupInfoClass
 CorpUserEditableInfo = CorpUserEditableInfoClass
 CorpUserInfo = CorpUserInfoClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/metadata/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ......schema_classes import ChartKeyClass
 from ......schema_classes import CorpGroupKeyClass
 from ......schema_classes import CorpUserKeyClass
@@ -32,3 +38,4 @@ MLFeatureTableKey = MLFeatureTableKeyClass
 MLModelKey = MLModelKeyClass
 MLPrimaryKeyKey = MLPrimaryKeyKeyClass
 TagKey = TagKeyClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/metadata/snapshot/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ......schema_classes import ChartSnapshotClass
 from ......schema_classes import CorpGroupSnapshotClass
 from ......schema_classes import CorpUserSnapshotClass
@@ -32,3 +38,4 @@ MLFeatureTableSnapshot = MLFeatureTableSnapshotClass
 MLModelSnapshot = MLModelSnapshotClass
 MLPrimaryKeySnapshot = MLPrimaryKeySnapshotClass
 TagSnapshot = TagSnapshotClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/ml/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from ......schema_classes import BaseDataClass
 from ......schema_classes import CaveatDetailsClass
 from ......schema_classes import CaveatsAndRecommendationsClass
@@ -38,3 +44,4 @@ SourceCode = SourceCodeClass
 SourceCodeUrl = SourceCodeUrlClass
 SourceCodeUrlType = SourceCodeUrlTypeClass
 TrainingData = TrainingDataClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/mxe/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import MetadataChangeEventClass
 
 
 MetadataChangeEvent = MetadataChangeEventClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/schema/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import ArrayTypeClass
 from .....schema_classes import BinaryJsonSchemaClass
 from .....schema_classes import BooleanTypeClass
@@ -62,3 +68,4 @@ StringType = StringTypeClass
 TimeType = TimeTypeClass
 UnionType = UnionTypeClass
 UrnForeignKey = UrnForeignKeyClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/com/linkedin/pegasus2avro/tag/__init__.py

@@ -1,4 +1,11 @@
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# fmt: off
 from .....schema_classes import TagPropertiesClass
 
 
 TagProperties = TagPropertiesClass
+# fmt: on

metadata-ingestion/src/datahub/metadata/schema_classes.py

@@ -1,4 +1,8 @@
 # flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
 # fmt: off
 import json
 import os.path