chore(gx-plugin): require pydantic v2 (#14059)

.github/workflows/gx-plugin.yml

@@ -32,13 +32,9 @@ jobs:
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.11"]
         include:
           - python-version: "3.9"
-            extraPythonRequirement: "great-expectations~=0.15.12"
-          - python-version: "3.10"
-            extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0"
-          - python-version: "3.11"
             extraPythonRequirement: "great-expectations~=0.17.0"
           - python-version: "3.11"
             extraPythonRequirement: "great-expectations~=0.18.0"

docs/how/updating-datahub.md

@@ -41,6 +41,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 - #14015: In the sql-queries source, the `default_dialect` configuration parameter has been renamed to `override_dialect`. This also affects the Python SDK methods:
   - `DataHubGraph.parse_sql_lineage(default_dialect=...)` → `DataHubGraph.parse_sql_lineage(override_dialect=...)`
   - `LineageClient.add_lineage_via_sql(default_dialect=...)` → `LineageClient.add_lineage_via_sql(override_dialect=...)`
+- #14059: The `acryl-datahub-gx-plugin` now requires pydantic v2, which means the effective minimum supported version of GX is 0.17.15 (from Sept 2023).
 
 ### Known Issues
 

metadata-ingestion-modules/gx-plugin/setup.py

@@ -27,12 +27,14 @@ base_requirements = {
     # This is temporary lower bound that we're open to loosening/tightening as requirements show up
     "sqlalchemy>=1.4.39, <2",
     # GE added handling for higher version of jinja2 in version 0.15.12
-    # https://github.com/great-expectations/great_expectations/pull/5382/files
-    # TODO: support GX 0.18.0
-    "great-expectations>=0.15.12, <1.0.0",
+    # https://github.com/great-expectations/great_expectations/pull/5382
+    # GX v0.17.15 is the earliest version that supports Pydantic v2.
+    # See https://github.com/great-expectations/great_expectations/pull/8604
+    "great-expectations>=0.17.15, <1.0.0",
+    "pydantic>=2.1.0",
     # datahub does not depend on traitlets directly but great expectations does.
     # https://github.com/ipython/traitlets/issues/741
-    "traitlets<5.2.2",
+    "traitlets!=5.2.2",
     *rest_common,
     f"acryl-datahub[datahub-rest,sql-parser]{_self_pin}",
 }
@@ -61,9 +63,6 @@ base_dev_requirements = {
     "coverage>=5.1",
     "ruff==0.11.7",
     "mypy==1.14.1",
-    # pydantic 1.8.2 is incompatible with mypy 0.910.
-    # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
-    "pydantic>=1.10.0,!=1.10.3",
     "pytest>=6.2.2",
     "pytest-asyncio>=0.16.0",
     "pytest-cov>=2.8.1",