yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 03:39:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): Fix for module level variable caching in sqllite check (#14861)

Browse Source
This commit is contained in:
Tamas Nemeth 2025-09-26 14:24:51 +02:00 committed by GitHub
parent 4bf3f0e66d
commit 7e9c525448
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
metadata-ingestion/src/datahub/utilities/file_backed_collections.py
View File

@ -33,13 +33,12 @@ from datahub.utilities.sentinels import Unset, unset
logger: logging.Logger = logging.getLogger(__name__)
OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR = (
os.environ.get("OVERRIDE_SQLITE_VERSION_REQ") or ""
)
OVERRIDE_SQLITE_VERSION_REQUIREMENT = (
OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR
and OVERRIDE_SQLITE_VERSION_REQUIREMENT_STR.lower() != "false"
)
def _get_sqlite_version_override() -> bool:
"""Check if SQLite version requirement should be overridden at runtime."""
override_str = os.environ.get("OVERRIDE_SQLITE_VERSION_REQ") or ""
return bool(override_str and override_str.lower() != "false")
_DEFAULT_FILE_NAME = "sqlite.db"
_DEFAULT_TABLE_NAME = "data"
@ -231,7 +230,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
# We use the ON CONFLICT clause to implement UPSERTs with sqlite.
# This was added in 3.24.0 from 2018-06-04.
# See https://www.sqlite.org/lang_conflict.html
if OVERRIDE_SQLITE_VERSION_REQUIREMENT:
if _get_sqlite_version_override():
self._use_sqlite_on_conflict = False
else:
raise RuntimeError("SQLite version 3.24.0 or later is required")

6
metadata-ingestion/tests/unit/utilities/test_file_backed_collections.py
View File

@ -1,5 +1,6 @@
import dataclasses
import json
import os
import pathlib
import random
import sqlite3
@ -35,10 +36,7 @@ def test_set_use_sqlite_on_conflict():
with (
patch("sqlite3.sqlite_version_info", (3, 23, 1)),
patch(
"datahub.utilities.file_backed_collections.OVERRIDE_SQLITE_VERSION_REQUIREMENT",
True,
),
patch.dict(os.environ, {"OVERRIDE_SQLITE_VERSION_REQ": "true"}),
):
cache = FileBackedDict[int](
tablename="cache",