mirror of
https://github.com/deepset-ai/haystack.git
synced 2026-01-01 17:47:19 +00:00
60ff46e4e1
* track eval results in mlflow * Update Documentation & Code Style * add pipeline.yaml and environment info * improve logging to mlflow * Update Documentation & Code Style * introduce ExperimentTracker * Update Documentation & Code Style * move modeling.utils.logger to utils.experiment_tracking * renaming: tracker and TrackingHead * Update Documentation & Code Style * refactor env tracking * fix pylint findings * Update Documentation & Code Style * rename MLFlowTrackingHead to MLflowTrackingHead * implement dataset hash * Update Documentation & Code Style * set docstrings * Update Documentation & Code Style * introduce PipelineBundle and Corpus * Update Documentation & Code Style * support reusing index * Update Documentation & Code Style * rename Corpus to FileCorpus * fix Corpus -> FileCorpus * Update Documentation & Code Style * resolve cyclic dependencies * fix linter issues * Update Documentation & Code Style * remove helper classes * Update Documentation & Code Style * fix imports * fix another unused import * update docstrings * Update Documentation & Code Style * simplify usage of experiment tracking tools * fix Literal import * revert schema changes * Update Documentation & Code Style * always end run * Update Documentation & Code Style * fix mypy issue * rename to execute_eval_run * Update Documentation & Code Style * fix merge of get_or_create_env_meta_data * improve docstrings * Update Documentation & Code Style Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
63 lines
2.3 KiB
Python
63 lines
2.3 KiB
Python
import os
|
|
import platform
|
|
import sys
|
|
from typing import Any, Dict
|
|
import torch
|
|
import transformers
|
|
|
|
from haystack import __version__
|
|
|
|
|
|
HAYSTACK_EXECUTION_CONTEXT = "HAYSTACK_EXECUTION_CONTEXT"
|
|
HAYSTACK_DOCKER_CONTAINER = "HAYSTACK_DOCKER_CONTAINER"
|
|
|
|
|
|
env_meta_data: Dict[str, Any] = {}
|
|
|
|
|
|
def get_or_create_env_meta_data() -> Dict[str, Any]:
|
|
"""
|
|
Collects meta data about the setup that is used with Haystack, such as: operating system, python version, Haystack version, transformers version, pytorch version, number of GPUs, execution environment, and the value stored in the env variable HAYSTACK_EXECUTION_CONTEXT.
|
|
"""
|
|
global env_meta_data # pylint: disable=global-statement
|
|
if not env_meta_data:
|
|
env_meta_data = {
|
|
"os_version": platform.release(),
|
|
"os_family": platform.system(),
|
|
"os_machine": platform.machine(),
|
|
"python_version": platform.python_version(),
|
|
"haystack_version": __version__,
|
|
"transformers_version": transformers.__version__,
|
|
"torch_version": torch.__version__,
|
|
"torch_cuda_version": torch.version.cuda if torch.cuda.is_available() else 0,
|
|
"n_gpu": torch.cuda.device_count() if torch.cuda.is_available() else 0,
|
|
"n_cpu": os.cpu_count(),
|
|
"context": os.environ.get(HAYSTACK_EXECUTION_CONTEXT),
|
|
"execution_env": _get_execution_environment(),
|
|
}
|
|
return env_meta_data
|
|
|
|
|
|
def _get_execution_environment():
|
|
"""
|
|
Identifies the execution environment that Haystack is running in.
|
|
Options are: colab notebook, kubernetes, CPU/GPU docker container, test environment, jupyter notebook, python script
|
|
"""
|
|
if os.environ.get("CI", "False").lower() == "true":
|
|
execution_env = "ci"
|
|
elif "google.colab" in sys.modules:
|
|
execution_env = "colab"
|
|
elif "KUBERNETES_SERVICE_HOST" in os.environ:
|
|
execution_env = "kubernetes"
|
|
elif HAYSTACK_DOCKER_CONTAINER in os.environ:
|
|
execution_env = os.environ.get(HAYSTACK_DOCKER_CONTAINER)
|
|
# check if pytest is imported
|
|
elif "pytest" in sys.modules:
|
|
execution_env = "test"
|
|
else:
|
|
try:
|
|
execution_env = get_ipython().__class__.__name__ # pylint: disable=undefined-variable
|
|
except NameError:
|
|
execution_env = "script"
|
|
return execution_env
|