Evidently
zenml.integrations.evidently
special
Initialization of the Evidently integration.
The Evidently integration provides a way to monitor your models in production. It includes a way to detect data drift and different kinds of model performance issues.
The results of Evidently calculations can either be exported as an interactive dashboard (visualized as an html file or in your Jupyter notebook), or as a JSON file.
EvidentlyIntegration (Integration)
Evidently integration for ZenML.
Source code in zenml/integrations/evidently/__init__.py
class EvidentlyIntegration(Integration):
"""[Evidently](https://github.com/evidentlyai/evidently) integration for ZenML."""
NAME = EVIDENTLY
REQUIREMENTS = ["evidently==0.2.2"]
@staticmethod
def activate() -> None:
"""Activate the Deepchecks integration."""
from zenml.integrations.evidently import materializers # noqa
from zenml.integrations.evidently import visualizers # noqa
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Great Expectations integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.evidently.flavors import (
EvidentlyDataValidatorFlavor,
)
return [EvidentlyDataValidatorFlavor]
activate()
staticmethod
Activate the Deepchecks integration.
Source code in zenml/integrations/evidently/__init__.py
@staticmethod
def activate() -> None:
"""Activate the Deepchecks integration."""
from zenml.integrations.evidently import materializers # noqa
from zenml.integrations.evidently import visualizers # noqa
flavors()
classmethod
Declare the stack component flavors for the Great Expectations integration.
Returns:
| Type | Description |
|---|---|
List[Type[zenml.stack.flavor.Flavor]] |
List of stack component flavors for this integration. |
Source code in zenml/integrations/evidently/__init__.py
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Great Expectations integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.evidently.flavors import (
EvidentlyDataValidatorFlavor,
)
return [EvidentlyDataValidatorFlavor]
column_mapping
ZenML representation of an Evidently column mapping.
EvidentlyColumnMapping (BaseModel)
pydantic-model
Column mapping configuration for Evidently.
This class is a 1-to-1 serializable analogue of Evidently's ColumnMapping data type that can be used as a step configuration field (see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).
Attributes:
| Name | Type | Description |
|---|---|---|
target |
Optional[str] |
target column |
prediction |
Union[str, Sequence[str]] |
target column |
datetime |
Optional[str] |
datetime column |
id |
Optional[str] |
id column |
numerical_features |
Optional[List[str]] |
numerical features |
categorical_features |
Optional[List[str]] |
categorical features |
datetime_features |
Optional[List[str]] |
datetime features |
target_names |
Optional[List[str]] |
target column names |
task |
Optional[str] |
model task |
pos_label |
Union[str, int] |
positive label |
text_features |
Optional[List[str]] |
text features |
Source code in zenml/integrations/evidently/column_mapping.py
class EvidentlyColumnMapping(BaseModel):
"""Column mapping configuration for Evidently.
This class is a 1-to-1 serializable analogue of Evidently's
ColumnMapping data type that can be used as a step configuration field
(see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).
Attributes:
target: target column
prediction: target column
datetime: datetime column
id: id column
numerical_features: numerical features
categorical_features: categorical features
datetime_features: datetime features
target_names: target column names
task: model task
pos_label: positive label
text_features: text features
"""
target: Optional[str] = None
prediction: Optional[Union[str, Sequence[str]]] = "prediction"
datetime: Optional[str] = None
id: Optional[str] = None
numerical_features: Optional[List[str]] = None
categorical_features: Optional[List[str]] = None
datetime_features: Optional[List[str]] = None
target_names: Optional[List[str]] = None
task: Optional[str] = None
pos_label: Optional[Union[str, int]] = 1
text_features: Optional[List[str]] = None
def to_evidently_column_mapping(self) -> ColumnMapping:
"""Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
An Evidently column mapping converted from this Pydantic object.
"""
column_mapping = ColumnMapping()
# preserve the Evidently defaults where possible
column_mapping.target = self.target or column_mapping.target
column_mapping.prediction = (
self.prediction or column_mapping.prediction
)
column_mapping.datetime = self.datetime or column_mapping.datetime
column_mapping.id = self.id or column_mapping.id
column_mapping.numerical_features = (
self.numerical_features or column_mapping.numerical_features
)
column_mapping.datetime_features = (
self.datetime_features or column_mapping.datetime_features
)
column_mapping.target_names = (
self.target_names or column_mapping.target_names
)
column_mapping.task = self.task or column_mapping.task
column_mapping.pos_label = self.pos_label or column_mapping.pos_label
column_mapping.text_features = (
self.text_features or column_mapping.text_features
)
return column_mapping
to_evidently_column_mapping(self)
Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
| Type | Description |
|---|---|
ColumnMapping |
An Evidently column mapping converted from this Pydantic object. |
Source code in zenml/integrations/evidently/column_mapping.py
def to_evidently_column_mapping(self) -> ColumnMapping:
"""Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
An Evidently column mapping converted from this Pydantic object.
"""
column_mapping = ColumnMapping()
# preserve the Evidently defaults where possible
column_mapping.target = self.target or column_mapping.target
column_mapping.prediction = (
self.prediction or column_mapping.prediction
)
column_mapping.datetime = self.datetime or column_mapping.datetime
column_mapping.id = self.id or column_mapping.id
column_mapping.numerical_features = (
self.numerical_features or column_mapping.numerical_features
)
column_mapping.datetime_features = (
self.datetime_features or column_mapping.datetime_features
)
column_mapping.target_names = (
self.target_names or column_mapping.target_names
)
column_mapping.task = self.task or column_mapping.task
column_mapping.pos_label = self.pos_label or column_mapping.pos_label
column_mapping.text_features = (
self.text_features or column_mapping.text_features
)
return column_mapping
data_validators
special
Initialization of the Evidently data validator for ZenML.
evidently_data_validator
Implementation of the Evidently data validator.
EvidentlyDataValidator (BaseDataValidator)
Evidently data validator stack component.
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidator(BaseDataValidator):
"""Evidently data validator stack component."""
NAME: ClassVar[str] = "Evidently"
FLAVOR: ClassVar[
Type[BaseDataValidatorFlavor]
] = EvidentlyDataValidatorFlavor
@classmethod
def _unpack_options(
cls, option_list: Sequence[Tuple[str, Dict[str, Any]]]
) -> Sequence[Any]:
"""Unpack Evidently options.
Implements de-serialization for [Evidently options](https://docs.evidentlyai.com/user-guide/customization)
that can be passed as constructor arguments when creating Profile and
Dashboard objects. The convention used is that each item in the list
consists of two elements:
* a string containing the full class path of a `dataclass` based
class with Evidently options
* a dictionary with kwargs used as parameters for the option instance
For example,
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
This is the same as saying:
```python
from evidently.options import ColorOptions
color_scheme = ColorOptions()
color_scheme.primary_color = "#5a86ad"
color_scheme.fill_color = "#fff4f2"
color_scheme.zero_line_color = "#016795"
color_scheme.current_data_color = "#c292a1"
color_scheme.reference_data_color = "#017b92"
```
Args:
option_list: list of packed Evidently options
Returns:
A list of unpacked Evidently options
Raises:
ValueError: if one of the passed Evidently class paths cannot be
resolved to an actual class.
"""
options = []
for option_clspath, option_args in option_list:
try:
option_cls = load_source_path(option_clspath)
except AttributeError:
raise ValueError(
f"Could not map the `{option_clspath}` Evidently option "
f"class path to a valid class."
)
option = option_cls(**option_args)
options.append(option)
return options
@staticmethod
def _download_nltk_data() -> None:
"""Download NLTK data for text metrics and tests.
Raises:
ImportError: if NLTK is not installed.
"""
try:
import nltk # type: ignore[import]
from nltk.data import path as nltk_path # type: ignore[import]
except ImportError:
raise ImportError(
"NLTK is not installed. Please install NLTK to use "
"Evidently text metrics and tests."
)
# Configure NLTK to use the current working directory to download and
# lookup data. This is necessary because the default download directory
# is not writable in some Docker containers.
nltk_path.append(os.getcwd())
# Download NLTK data. We need this later on for the Evidently text report.
nltk.download("words", download_dir=os.getcwd())
nltk.download("wordnet", download_dir=os.getcwd())
nltk.download("omw-1.4", download_dir=os.getcwd())
def data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
column_mapping: Optional[ColumnMapping] = None,
report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
download_nltk_data: bool = False,
**kwargs: Any,
) -> Report:
"""Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (`report_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
Args:
dataset: Target dataset to be profiled. When a comparison dataset
is provided, this dataset is considered the reference dataset.
comparison_dataset: Optional dataset to be used for data profiles
that require a current dataset for comparison (e.g data drift
profiles).
profile_list: List of Evidently metric configurations to
be included in the report. If not provided, all available
metric presets will be included.
column_mapping: Properties of the DataFrame columns used
report_options: List of Evidently options to be passed to the
report constructor.
download_nltk_data: Whether to download NLTK data for text metrics.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Report as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
metrics = [metric.to_evidently_metric() for metric in profile_list]
unpacked_report_options = self._unpack_options(report_options)
report = Report(metrics=metrics, options=unpacked_report_options)
report.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return report
def data_validation(
self,
dataset: Any,
comparison_dataset: Optional[Any] = None,
check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
column_mapping: Optional[ColumnMapping] = None,
download_nltk_data: bool = False,
**kwargs: Any,
) -> TestSuite:
"""Validate a dataset with Evidently.
Args:
dataset: Target dataset to be validated.
comparison_dataset: Optional dataset to be used for data validation
that require a baseline for comparison (e.g data drift
validation).
check_list: List of Evidently test configurations to be
included in the test suite. If not provided, all available
test presets will be included.
test_options: List of Evidently options to be passed to the
test suite constructor.
column_mapping: Properties of the DataFrame columns used
download_nltk_data: Whether to download NLTK data for text tests.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Test Suite as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
check_list = check_list or EvidentlyTestConfig.default_tests()
tests = [test.to_evidently_test() for test in check_list]
unpacked_test_options = self._unpack_options(test_options)
test_suite = TestSuite(tests=tests, options=unpacked_test_options)
test_suite.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return test_suite
def legacy_data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[str]] = None,
column_mapping: Optional[ColumnMapping] = None,
verbose_level: int = 1,
profile_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
dashboard_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
**kwargs: Any,
) -> Tuple[Profile, Dashboard]:
"""Analyze a dataset and generate a data profile with Evidently.
The method takes in an optional list of Evidently options to be passed
to the profile constructor (`profile_options`) and the dashboard
constructor (`dashboard_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters.
Args:
dataset: Target dataset to be profiled.
comparison_dataset: Optional dataset to be used for data profiles
that require a baseline for comparison (e.g data drift profiles).
profile_list: Optional list identifying the categories of Evidently
data profiles to be generated.
column_mapping: Properties of the DataFrame columns used
verbose_level: Level of verbosity for the Evidently dashboards. Use
0 for a brief dashboard, 1 for a detailed dashboard.
profile_options: Optional list of options to pass to the
profile constructor.
dashboard_options: Optional list of options to pass to the
dashboard constructor.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Profile and Dashboard objects corresponding to the set
of generated profiles.
"""
logger.warning(
"The ZenML Evidently data profile step and data validator "
"methods that are still using Evidently Profile and Dashboard "
"objects are deprecated and will be removed in a future release. "
"Please use the new data report step and data validator methods "
"that make use of the Evidently Report and Test Suite objects "
"instead."
)
sections, tabs = get_profile_sections_and_tabs(
profile_list, verbose_level
)
unpacked_profile_options = self._unpack_options(profile_options)
unpacked_dashboard_options = self._unpack_options(dashboard_options)
dashboard = Dashboard(tabs=tabs, options=unpacked_dashboard_options)
dashboard.calculate(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
profile = Profile(sections=sections, options=unpacked_profile_options)
profile.calculate(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return profile, dashboard
FLAVOR (BaseDataValidatorFlavor)
Evidently data validator flavor.
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
"""Evidently data validator flavor."""
@property
def name(self) -> str:
"""Name of the flavor.
Returns:
The name of the flavor.
"""
return EVIDENTLY_DATA_VALIDATOR_FLAVOR
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at SDK docs explaining this flavor.
Returns:
A flavor SDK docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"
@property
def implementation_class(self) -> Type["EvidentlyDataValidator"]:
"""Implementation class.
Returns:
The implementation class.
"""
from zenml.integrations.evidently.data_validators import (
EvidentlyDataValidator,
)
return EvidentlyDataValidator
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[EvidentlyDataValidator]
property
readonly
Implementation class.
Returns:
| Type | Description |
|---|---|
Type[EvidentlyDataValidator] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
| Type | Description |
|---|---|
str |
The flavor logo. |
name: str
property
readonly
Name of the flavor.
Returns:
| Type | Description |
|---|---|
str |
The name of the flavor. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at SDK docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor SDK docs url. |
data_profiling(self, dataset, comparison_dataset=None, profile_list=None, column_mapping=None, report_options=[], download_nltk_data=False, **kwargs)
Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (report_options). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option dataclass, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
DataFrame |
Target dataset to be profiled. When a comparison dataset is provided, this dataset is considered the reference dataset. |
required |
comparison_dataset |
Optional[pandas.core.frame.DataFrame] |
Optional dataset to be used for data profiles that require a current dataset for comparison (e.g data drift profiles). |
None |
profile_list |
Optional[Sequence[zenml.integrations.evidently.metrics.EvidentlyMetricConfig]] |
List of Evidently metric configurations to be included in the report. If not provided, all available metric presets will be included. |
None |
column_mapping |
Optional[evidently.pipeline.column_mapping.ColumnMapping] |
Properties of the DataFrame columns used |
None |
report_options |
Sequence[Tuple[str, Dict[str, Any]]] |
List of Evidently options to be passed to the report constructor. |
[] |
download_nltk_data |
bool |
Whether to download NLTK data for text metrics. Defaults to False. |
False |
**kwargs |
Any |
Extra keyword arguments (unused). |
{} |
Returns:
| Type | Description |
|---|---|
Report |
The Evidently Report as JSON object and as HTML. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
column_mapping: Optional[ColumnMapping] = None,
report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
download_nltk_data: bool = False,
**kwargs: Any,
) -> Report:
"""Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (`report_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
Args:
dataset: Target dataset to be profiled. When a comparison dataset
is provided, this dataset is considered the reference dataset.
comparison_dataset: Optional dataset to be used for data profiles
that require a current dataset for comparison (e.g data drift
profiles).
profile_list: List of Evidently metric configurations to
be included in the report. If not provided, all available
metric presets will be included.
column_mapping: Properties of the DataFrame columns used
report_options: List of Evidently options to be passed to the
report constructor.
download_nltk_data: Whether to download NLTK data for text metrics.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Report as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
metrics = [metric.to_evidently_metric() for metric in profile_list]
unpacked_report_options = self._unpack_options(report_options)
report = Report(metrics=metrics, options=unpacked_report_options)
report.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return report
data_validation(self, dataset, comparison_dataset=None, check_list=None, test_options=[], column_mapping=None, download_nltk_data=False, **kwargs)
Validate a dataset with Evidently.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
Any |
Target dataset to be validated. |
required |
comparison_dataset |
Optional[Any] |
Optional dataset to be used for data validation that require a baseline for comparison (e.g data drift validation). |
None |
check_list |
Optional[Sequence[zenml.integrations.evidently.tests.EvidentlyTestConfig]] |
List of Evidently test configurations to be included in the test suite. If not provided, all available test presets will be included. |
None |
test_options |
Sequence[Tuple[str, Dict[str, Any]]] |
List of Evidently options to be passed to the test suite constructor. |
[] |
column_mapping |
Optional[evidently.pipeline.column_mapping.ColumnMapping] |
Properties of the DataFrame columns used |
None |
download_nltk_data |
bool |
Whether to download NLTK data for text tests. Defaults to False. |
False |
**kwargs |
Any |
Extra keyword arguments (unused). |
{} |
Returns:
| Type | Description |
|---|---|
TestSuite |
The Evidently Test Suite as JSON object and as HTML. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_validation(
self,
dataset: Any,
comparison_dataset: Optional[Any] = None,
check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
column_mapping: Optional[ColumnMapping] = None,
download_nltk_data: bool = False,
**kwargs: Any,
) -> TestSuite:
"""Validate a dataset with Evidently.
Args:
dataset: Target dataset to be validated.
comparison_dataset: Optional dataset to be used for data validation
that require a baseline for comparison (e.g data drift
validation).
check_list: List of Evidently test configurations to be
included in the test suite. If not provided, all available
test presets will be included.
test_options: List of Evidently options to be passed to the
test suite constructor.
column_mapping: Properties of the DataFrame columns used
download_nltk_data: Whether to download NLTK data for text tests.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Test Suite as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
check_list = check_list or EvidentlyTestConfig.default_tests()
tests = [test.to_evidently_test() for test in check_list]
unpacked_test_options = self._unpack_options(test_options)
test_suite = TestSuite(tests=tests, options=unpacked_test_options)
test_suite.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return test_suite
legacy_data_profiling(self, dataset, comparison_dataset=None, profile_list=None, column_mapping=None, verbose_level=1, profile_options=[], dashboard_options=[], **kwargs)
Analyze a dataset and generate a data profile with Evidently.
The method takes in an optional list of Evidently options to be passed
to the profile constructor (profile_options) and the dashboard
constructor (dashboard_options). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option dataclass, the second is a dictionary of kwargs with the actual
option parameters.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
DataFrame |
Target dataset to be profiled. |
required |
comparison_dataset |
Optional[pandas.core.frame.DataFrame] |
Optional dataset to be used for data profiles that require a baseline for comparison (e.g data drift profiles). |
None |
profile_list |
Optional[Sequence[str]] |
Optional list identifying the categories of Evidently data profiles to be generated. |
None |
column_mapping |
Optional[evidently.pipeline.column_mapping.ColumnMapping] |
Properties of the DataFrame columns used |
None |
verbose_level |
int |
Level of verbosity for the Evidently dashboards. Use 0 for a brief dashboard, 1 for a detailed dashboard. |
1 |
profile_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the profile constructor. |
[] |
dashboard_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the dashboard constructor. |
[] |
**kwargs |
Any |
Extra keyword arguments (unused). |
{} |
Returns:
| Type | Description |
|---|---|
Tuple[evidently.model_profile.model_profile.Profile, evidently.dashboard.dashboard.Dashboard] |
The Evidently Profile and Dashboard objects corresponding to the set of generated profiles. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def legacy_data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[str]] = None,
column_mapping: Optional[ColumnMapping] = None,
verbose_level: int = 1,
profile_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
dashboard_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
**kwargs: Any,
) -> Tuple[Profile, Dashboard]:
"""Analyze a dataset and generate a data profile with Evidently.
The method takes in an optional list of Evidently options to be passed
to the profile constructor (`profile_options`) and the dashboard
constructor (`dashboard_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters.
Args:
dataset: Target dataset to be profiled.
comparison_dataset: Optional dataset to be used for data profiles
that require a baseline for comparison (e.g data drift profiles).
profile_list: Optional list identifying the categories of Evidently
data profiles to be generated.
column_mapping: Properties of the DataFrame columns used
verbose_level: Level of verbosity for the Evidently dashboards. Use
0 for a brief dashboard, 1 for a detailed dashboard.
profile_options: Optional list of options to pass to the
profile constructor.
dashboard_options: Optional list of options to pass to the
dashboard constructor.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Profile and Dashboard objects corresponding to the set
of generated profiles.
"""
logger.warning(
"The ZenML Evidently data profile step and data validator "
"methods that are still using Evidently Profile and Dashboard "
"objects are deprecated and will be removed in a future release. "
"Please use the new data report step and data validator methods "
"that make use of the Evidently Report and Test Suite objects "
"instead."
)
sections, tabs = get_profile_sections_and_tabs(
profile_list, verbose_level
)
unpacked_profile_options = self._unpack_options(profile_options)
unpacked_dashboard_options = self._unpack_options(dashboard_options)
dashboard = Dashboard(tabs=tabs, options=unpacked_dashboard_options)
dashboard.calculate(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
profile = Profile(sections=sections, options=unpacked_profile_options)
profile.calculate(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return profile, dashboard
get_profile_sections_and_tabs(profile_list, verbose_level=1)
Get the profile sections and dashboard tabs for a profile list.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
profile_list |
Optional[Sequence[str]] |
List of identifiers for Evidently profiles. |
required |
verbose_level |
int |
Verbosity level for the rendered dashboard. Use 0 for a brief dashboard, 1 for a detailed dashboard. |
1 |
Returns:
| Type | Description |
|---|---|
Tuple[List[evidently.model_profile.sections.base_profile_section.ProfileSection], List[evidently.dashboard.tabs.base_tab.Tab]] |
A tuple of two lists of profile sections and tabs. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
if the profile_section is not supported. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def get_profile_sections_and_tabs(
profile_list: Optional[Sequence[str]],
verbose_level: int = 1,
) -> Tuple[List[ProfileSection], List[Tab]]:
"""Get the profile sections and dashboard tabs for a profile list.
Args:
profile_list: List of identifiers for Evidently profiles.
verbose_level: Verbosity level for the rendered dashboard. Use
0 for a brief dashboard, 1 for a detailed dashboard.
Returns:
A tuple of two lists of profile sections and tabs.
Raises:
ValueError: if the profile_section is not supported.
"""
profile_list = profile_list or list(profile_mapper.keys())
try:
return (
[profile_mapper[profile]() for profile in profile_list],
[
dashboard_mapper[profile](verbose_level=verbose_level)
for profile in profile_list
],
)
except KeyError as e:
nl = "\n"
raise ValueError(
f"Invalid profile sections: {profile_list} \n\n"
f"Valid and supported options are: {nl}- "
f'{f"{nl}- ".join(list(profile_mapper.keys()))}'
) from e
flavors
special
Evidently integration flavors.
evidently_data_validator_flavor
Evidently data validator flavor.
EvidentlyDataValidatorFlavor (BaseDataValidatorFlavor)
Evidently data validator flavor.
Source code in zenml/integrations/evidently/flavors/evidently_data_validator_flavor.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
"""Evidently data validator flavor."""
@property
def name(self) -> str:
"""Name of the flavor.
Returns:
The name of the flavor.
"""
return EVIDENTLY_DATA_VALIDATOR_FLAVOR
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at SDK docs explaining this flavor.
Returns:
A flavor SDK docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"
@property
def implementation_class(self) -> Type["EvidentlyDataValidator"]:
"""Implementation class.
Returns:
The implementation class.
"""
from zenml.integrations.evidently.data_validators import (
EvidentlyDataValidator,
)
return EvidentlyDataValidator
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[EvidentlyDataValidator]
property
readonly
Implementation class.
Returns:
| Type | Description |
|---|---|
Type[EvidentlyDataValidator] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
| Type | Description |
|---|---|
str |
The flavor logo. |
name: str
property
readonly
Name of the flavor.
Returns:
| Type | Description |
|---|---|
str |
The name of the flavor. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at SDK docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor SDK docs url. |
materializers
special
Evidently materializers.
evidently_profile_materializer
Implementation of Evidently profile materializer.
EvidentlyProfileMaterializer (BaseMaterializer)
Materializer to read data to and from an Evidently Profile.
Source code in zenml/integrations/evidently/materializers/evidently_profile_materializer.py
class EvidentlyProfileMaterializer(BaseMaterializer):
"""Materializer to read data to and from an Evidently Profile."""
ASSOCIATED_TYPES = (Profile,)
ASSOCIATED_ARTIFACT_TYPE = ArtifactType.DATA_ANALYSIS
def load(self, data_type: Type[Any]) -> Profile:
"""Reads an Evidently Profile object from a json file.
Args:
data_type: The type of the data to read.
Returns:
The Evidently Profile
Raises:
TypeError: if the json file contains an invalid data type.
"""
super().load(data_type)
filepath = os.path.join(self.uri, DEFAULT_FILENAME)
contents = yaml_utils.read_json(filepath)
if type(contents) != dict:
raise TypeError(
f"Contents {contents} was type {type(contents)} but expected "
f"dictionary"
)
section_types = contents.pop("section_types", [])
sections = []
for section_type in section_types:
section_cls = import_class_by_path(section_type)
section = section_cls()
section._result = contents[section.part_id()]
sections.append(section)
return Profile(sections=sections)
def save(self, data: Profile) -> None:
"""Serialize an Evidently Profile to a json file.
Args:
data: The Evidently Profile to be serialized.
"""
super().save(data)
contents = data.object()
# include the list of profile sections in the serialized dictionary,
# so we'll be able to re-create them during de-serialization
contents["section_types"] = [
resolve_class(stage.__class__) for stage in data.stages
]
filepath = os.path.join(self.uri, DEFAULT_FILENAME)
yaml_utils.write_json(filepath, contents, encoder=NumpyEncoder)
load(self, data_type)
Reads an Evidently Profile object from a json file.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data_type |
Type[Any] |
The type of the data to read. |
required |
Returns:
| Type | Description |
|---|---|
Profile |
The Evidently Profile |
Exceptions:
| Type | Description |
|---|---|
TypeError |
if the json file contains an invalid data type. |
Source code in zenml/integrations/evidently/materializers/evidently_profile_materializer.py
def load(self, data_type: Type[Any]) -> Profile:
"""Reads an Evidently Profile object from a json file.
Args:
data_type: The type of the data to read.
Returns:
The Evidently Profile
Raises:
TypeError: if the json file contains an invalid data type.
"""
super().load(data_type)
filepath = os.path.join(self.uri, DEFAULT_FILENAME)
contents = yaml_utils.read_json(filepath)
if type(contents) != dict:
raise TypeError(
f"Contents {contents} was type {type(contents)} but expected "
f"dictionary"
)
section_types = contents.pop("section_types", [])
sections = []
for section_type in section_types:
section_cls = import_class_by_path(section_type)
section = section_cls()
section._result = contents[section.part_id()]
sections.append(section)
return Profile(sections=sections)
save(self, data)
Serialize an Evidently Profile to a json file.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
data |
Profile |
The Evidently Profile to be serialized. |
required |
Source code in zenml/integrations/evidently/materializers/evidently_profile_materializer.py
def save(self, data: Profile) -> None:
"""Serialize an Evidently Profile to a json file.
Args:
data: The Evidently Profile to be serialized.
"""
super().save(data)
contents = data.object()
# include the list of profile sections in the serialized dictionary,
# so we'll be able to re-create them during de-serialization
contents["section_types"] = [
resolve_class(stage.__class__) for stage in data.stages
]
filepath = os.path.join(self.uri, DEFAULT_FILENAME)
yaml_utils.write_json(filepath, contents, encoder=NumpyEncoder)
metrics
ZenML declarative representation of Evidently Metrics.
EvidentlyMetricConfig (BaseModel)
pydantic-model
Declarative Evidently Metric configuration.
This is a declarative representation of the configuration that goes into an Evidently Metric, MetricPreset or Metric generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Metric from it.
This representation covers all 3 possible ways of configuring an Evidently Metric or Metric-like object that can later be used in an Evidently Report:
- A Metric (derived from the Metric class).
- A MetricPreset (derived from the MetricPreset class).
- A column Metric generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Metric or Metric-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Metric classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.
Attributes:
| Name | Type | Description |
|---|---|---|
class_path |
str |
The full class path of the Evidently Metric class. |
parameters |
Dict[str, Any] |
The parameters of the Evidently Metric. |
is_generator |
bool |
Whether this is an Evidently column Metric generator. |
columns |
Union[str, List[str]] |
The columns that the Evidently column Metric generator is
applied to. Only used if |
skip_id_column |
bool |
Whether to skip the ID column when applying the
Evidently Metric generator. Only used if |
Source code in zenml/integrations/evidently/metrics.py
class EvidentlyMetricConfig(BaseModel):
"""Declarative Evidently Metric configuration.
This is a declarative representation of the configuration that goes into an
Evidently Metric, MetricPreset or Metric generator instance. We need this to
be able to store the configuration as part of a ZenML step parameter and
later instantiate the Evidently Metric from it.
This representation covers all 3 possible ways of configuring an Evidently
Metric or Metric-like object that can later be used in an Evidently Report:
1. A Metric (derived from the Metric class).
2. A MetricPreset (derived from the MetricPreset class).
3. A column Metric generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Metric or Metric-like
object to this class and have it automatically derive the configuration used
to instantiate it. Unfortunately, this is not possible because the Evidently
Metric classes are not designed in a way that allows us to extract the
constructor parameters from them in a generic way.
Attributes:
class_path: The full class path of the Evidently Metric class.
parameters: The parameters of the Evidently Metric.
is_generator: Whether this is an Evidently column Metric generator.
columns: The columns that the Evidently column Metric generator is
applied to. Only used if `generator` is True.
skip_id_column: Whether to skip the ID column when applying the
Evidently Metric generator. Only used if `generator` is True.
"""
class_path: str
parameters: Dict[str, Any] = Field(default_factory=dict)
is_generator: bool = False
columns: Optional[Union[str, List[str]]] = None
skip_id_column: bool = False
@staticmethod
def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
"""Get the Evidently metric or metric preset class from a string.
Args:
metric_name: The metric or metric preset class or full class
path.
Returns:
The Evidently metric or metric preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
metric or metric preset class.
"""
# First, try to interpret the metric name as a full class path.
if "." in metric_name:
try:
metric_class = import_class_by_path(metric_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`: {str(e)}"
)
else:
# Next, try to interpret the metric as a Metric or MetricPreset
# class name
if hasattr(metrics, metric_name):
metric_class = getattr(metrics, metric_name)
elif hasattr(metric_preset, metric_name):
metric_class = getattr(metric_preset, metric_name)
else:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`"
)
if not issubclass(metric_class, (Metric, MetricPreset)):
raise ValueError(
f"Class `{metric_name}` is not a valid Evidently "
f"Metric or MetricPreset."
)
return metric_class
@classmethod
def metric_generator(
cls,
metric: Union[Type[Metric], str],
columns: Optional[Union[str, List[str]]] = None,
skip_id_column: bool = False,
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Metric generator.
The `columns`, `skip_id_column` and `parameters` arguments will be
passed to the Evidently `generate_column_metrics` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
```
Args:
metric: The Evidently Metric class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_metrics` function
takes.
skip_id_column: Whether to skip the ID column when applying the
generator.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
`generate_column_metrics` function.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric generator configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(f"Invalid Evidently Metric class: {metric}")
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
skip_id_column=skip_id_column,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
@classmethod
def metric(
cls,
metric: Union[Type[Metric], Type[MetricPreset], str],
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the
configuration of an Evidently Metric.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
```
Args:
metric: The Evidently Metric or MetricPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric or MetricPreset.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(
f"Invalid Evidently Metric or MetricPreset class: {metric}"
)
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
@classmethod
def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
"""Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
`TextOverviewPreset` which requires a text column, which we don't have
by default.
Returns:
A list of EvidentlyMetricConfig objects to use as default metrics.
"""
return [
cls.metric(metric=metric_preset_class_name)
for metric_preset_class_name in metric_preset.__all__
# TextOverviewPreset requires a text column, which we don't
# have by default
if metric_preset_class_name != "TextOverviewPreset"
]
def to_evidently_metric(
self,
) -> Union[Metric, MetricPreset, BaseGenerator]:
"""Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric
generator instance from its declarative representation.
Returns:
The Evidently Metric, MetricPreset or metric generator object.
Raises:
ValueError: If the Evidently Metric, MetricPreset or column metric
generator could not be instantiated.
"""
metric_class = self.get_metric_class(self.class_path)
if self.is_generator:
try:
return generate_column_metrics(
metric_class=metric_class,
columns=self.columns,
skip_id_column=self.skip_id_column,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Metric generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return metric_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Metric or MetricPreset "
f"`{self.class_path}`: {str(e)}"
)
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/metrics.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_metrics()
classmethod
Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
TextOverviewPreset which requires a text column, which we don't have
by default.
Returns:
| Type | Description |
|---|---|
List[EvidentlyMetricConfig] |
A list of EvidentlyMetricConfig objects to use as default metrics. |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
"""Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
`TextOverviewPreset` which requires a text column, which we don't have
by default.
Returns:
A list of EvidentlyMetricConfig objects to use as default metrics.
"""
return [
cls.metric(metric=metric_preset_class_name)
for metric_preset_class_name in metric_preset.__all__
# TextOverviewPreset requires a text column, which we don't
# have by default
if metric_preset_class_name != "TextOverviewPreset"
]
get_metric_class(metric_name)
staticmethod
Get the Evidently metric or metric preset class from a string.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric_name |
str |
The metric or metric preset class or full class path. |
required |
Returns:
| Type | Description |
|---|---|
Union[evidently.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset] |
The Evidently metric or metric preset class. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the name cannot be converted into a valid Evidently metric or metric preset class. |
Source code in zenml/integrations/evidently/metrics.py
@staticmethod
def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
"""Get the Evidently metric or metric preset class from a string.
Args:
metric_name: The metric or metric preset class or full class
path.
Returns:
The Evidently metric or metric preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
metric or metric preset class.
"""
# First, try to interpret the metric name as a full class path.
if "." in metric_name:
try:
metric_class = import_class_by_path(metric_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`: {str(e)}"
)
else:
# Next, try to interpret the metric as a Metric or MetricPreset
# class name
if hasattr(metrics, metric_name):
metric_class = getattr(metrics, metric_name)
elif hasattr(metric_preset, metric_name):
metric_class = getattr(metric_preset, metric_name)
else:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`"
)
if not issubclass(metric_class, (Metric, MetricPreset)):
raise ValueError(
f"Class `{metric_name}` is not a valid Evidently "
f"Metric or MetricPreset."
)
return metric_class
metric(metric, **parameters)
classmethod
Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the configuration of an Evidently Metric.
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric |
Union[Type[evidently.base_metric.Metric], Type[evidently.metric_preset.metric_preset.MetricPreset], str] |
The Evidently Metric or MetricPreset class, class name or class path. |
required |
parameters |
Any |
Additional optional parameters needed to instantiate the Evidently Metric or MetricPreset. |
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyMetricConfig |
The EvidentlyMetric declarative representation of the Evidently Metric configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric(
cls,
metric: Union[Type[Metric], Type[MetricPreset], str],
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the
configuration of an Evidently Metric.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
```
Args:
metric: The Evidently Metric or MetricPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric or MetricPreset.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(
f"Invalid Evidently Metric or MetricPreset class: {metric}"
)
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
metric_generator(metric, columns=None, skip_id_column=False, **parameters)
classmethod
Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the configuration of an Evidently column Metric generator.
The columns, skip_id_column and parameters arguments will be
passed to the Evidently generate_column_metrics function:
- if
columnsis a list, it is interpreted as a list of column names. - if
columnsis a string, it can be one of values:- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric |
Union[Type[evidently.base_metric.Metric], str] |
The Evidently Metric class, class name or class path to use for the generator. |
required |
columns |
Union[str, List[str]] |
The columns to apply the generator to. Takes the same
values that the Evidently |
None |
skip_id_column |
bool |
Whether to skip the ID column when applying the generator. |
False |
parameters |
Any |
Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
|
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyMetricConfig |
The EvidentlyMetric declarative representation of the Evidently Metric generator configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric_generator(
cls,
metric: Union[Type[Metric], str],
columns: Optional[Union[str, List[str]]] = None,
skip_id_column: bool = False,
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Metric generator.
The `columns`, `skip_id_column` and `parameters` arguments will be
passed to the Evidently `generate_column_metrics` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
```
Args:
metric: The Evidently Metric class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_metrics` function
takes.
skip_id_column: Whether to skip the ID column when applying the
generator.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
`generate_column_metrics` function.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric generator configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(f"Invalid Evidently Metric class: {metric}")
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
skip_id_column=skip_id_column,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
to_evidently_metric(self)
Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric generator instance from its declarative representation.
Returns:
| Type | Description |
|---|---|
Union[evidently.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset, evidently.utils.generators.BaseGenerator] |
The Evidently Metric, MetricPreset or metric generator object. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the Evidently Metric, MetricPreset or column metric generator could not be instantiated. |
Source code in zenml/integrations/evidently/metrics.py
def to_evidently_metric(
self,
) -> Union[Metric, MetricPreset, BaseGenerator]:
"""Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric
generator instance from its declarative representation.
Returns:
The Evidently Metric, MetricPreset or metric generator object.
Raises:
ValueError: If the Evidently Metric, MetricPreset or column metric
generator could not be instantiated.
"""
metric_class = self.get_metric_class(self.class_path)
if self.is_generator:
try:
return generate_column_metrics(
metric_class=metric_class,
columns=self.columns,
skip_id_column=self.skip_id_column,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Metric generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return metric_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Metric or MetricPreset "
f"`{self.class_path}`: {str(e)}"
)
steps
special
Initialization of the Evidently Standard Steps.
evidently_profile
Implementation of the Evidently Profile Step.
EvidentlyProfileParameters (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently profile step |
profile_sections |
Optional[Sequence[str]] |
a list identifying the Evidently profile sections to be used. The following are valid options supported by Evidently: - "datadrift" - "categoricaltargetdrift" - "numericaltargetdrift" - "classificationmodelperformance" - "regressionmodelperformance" - "probabilisticmodelperformance" |
verbose_level |
int |
Verbosity level for the Evidently dashboards. Use 0 for a brief dashboard, 1 for a detailed dashboard. |
profile_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the
profile constructor. See |
dashboard_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the
dashboard constructor. See |
Source code in zenml/integrations/evidently/steps/evidently_profile.py
class EvidentlyProfileParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently profile step
profile_sections: a list identifying the Evidently profile sections to be
used. The following are valid options supported by Evidently:
- "datadrift"
- "categoricaltargetdrift"
- "numericaltargetdrift"
- "classificationmodelperformance"
- "regressionmodelperformance"
- "probabilisticmodelperformance"
verbose_level: Verbosity level for the Evidently dashboards. Use
0 for a brief dashboard, 1 for a detailed dashboard.
profile_options: Optional list of options to pass to the
profile constructor. See `EvidentlyDataValidator._unpack_options`.
dashboard_options: Optional list of options to pass to the
dashboard constructor. See `EvidentlyDataValidator._unpack_options`.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
profile_sections: Optional[Sequence[str]] = None
verbose_level: int = 1
profile_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
dashboard_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
EvidentlyProfileStep (BaseStep)
Step implementation implementing an Evidently Profile Step.
Source code in zenml/integrations/evidently/steps/evidently_profile.py
class EvidentlyProfileStep(BaseStep):
"""Step implementation implementing an Evidently Profile Step."""
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyProfileParameters,
) -> Output( # type:ignore[valid-type]
profile=Profile, dashboard=str
):
"""Main entrypoint for the Evidently categorical target drift detection step.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
profile: Evidently Profile generated for the data drift
dashboard: HTML report extracted from an Evidently Dashboard
generated for the data drift.
Raises:
ValueError: If ignored_cols is an empty list
ValueError: If column is not found in reference or comparison
dataset
"""
data_validator = cast(
EvidentlyDataValidator,
EvidentlyDataValidator.get_active_data_validator(),
)
column_mapping = None
if params.ignored_cols is None:
pass
elif not params.ignored_cols:
raise ValueError(
f"Expects None or list of columns in strings, but got {params.ignored_cols}"
)
elif not (
set(params.ignored_cols).issubset(set(reference_dataset.columns))
) or not (
set(params.ignored_cols).issubset(set(comparison_dataset.columns))
):
raise ValueError(
"Column is not found in reference or comparison datasets"
)
else:
reference_dataset = reference_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
comparison_dataset = comparison_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if params.column_mapping:
column_mapping = (
params.column_mapping.to_evidently_column_mapping()
)
profile, dashboard = data_validator.legacy_data_profiling(
dataset=reference_dataset,
comparison_dataset=comparison_dataset,
profile_list=params.profile_sections,
column_mapping=column_mapping,
verbose_level=params.verbose_level,
profile_options=params.profile_options,
dashboard_options=params.dashboard_options,
)
return [profile, dashboard.html()]
PARAMETERS_CLASS (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently profile step |
profile_sections |
Optional[Sequence[str]] |
a list identifying the Evidently profile sections to be used. The following are valid options supported by Evidently: - "datadrift" - "categoricaltargetdrift" - "numericaltargetdrift" - "classificationmodelperformance" - "regressionmodelperformance" - "probabilisticmodelperformance" |
verbose_level |
int |
Verbosity level for the Evidently dashboards. Use 0 for a brief dashboard, 1 for a detailed dashboard. |
profile_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the
profile constructor. See |
dashboard_options |
Sequence[Tuple[str, Dict[str, Any]]] |
Optional list of options to pass to the
dashboard constructor. See |
Source code in zenml/integrations/evidently/steps/evidently_profile.py
class EvidentlyProfileParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently profile step
profile_sections: a list identifying the Evidently profile sections to be
used. The following are valid options supported by Evidently:
- "datadrift"
- "categoricaltargetdrift"
- "numericaltargetdrift"
- "classificationmodelperformance"
- "regressionmodelperformance"
- "probabilisticmodelperformance"
verbose_level: Verbosity level for the Evidently dashboards. Use
0 for a brief dashboard, 1 for a detailed dashboard.
profile_options: Optional list of options to pass to the
profile constructor. See `EvidentlyDataValidator._unpack_options`.
dashboard_options: Optional list of options to pass to the
dashboard constructor. See `EvidentlyDataValidator._unpack_options`.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
profile_sections: Optional[Sequence[str]] = None
verbose_level: int = 1
profile_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
dashboard_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
entrypoint(self, reference_dataset, comparison_dataset, params)
Main entrypoint for the Evidently categorical target drift detection step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
reference_dataset |
DataFrame |
a Pandas DataFrame |
required |
comparison_dataset |
DataFrame |
a Pandas DataFrame of new data you wish to compare against the reference data |
required |
params |
EvidentlyProfileParameters |
the parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
profile |
Evidently Profile generated for the data drift dashboard: HTML report extracted from an Evidently Dashboard generated for the data drift. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If ignored_cols is an empty list |
ValueError |
If column is not found in reference or comparison dataset |
Source code in zenml/integrations/evidently/steps/evidently_profile.py
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyProfileParameters,
) -> Output( # type:ignore[valid-type]
profile=Profile, dashboard=str
):
"""Main entrypoint for the Evidently categorical target drift detection step.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
profile: Evidently Profile generated for the data drift
dashboard: HTML report extracted from an Evidently Dashboard
generated for the data drift.
Raises:
ValueError: If ignored_cols is an empty list
ValueError: If column is not found in reference or comparison
dataset
"""
data_validator = cast(
EvidentlyDataValidator,
EvidentlyDataValidator.get_active_data_validator(),
)
column_mapping = None
if params.ignored_cols is None:
pass
elif not params.ignored_cols:
raise ValueError(
f"Expects None or list of columns in strings, but got {params.ignored_cols}"
)
elif not (
set(params.ignored_cols).issubset(set(reference_dataset.columns))
) or not (
set(params.ignored_cols).issubset(set(comparison_dataset.columns))
):
raise ValueError(
"Column is not found in reference or comparison datasets"
)
else:
reference_dataset = reference_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
comparison_dataset = comparison_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if params.column_mapping:
column_mapping = (
params.column_mapping.to_evidently_column_mapping()
)
profile, dashboard = data_validator.legacy_data_profiling(
dataset=reference_dataset,
comparison_dataset=comparison_dataset,
profile_list=params.profile_sections,
column_mapping=column_mapping,
verbose_level=params.verbose_level,
profile_options=params.profile_options,
dashboard_options=params.dashboard_options,
)
return [profile, dashboard.html()]
evidently_profile_step(step_name, params)
Shortcut function to create a new instance of the EvidentlyProfileConfig step.
The returned EvidentlyProfileStep can be used in a pipeline to run model drift analyses on two input pd.DataFrame datasets and return the results as an Evidently profile object and a rendered dashboard object.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_name |
str |
The name of the step |
required |
params |
EvidentlyProfileParameters |
The parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
BaseStep |
a EvidentlyProfileStep step instance. |
Source code in zenml/integrations/evidently/steps/evidently_profile.py
def evidently_profile_step(
step_name: str,
params: EvidentlyProfileParameters,
) -> BaseStep:
"""Shortcut function to create a new instance of the EvidentlyProfileConfig step.
The returned EvidentlyProfileStep can be used in a pipeline to
run model drift analyses on two input pd.DataFrame datasets and return the
results as an Evidently profile object and a rendered dashboard object.
Args:
step_name: The name of the step
params: The parameters for the step
Returns:
a EvidentlyProfileStep step instance.
"""
return EvidentlyProfileStep(name=step_name, params=params)
evidently_report
Implementation of the Evidently Report Step.
EvidentlyReportBaseStep
Base implementation for an Evidently Report Step.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlyReportBaseStep:
"""Base implementation for an Evidently Report Step."""
def _run_entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame],
params: EvidentlyReportParameters,
) -> Output( # type:ignore[valid-type]
report_json=str, report_html=str
):
"""Evidently report step for one or two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Raises:
ValueError: If ignored_cols is an empty list
ValueError: If column is not found in reference or comparison
dataset
Returns:
A tuple containing the Evidently report in JSON and HTML
formats.
"""
data_validator = cast(
EvidentlyDataValidator,
EvidentlyDataValidator.get_active_data_validator(),
)
column_mapping = None
if params.ignored_cols:
extra_cols = set(params.ignored_cols) - set(
reference_dataset.columns
)
if extra_cols:
raise ValueError(
f"Columns {extra_cols} configured in the ignored_cols "
"parameter are not found in the reference dataset."
)
reference_dataset = reference_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if comparison_dataset is not None:
extra_cols = set(params.ignored_cols) - set(
comparison_dataset.columns
)
if extra_cols:
raise ValueError(
f"Columns {extra_cols} configured in the ignored_cols "
"parameter are not found in the comparison dataset."
)
comparison_dataset = comparison_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if params.column_mapping:
column_mapping = (
params.column_mapping.to_evidently_column_mapping()
)
report = data_validator.data_profiling(
dataset=reference_dataset,
comparison_dataset=comparison_dataset,
profile_list=params.metrics,
column_mapping=column_mapping,
report_options=params.report_options,
download_nltk_data=params.download_nltk_data,
)
return [report.json(), report.show(mode="inline").data]
EvidentlyReportParameters (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently report step |
metrics |
List[zenml.integrations.evidently.metrics.EvidentlyMetricConfig] |
a list of Evidently metric configurations to use for the report. |
report_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the report and a dictionary of options for the report. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlyReportParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently report step
metrics: a list of Evidently metric configurations to use for the
report.
report_options: a list of tuples containing the name of the report
and a dictionary of options for the report.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
metrics: List[EvidentlyMetricConfig]
report_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_metrics(values)
classmethod
Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyReportParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default metrics added if no metrics were configured. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
EvidentlyReportStep (BaseStep, EvidentlyReportBaseStep)
Implementation for an Evidently Report Step using two datasets.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlyReportStep(BaseStep, EvidentlyReportBaseStep):
"""Implementation for an Evidently Report Step using two datasets."""
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyReportParameters,
) -> Output( # type:ignore[valid-type]
report_json=str, report_html=str
):
"""Evidently report step for two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
A tuple containing the Evidently report in JSON and HTML
formats.
"""
return self._run_entrypoint(
reference_dataset=reference_dataset,
comparison_dataset=comparison_dataset,
params=params,
)
PARAMETERS_CLASS (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently report step |
metrics |
List[zenml.integrations.evidently.metrics.EvidentlyMetricConfig] |
a list of Evidently metric configurations to use for the report. |
report_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the report and a dictionary of options for the report. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlyReportParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently report step
metrics: a list of Evidently metric configurations to use for the
report.
report_options: a list of tuples containing the name of the report
and a dictionary of options for the report.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
metrics: List[EvidentlyMetricConfig]
report_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_metrics(values)
classmethod
Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyReportParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default metrics added if no metrics were configured. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
entrypoint(self, reference_dataset, comparison_dataset, params)
Evidently report step for two datasets.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
reference_dataset |
DataFrame |
a Pandas DataFrame |
required |
comparison_dataset |
DataFrame |
a Pandas DataFrame of new data you wish to compare against the reference data |
required |
params |
EvidentlyReportParameters |
the parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
<zenml.steps.step_output.Output object at 0x7f86052d1b50> |
A tuple containing the Evidently report in JSON and HTML formats. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyReportParameters,
) -> Output( # type:ignore[valid-type]
report_json=str, report_html=str
):
"""Evidently report step for two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
A tuple containing the Evidently report in JSON and HTML
formats.
"""
return self._run_entrypoint(
reference_dataset=reference_dataset,
comparison_dataset=comparison_dataset,
params=params,
)
EvidentlySingleDatasetReportStep (BaseStep, EvidentlyReportBaseStep)
Implementation for an Evidently Report Step using a single dataset.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlySingleDatasetReportStep(BaseStep, EvidentlyReportBaseStep):
"""Implementation for an Evidently Report Step using a single dataset."""
def entrypoint(
self,
dataset: pd.DataFrame,
params: EvidentlyReportParameters,
) -> Output( # type:ignore[valid-type]
report_json=str, report_html=str
):
"""Evidently report step for a single dataset.
Args:
dataset: a Pandas DataFrame
params: the parameters for the step
Returns:
A tuple containing the Evidently report in JSON and HTML
formats.
"""
return self._run_entrypoint(
reference_dataset=dataset, comparison_dataset=None, params=params
)
PARAMETERS_CLASS (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently report step |
metrics |
List[zenml.integrations.evidently.metrics.EvidentlyMetricConfig] |
a list of Evidently metric configurations to use for the report. |
report_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the report and a dictionary of options for the report. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
class EvidentlyReportParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently report step
metrics: a list of Evidently metric configurations to use for the
report.
report_options: a list of tuples containing the name of the report
and a dictionary of options for the report.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
metrics: List[EvidentlyMetricConfig]
report_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_report.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_metrics(values)
classmethod
Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyReportParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default metrics added if no metrics were configured. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
@root_validator(pre=True)
def default_metrics(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently metrics to use if none are provided.
If no metrics are configured, use all available MetricPreset metrics
by default.
Args:
values: The valued configured for the EvidentlyReportParameters
instance.
Returns:
The values with the default metrics added if no metrics were
configured.
"""
if not values.get("metrics"):
values["metrics"] = EvidentlyMetricConfig.default_metrics()
return values
entrypoint(self, dataset, params)
Evidently report step for a single dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
DataFrame |
a Pandas DataFrame |
required |
params |
EvidentlyReportParameters |
the parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
<zenml.steps.step_output.Output object at 0x7f86052d1b80> |
A tuple containing the Evidently report in JSON and HTML formats. |
Source code in zenml/integrations/evidently/steps/evidently_report.py
def entrypoint(
self,
dataset: pd.DataFrame,
params: EvidentlyReportParameters,
) -> Output( # type:ignore[valid-type]
report_json=str, report_html=str
):
"""Evidently report step for a single dataset.
Args:
dataset: a Pandas DataFrame
params: the parameters for the step
Returns:
A tuple containing the Evidently report in JSON and HTML
formats.
"""
return self._run_entrypoint(
reference_dataset=dataset, comparison_dataset=None, params=params
)
evidently_report_step(step_name, params, single_dataset=False, **kwargs)
Create an instance of the Evidently report step.
The returned step can be used in a pipeline to generate an Evidently report on one or two input pd.DataFrame datasets and return the results as an Evidently Report object in dictionary and HTML formats.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_name |
str |
The name of the step |
required |
params |
EvidentlyReportParameters |
The parameters for the step |
required |
single_dataset |
bool |
Whether to use a single dataset or two datasets as input. |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the step constructor. |
{} |
Returns:
| Type | Description |
|---|---|
BaseStep |
a Evidently report step instance |
Source code in zenml/integrations/evidently/steps/evidently_report.py
def evidently_report_step(
step_name: str,
params: EvidentlyReportParameters,
single_dataset: bool = False,
**kwargs: Any,
) -> BaseStep:
"""Create an instance of the Evidently report step.
The returned step can be used in a pipeline to generate an Evidently report
on one or two input pd.DataFrame datasets and return the results as an
Evidently Report object in dictionary and HTML formats.
Args:
step_name: The name of the step
params: The parameters for the step
single_dataset: Whether to use a single dataset or two datasets
as input.
**kwargs: Additional keyword arguments to pass to the step constructor.
Returns:
a Evidently report step instance
"""
if single_dataset:
return EvidentlySingleDatasetReportStep(
name=step_name, params=params, **kwargs
)
return EvidentlyReportStep(name=step_name, params=params, **kwargs)
evidently_test
Implementation of the Evidently Test Step.
EvidentlyBaseTestStep
Base implementation for an Evidently Test Step.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlyBaseTestStep:
"""Base implementation for an Evidently Test Step."""
def _run_entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame],
params: EvidentlyTestParameters,
) -> Output( # type:ignore[valid-type]
test_json=str, test_html=str
):
"""Evidently test step for one or two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Raises:
ValueError: If ignored_cols is an empty list
ValueError: If column is not found in reference or comparison
dataset
Returns:
A tuple containing the TestSuite in JSON and HTML formats.
"""
data_validator = cast(
EvidentlyDataValidator,
EvidentlyDataValidator.get_active_data_validator(),
)
column_mapping = None
if params.ignored_cols:
extra_cols = set(params.ignored_cols) - set(
reference_dataset.columns
)
if extra_cols:
raise ValueError(
f"Columns {extra_cols} configured in the ignored_cols "
"parameter are not found in the reference dataset."
)
reference_dataset = reference_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if comparison_dataset is not None:
extra_cols = set(params.ignored_cols) - set(
comparison_dataset.columns
)
if extra_cols:
raise ValueError(
f"Columns {extra_cols} configured in the ignored_cols "
"parameter are not found in the comparison dataset."
)
comparison_dataset = comparison_dataset.drop(
labels=list(params.ignored_cols), axis=1
)
if params.column_mapping:
column_mapping = (
params.column_mapping.to_evidently_column_mapping()
)
test_suite = data_validator.data_validation(
dataset=reference_dataset,
comparison_dataset=comparison_dataset,
check_list=params.tests,
column_mapping=column_mapping,
test_options=params.test_options,
download_nltk_data=params.download_nltk_data,
)
return [test_suite.json(), test_suite.show(mode="inline").data]
EvidentlySingleDatasetTestStep (BaseStep, EvidentlyBaseTestStep)
Implementation for an Evidently Test Step using a single dataset.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlySingleDatasetTestStep(BaseStep, EvidentlyBaseTestStep):
"""Implementation for an Evidently Test Step using a single dataset."""
def entrypoint(
self,
dataset: pd.DataFrame,
params: EvidentlyTestParameters,
) -> Output( # type:ignore[valid-type]
test_json=str, test_html=str
):
"""Evidently test step for a single dataset.
Args:
dataset: a Pandas DataFrame
params: the parameters for the step
Returns:
A tuple containing the Evidently TestSuite in JSON and HTML formats.
"""
return self._run_entrypoint(
reference_dataset=dataset, comparison_dataset=None, params=params
)
PARAMETERS_CLASS (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently profile step |
tests |
List[zenml.integrations.evidently.tests.EvidentlyTestConfig] |
a list of Evidently test configuration to use for the test suite. |
test_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the test and a dictionary of options for the test. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlyTestParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently profile step
tests: a list of Evidently test configuration to use for the test suite.
test_options: a list of tuples containing the name of the test
and a dictionary of options for the test.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
tests: List[EvidentlyTestConfig]
test_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_tests(values)
classmethod
Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyTestParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default tests added if no tests were configured. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
entrypoint(self, dataset, params)
Evidently test step for a single dataset.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
DataFrame |
a Pandas DataFrame |
required |
params |
EvidentlyTestParameters |
the parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
<zenml.steps.step_output.Output object at 0x7f86052e4c10> |
A tuple containing the Evidently TestSuite in JSON and HTML formats. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
def entrypoint(
self,
dataset: pd.DataFrame,
params: EvidentlyTestParameters,
) -> Output( # type:ignore[valid-type]
test_json=str, test_html=str
):
"""Evidently test step for a single dataset.
Args:
dataset: a Pandas DataFrame
params: the parameters for the step
Returns:
A tuple containing the Evidently TestSuite in JSON and HTML formats.
"""
return self._run_entrypoint(
reference_dataset=dataset, comparison_dataset=None, params=params
)
EvidentlyTestParameters (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently profile step |
tests |
List[zenml.integrations.evidently.tests.EvidentlyTestConfig] |
a list of Evidently test configuration to use for the test suite. |
test_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the test and a dictionary of options for the test. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlyTestParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently profile step
tests: a list of Evidently test configuration to use for the test suite.
test_options: a list of tuples containing the name of the test
and a dictionary of options for the test.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
tests: List[EvidentlyTestConfig]
test_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_tests(values)
classmethod
Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyTestParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default tests added if no tests were configured. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
EvidentlyTestStep (BaseStep, EvidentlyBaseTestStep)
Implementation for an Evidently Test Step using two datasets.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlyTestStep(BaseStep, EvidentlyBaseTestStep):
"""Implementation for an Evidently Test Step using two datasets."""
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyTestParameters,
) -> Output( # type:ignore[valid-type]
test_json=str, test_html=str
):
"""Evidently test step for two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
A tuple containing the Evidently TestSuite in JSON and HTML formats.
"""
return self._run_entrypoint(
reference_dataset=reference_dataset,
comparison_dataset=comparison_dataset,
params=params,
)
PARAMETERS_CLASS (BaseParameters)
pydantic-model
Parameters class for Evidently profile steps.
Attributes:
| Name | Type | Description |
|---|---|---|
column_mapping |
Optional[zenml.integrations.evidently.column_mapping.EvidentlyColumnMapping] |
properties of the DataFrame columns used |
ignored_cols |
Optional[List[str]] |
columns to ignore during the Evidently profile step |
tests |
List[zenml.integrations.evidently.tests.EvidentlyTestConfig] |
a list of Evidently test configuration to use for the test suite. |
test_options |
Sequence[Tuple[str, Dict[str, Any]]] |
a list of tuples containing the name of the test and a dictionary of options for the test. |
download_nltk_data |
bool |
whether to download the NLTK data for the report step. Defaults to False. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
class EvidentlyTestParameters(BaseParameters):
"""Parameters class for Evidently profile steps.
Attributes:
column_mapping: properties of the DataFrame columns used
ignored_cols: columns to ignore during the Evidently profile step
tests: a list of Evidently test configuration to use for the test suite.
test_options: a list of tuples containing the name of the test
and a dictionary of options for the test.
download_nltk_data: whether to download the NLTK data for the report
step. Defaults to False.
"""
column_mapping: Optional[EvidentlyColumnMapping] = None
ignored_cols: Optional[List[str]] = None
tests: List[EvidentlyTestConfig]
test_options: Sequence[Tuple[str, Dict[str, Any]]] = Field(
default_factory=list
)
download_nltk_data: bool = False
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/steps/evidently_test.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_tests(values)
classmethod
Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests by default.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
The valued configured for the EvidentlyTestParameters instance. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values with the default tests added if no tests were configured. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
@root_validator(pre=True)
def default_tests(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Default Evidently tests to use if none are provided.
If no tests are configured, use all available TestPreset tests
by default.
Args:
values: The valued configured for the EvidentlyTestParameters
instance.
Returns:
The values with the default tests added if no tests were
configured.
"""
if not values.get("tests"):
values["tests"] = EvidentlyTestConfig.default_tests()
return values
entrypoint(self, reference_dataset, comparison_dataset, params)
Evidently test step for two datasets.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
reference_dataset |
DataFrame |
a Pandas DataFrame |
required |
comparison_dataset |
DataFrame |
a Pandas DataFrame of new data you wish to compare against the reference data |
required |
params |
EvidentlyTestParameters |
the parameters for the step |
required |
Returns:
| Type | Description |
|---|---|
<zenml.steps.step_output.Output object at 0x7f86052e4e80> |
A tuple containing the Evidently TestSuite in JSON and HTML formats. |
Source code in zenml/integrations/evidently/steps/evidently_test.py
def entrypoint(
self,
reference_dataset: pd.DataFrame,
comparison_dataset: pd.DataFrame,
params: EvidentlyTestParameters,
) -> Output( # type:ignore[valid-type]
test_json=str, test_html=str
):
"""Evidently test step for two datasets.
Args:
reference_dataset: a Pandas DataFrame
comparison_dataset: a Pandas DataFrame of new data you wish to
compare against the reference data
params: the parameters for the step
Returns:
A tuple containing the Evidently TestSuite in JSON and HTML formats.
"""
return self._run_entrypoint(
reference_dataset=reference_dataset,
comparison_dataset=comparison_dataset,
params=params,
)
evidently_test_step(step_name, params, single_dataset=False, **kwargs)
Create an instance of the Evidently test step.
The returned step can be used in a pipeline to run an Evidently test suite on one or two input pd.DataFrame datasets and return the results as an Evidently TestSuite object in JSON and HTML formats.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_name |
str |
The name of the step |
required |
params |
EvidentlyTestParameters |
The parameters for the step |
required |
single_dataset |
bool |
Whether to use a single dataset or two datasets as input. |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the step constructor. |
{} |
Returns:
| Type | Description |
|---|---|
BaseStep |
a Evidently test step instance |
Source code in zenml/integrations/evidently/steps/evidently_test.py
def evidently_test_step(
step_name: str,
params: EvidentlyTestParameters,
single_dataset: bool = False,
**kwargs: Any,
) -> BaseStep:
"""Create an instance of the Evidently test step.
The returned step can be used in a pipeline to run an Evidently test suite
on one or two input pd.DataFrame datasets and return the results as an
Evidently TestSuite object in JSON and HTML formats.
Args:
step_name: The name of the step
params: The parameters for the step
single_dataset: Whether to use a single dataset or two datasets
as input.
**kwargs: Additional keyword arguments to pass to the step constructor.
Returns:
a Evidently test step instance
"""
if single_dataset:
return EvidentlySingleDatasetTestStep(
name=step_name, params=params, **kwargs
)
return EvidentlyTestStep(name=step_name, params=params, **kwargs)
tests
ZenML declarative representation of Evidently Tests.
EvidentlyTestConfig (BaseModel)
pydantic-model
Declarative Evidently Test configuration.
This is a declarative representation of the configuration that goes into an Evidently Test, TestPreset or Test generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Test from it.
This representation covers all 3 possible ways of configuring an Evidently Test or Test-like object that can later be used in an Evidently TestSuite:
- A Test (derived from the Test class).
- A TestPreset (derived from the TestPreset class).
- A column Test generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Test or Test-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Test classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.
Attributes:
| Name | Type | Description |
|---|---|---|
class_path |
str |
The full class path of the Evidently Test class. |
parameters |
Dict[str, Any] |
The parameters of the Evidently Test. |
is_generator |
bool |
Whether this is an Evidently column Test generator. |
columns |
Union[str, List[str]] |
The columns that the Evidently column Test generator is
applied to. Only used if |
Source code in zenml/integrations/evidently/tests.py
class EvidentlyTestConfig(BaseModel):
"""Declarative Evidently Test configuration.
This is a declarative representation of the configuration that goes into an
Evidently Test, TestPreset or Test generator instance. We need this to
be able to store the configuration as part of a ZenML step parameter and
later instantiate the Evidently Test from it.
This representation covers all 3 possible ways of configuring an Evidently
Test or Test-like object that can later be used in an Evidently TestSuite:
1. A Test (derived from the Test class).
2. A TestPreset (derived from the TestPreset class).
3. A column Test generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Test or Test-like
object to this class and have it automatically derive the configuration used
to instantiate it. Unfortunately, this is not possible because the Evidently
Test classes are not designed in a way that allows us to extract the
constructor parameters from them in a generic way.
Attributes:
class_path: The full class path of the Evidently Test class.
parameters: The parameters of the Evidently Test.
is_generator: Whether this is an Evidently column Test generator.
columns: The columns that the Evidently column Test generator is
applied to. Only used if `generator` is True.
"""
class_path: str
parameters: Dict[str, Any] = Field(default_factory=dict)
is_generator: bool = False
columns: Optional[Union[str, List[str]]] = None
@staticmethod
def get_test_class(test_name: str) -> Union[Test, TestPreset]:
"""Get the Evidently test or test preset class from a string.
Args:
test_name: The test or test preset class or full class
path.
Returns:
The Evidently test or test preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
test or test preset class.
"""
# First, try to interpret the test name as a full class path.
if "." in test_name:
try:
test_class = import_class_by_path(test_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`: {str(e)}"
)
else:
# Next, try to interpret the test as a Test or TestPreset
# class name
if hasattr(tests, test_name):
test_class = getattr(tests, test_name)
elif hasattr(test_preset, test_name):
test_class = getattr(test_preset, test_name)
else:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`"
)
if not issubclass(test_class, (Test, TestPreset)):
raise ValueError(
f"Class `{test_name}` is not a valid Evidently "
f"Test or TestPreset."
)
return test_class
@classmethod
def test_generator(
cls,
test: Union[Type[Test], str],
columns: Optional[Union[str, List[str]]] = None,
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Test generator.
The `columns`, `parameters` arguments will be
passed to the Evidently `generate_column_tests` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
```
Args:
test: The Evidently Test class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_tests` function
takes.
parameters: Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
`generate_column_tests` function.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test generator configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(f"Invalid Evidently Test class: {test}")
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
@classmethod
def test(
cls,
test: Union[Type[Test], Type[TestPreset], str],
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the
configuration of an Evidently Test.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
```
Args:
test: The Evidently Test or TestPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Test or TestPreset.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(
f"Invalid Evidently Test or TestPreset class: {test}"
)
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
@classmethod
def default_tests(cls) -> List["EvidentlyTestConfig"]:
"""Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases
where no tests are explicitly configured for a data validator.
All available Evidently TestPreset classes are used.
Returns:
A list of EvidentlyTestConfig objects to use as default tests.
"""
return [
cls.test(test=test_preset_class_name)
for test_preset_class_name in test_preset.__all__
]
def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
"""Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test
generator instance from its declarative representation.
Returns:
The Evidently Test, TestPreset or test generator object.
Raises:
ValueError: If the Evidently Test, TestPreset or column test
generator could not be instantiated.
"""
test_class = self.get_test_class(self.class_path)
if self.is_generator:
try:
return generate_column_tests(
test_class=test_class,
columns=self.columns,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Test generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return test_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Test or TestPreset "
f"`{self.class_path}`: {str(e)}"
)
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/tests.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_tests()
classmethod
Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases where no tests are explicitly configured for a data validator. All available Evidently TestPreset classes are used.
Returns:
| Type | Description |
|---|---|
List[EvidentlyTestConfig] |
A list of EvidentlyTestConfig objects to use as default tests. |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def default_tests(cls) -> List["EvidentlyTestConfig"]:
"""Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases
where no tests are explicitly configured for a data validator.
All available Evidently TestPreset classes are used.
Returns:
A list of EvidentlyTestConfig objects to use as default tests.
"""
return [
cls.test(test=test_preset_class_name)
for test_preset_class_name in test_preset.__all__
]
get_test_class(test_name)
staticmethod
Get the Evidently test or test preset class from a string.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test_name |
str |
The test or test preset class or full class path. |
required |
Returns:
| Type | Description |
|---|---|
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset] |
The Evidently test or test preset class. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the name cannot be converted into a valid Evidently test or test preset class. |
Source code in zenml/integrations/evidently/tests.py
@staticmethod
def get_test_class(test_name: str) -> Union[Test, TestPreset]:
"""Get the Evidently test or test preset class from a string.
Args:
test_name: The test or test preset class or full class
path.
Returns:
The Evidently test or test preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
test or test preset class.
"""
# First, try to interpret the test name as a full class path.
if "." in test_name:
try:
test_class = import_class_by_path(test_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`: {str(e)}"
)
else:
# Next, try to interpret the test as a Test or TestPreset
# class name
if hasattr(tests, test_name):
test_class = getattr(tests, test_name)
elif hasattr(test_preset, test_name):
test_class = getattr(test_preset, test_name)
else:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`"
)
if not issubclass(test_class, (Test, TestPreset)):
raise ValueError(
f"Class `{test_name}` is not a valid Evidently "
f"Test or TestPreset."
)
return test_class
test(test, **parameters)
classmethod
Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the configuration of an Evidently Test.
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test |
Union[Type[evidently.tests.base_test.Test], Type[evidently.test_preset.test_preset.TestPreset], str] |
The Evidently Test or TestPreset class, class name or class path. |
required |
parameters |
Any |
Additional optional parameters needed to instantiate the Evidently Test or TestPreset. |
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyTestConfig |
The EvidentlyTest declarative representation of the Evidently Test configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def test(
cls,
test: Union[Type[Test], Type[TestPreset], str],
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the
configuration of an Evidently Test.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
```
Args:
test: The Evidently Test or TestPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Test or TestPreset.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(
f"Invalid Evidently Test or TestPreset class: {test}"
)
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
test_generator(test, columns=None, **parameters)
classmethod
Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the configuration of an Evidently column Test generator.
The columns, parameters arguments will be
passed to the Evidently generate_column_tests function:
- if
columnsis a list, it is interpreted as a list of column names. - if
columnsis a string, it can be one of values:- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test |
Union[Type[evidently.tests.base_test.Test], str] |
The Evidently Test class, class name or class path to use for the generator. |
required |
columns |
Union[str, List[str]] |
The columns to apply the generator to. Takes the same
values that the Evidently |
None |
parameters |
Any |
Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
|
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyTestConfig |
The EvidentlyTest declarative representation of the Evidently Test generator configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def test_generator(
cls,
test: Union[Type[Test], str],
columns: Optional[Union[str, List[str]]] = None,
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Test generator.
The `columns`, `parameters` arguments will be
passed to the Evidently `generate_column_tests` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
```
Args:
test: The Evidently Test class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_tests` function
takes.
parameters: Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
`generate_column_tests` function.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test generator configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(f"Invalid Evidently Test class: {test}")
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
to_evidently_test(self)
Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test generator instance from its declarative representation.
Returns:
| Type | Description |
|---|---|
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset, evidently.utils.generators.BaseGenerator] |
The Evidently Test, TestPreset or test generator object. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the Evidently Test, TestPreset or column test generator could not be instantiated. |
Source code in zenml/integrations/evidently/tests.py
def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
"""Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test
generator instance from its declarative representation.
Returns:
The Evidently Test, TestPreset or test generator object.
Raises:
ValueError: If the Evidently Test, TestPreset or column test
generator could not be instantiated.
"""
test_class = self.get_test_class(self.class_path)
if self.is_generator:
try:
return generate_column_tests(
test_class=test_class,
columns=self.columns,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Test generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return test_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Test or TestPreset "
f"`{self.class_path}`: {str(e)}"
)
visualizers
special
Initialization for Evidently visualizer.
evidently_visualizer
Implementation of the Evidently visualizer.
EvidentlyVisualizer (BaseVisualizer)
The implementation of an Evidently Visualizer.
Source code in zenml/integrations/evidently/visualizers/evidently_visualizer.py
class EvidentlyVisualizer(BaseVisualizer):
"""The implementation of an Evidently Visualizer."""
@abstractmethod
def visualize(self, object: StepView, *args: Any, **kwargs: Any) -> None:
"""Method to visualize components.
Args:
object: StepView fetched from run.get_step().
*args: Additional arguments.
**kwargs: Additional keyword arguments.
"""
artifact: Optional[str] = None
for artifact_view in object.outputs.values():
# filter out anything but data artifacts
if (
artifact_view.type == ArtifactType.DATA
and artifact_view.data_type == "builtins.str"
and artifact_view.name
in ["report_html", "test_html", "dashboard"]
):
artifact = cast(str, artifact_view.read())
# Display the last artifact
if artifact:
self.generate_facet(artifact)
def generate_facet(self, html_: str) -> None:
"""Generate a Facet Overview.
Args:
html_: HTML represented as a string.
"""
if Environment.in_notebook() or Environment.in_google_colab():
from IPython.core.display import HTML, display
display(HTML(html_))
else:
logger.warning(
"The magic functions are only usable in a Jupyter notebook."
)
with tempfile.NamedTemporaryFile(
mode="w", delete=False, suffix=".html", encoding="utf-8"
) as f:
f.write(html_)
url = f"file:///{f.name}"
logger.info("Opening %s in a new browser.." % f.name)
webbrowser.open(url, new=2)
generate_facet(self, html_)
Generate a Facet Overview.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
html_ |
str |
HTML represented as a string. |
required |
Source code in zenml/integrations/evidently/visualizers/evidently_visualizer.py
def generate_facet(self, html_: str) -> None:
"""Generate a Facet Overview.
Args:
html_: HTML represented as a string.
"""
if Environment.in_notebook() or Environment.in_google_colab():
from IPython.core.display import HTML, display
display(HTML(html_))
else:
logger.warning(
"The magic functions are only usable in a Jupyter notebook."
)
with tempfile.NamedTemporaryFile(
mode="w", delete=False, suffix=".html", encoding="utf-8"
) as f:
f.write(html_)
url = f"file:///{f.name}"
logger.info("Opening %s in a new browser.." % f.name)
webbrowser.open(url, new=2)
visualize(self, object, *args, **kwargs)
Method to visualize components.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
object |
StepView |
StepView fetched from run.get_step(). |
required |
*args |
Any |
Additional arguments. |
() |
**kwargs |
Any |
Additional keyword arguments. |
{} |
Source code in zenml/integrations/evidently/visualizers/evidently_visualizer.py
@abstractmethod
def visualize(self, object: StepView, *args: Any, **kwargs: Any) -> None:
"""Method to visualize components.
Args:
object: StepView fetched from run.get_step().
*args: Additional arguments.
**kwargs: Additional keyword arguments.
"""
artifact: Optional[str] = None
for artifact_view in object.outputs.values():
# filter out anything but data artifacts
if (
artifact_view.type == ArtifactType.DATA
and artifact_view.data_type == "builtins.str"
and artifact_view.name
in ["report_html", "test_html", "dashboard"]
):
artifact = cast(str, artifact_view.read())
# Display the last artifact
if artifact:
self.generate_facet(artifact)