Changeset View
Changeset View
Standalone View
Standalone View
python_modules/libraries/dagster-pandas/dagster_pandas/constraints.py
| import sys | import sys | ||||
| from collections import defaultdict | from collections import defaultdict | ||||
| from datetime import datetime | from datetime import datetime | ||||
| from functools import wraps | from functools import wraps | ||||
| import pandas as pd | import pandas as pd | ||||
| from dagster import DagsterType, EventMetadataEntry, TypeCheck, check | from dagster import DagsterType, EventMetadataEntry, TypeCheck, check | ||||
| from dagster.utils.backcompat import experimental_class_warning | from dagster.utils.backcompat import experimental | ||||
| from pandas import DataFrame | from pandas import DataFrame | ||||
| class ConstraintViolationException(Exception): | class ConstraintViolationException(Exception): | ||||
| """Indicates that a constraint has been violated.""" | """Indicates that a constraint has been violated.""" | ||||
| class ConstraintWithMetadataException(Exception): | class ConstraintWithMetadataException(Exception): | ||||
| ▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines | class Constraint: | ||||
| """ | """ | ||||
| def __init__(self, error_description=None, markdown_description=None): | def __init__(self, error_description=None, markdown_description=None): | ||||
| self.name = self.__class__.__name__ | self.name = self.__class__.__name__ | ||||
| self.markdown_description = check.str_param(markdown_description, "markdown_description") | self.markdown_description = check.str_param(markdown_description, "markdown_description") | ||||
| self.error_description = check.str_param(error_description, "error_description") | self.error_description = check.str_param(error_description, "error_description") | ||||
| @experimental | |||||
| class ConstraintWithMetadata: | class ConstraintWithMetadata: | ||||
| """ | """ | ||||
| This class defines a base constraint over pandas DFs with organized metadata | This class defines a base constraint over pandas DFs with organized metadata | ||||
| args: | args: | ||||
| description (str): description of the constraint | description (str): description of the constraint | ||||
| validation_fn (Callable[[DataFrame], Tuple[bool, dict[str, Union[dict,list, str, set]]]]: | validation_fn (Callable[[DataFrame], Tuple[bool, dict[str, Union[dict,list, str, set]]]]: | ||||
| the validation function to run over inputted data | the validation function to run over inputted data | ||||
| This function should return a tuple of a boolean for success or failure, and a dict containing | This function should return a tuple of a boolean for success or failure, and a dict containing | ||||
| metadata about the test -- this metadata will be passed to the resulting exception if validation | metadata about the test -- this metadata will be passed to the resulting exception if validation | ||||
| fails. | fails. | ||||
| resulting_exception (ConstraintWithMetadataException): what response a failed typecheck should induce | resulting_exception (ConstraintWithMetadataException): what response a failed typecheck should induce | ||||
| raise_or_typecheck (Optional[bool]): whether to raise an exception (if set to True) or emit a failed typecheck event | raise_or_typecheck (Optional[bool]): whether to raise an exception (if set to True) or emit a failed typecheck event | ||||
| (if set to False) when validation fails | (if set to False) when validation fails | ||||
| name (Optional[str]): what to call the constraint, defaults to the class name. | name (Optional[str]): what to call the constraint, defaults to the class name. | ||||
| """ | """ | ||||
| # TODO: validation_fn returning metadata is sorta broken. maybe have it yield typecheck events and grab metadata? | # TODO: validation_fn returning metadata is sorta broken. maybe have it yield typecheck events and grab metadata? | ||||
| def __init__( | def __init__( | ||||
| self, description, validation_fn, resulting_exception, raise_or_typecheck=True, name=None | self, description, validation_fn, resulting_exception, raise_or_typecheck=True, name=None | ||||
| ): | ): | ||||
| experimental_class_warning(self.__class__.__name__) | |||||
| if name is None: | if name is None: | ||||
| self.name = self.__class__.__name__ | self.name = self.__class__.__name__ | ||||
| else: | else: | ||||
| self.name = name | self.name = name | ||||
| self.description = description | self.description = description | ||||
| # should return a tuple of (bool, and either an empty dict or a dict of extra params) | # should return a tuple of (bool, and either an empty dict or a dict of extra params) | ||||
| self.validation_fn = validation_fn | self.validation_fn = validation_fn | ||||
| self.resulting_exception = resulting_exception | self.resulting_exception = resulting_exception | ||||
| ▲ Show 20 Lines • Show All 966 Lines • Show Last 20 Lines | |||||