Configuring the Test Suite
All of the tests Robust Intelligence runs are easily configurable via the "test_suite_config" key in the test run config. The "test_suite_config" key expects a dictionary as its value, the configuration options for which are presented below.
Global Configuration Options
This dictionary contains several global configuration options, which, if specified, will apply to all relevant tests. All of these default to null, which means Robust Intelligence will rely on the specific test configuration to provide this value. These are:
global_test_sensitivity: str, default ="TEST_SENSITIVITY_DEFAULT"The global setting for test sensitivity to be applied for all tests. The options are"TEST_SENSITIVITY_LESS_SENSITIVE","TEST_SENSITIVITY_DEFAULT", and"TEST_SENSITIVITY_MORE_SENSITIVE".global_exclude_columns: Optional[List[str]], default =nullColumn names in the dataset to exclude from all tests.
custom_tests: List[dict], default = [] Specification for custom tests. Robust Intelligence also offers the ability to add user-specified custom tests to the test suite. This is useful for adding testing for your models that is not covered by the standard tests, and is available as a beta feature. Contact RI support for help setting up custom tests for your Robust Intelligence installation.individual_tests_config: Optional[dict], default =nullSpecification for configurations on individual tests run by Robust Intelligence. Each test’s configuration will be specified under a unique key within"individual_tests_config"; for example, the Correlation Label Drift test will be configured under the"correlation_label_drift"key. See below for an example.
To use a configuration JSON object, load the configuration JSON into Python as a dictionary, for example by using json.load. Specify the dictionary as the value of the "test_suite_config" key in the test run configuration.
Test Suite Configuration Template
{
"custom_tests": null,
"global_test_sensitivity": 2,
"global_exclude_columns": null,
"individual_tests_config": {
"avg_confidence": {
"run": true,
"severity_thresholds": null
},
"atc": {
"run": true,
"severity_thresholds": null
},
"calibration_comparison": {
"run": true,
"num_bins": 10,
"severity_thresholds": null
},
"out_of_range_substitution": {
"exclude_columns": [],
"run": false,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
},
"std_factor": 3.0
},
"outlier_substitution": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
},
"bound_multiplier": 3.0
},
"feature_type_change": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"empty_string_substitution": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"req_characters_deletion": {
"column_specific_params": {},
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"unseen_categorical_substitution": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"null_substitution": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"capitalization_change": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"ignore_errors": false,
"performance_thresholds": null,
"num_samples_to_simulate": 100
}
},
"text_identity": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_upper_case": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_lower_case": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_remove_special_chars": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_ascii": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_char_substitution_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_char_deletion_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_char_insertion_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_char_swap_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_keyboard_aug_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_common_misspelling_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"aug_min": 1,
"aug_max": null,
"aug_p": 0.05
}
},
"text_ocr_error_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"min_char": 2,
"aug_char_max": 1,
"aug_char_min": 1,
"aug_char_p": 0.03,
"aug_word_min": 1,
"aug_word_max": null,
"aug_word_p": 0.05
}
},
"text_synonym_swap_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"aug_min": 1,
"aug_max": null,
"aug_p": 0.05
}
},
"text_lm_word_substitution_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"aug_min": 1,
"aug_max": null,
"aug_p": 0.05,
"model_path": "distilbert-base-cased"
}
},
"text_lm_word_insertion_attack": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"attack_params": {
"aug_min": 1,
"aug_max": null,
"aug_p": 0.05,
"model_path": "distilbert-base-cased"
}
},
"entity_lower_case": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_upper_case": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_ampersand": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_abbreviation": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_white_space_special_chars": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_ascii": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_remove_special_chars": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_swap_seen": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"entity_swap_unseen": {
"severity_thresholds": null,
"sample_size": 20,
"run": true
},
"gaussian_blur": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"kernel_size": [
19,
19
],
"sigma": 5.0
},
"motion_blur": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"kernel_size": 13
},
"color_jitter": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"brightness": 0.1,
"contrast": 0.1,
"saturation": 0.1,
"hue": 0.1
},
"gaussian_noise": {
"severity_thresholds": [
0.01,
0.25
],
"sample_size": 50,
"run": true,
"mean": 0.0,
"sigma": 0.03
},
"vertical_flip": {
"severity_thresholds": null,
"sample_size": 50,
"run": true
},
"horizontal_flip": {
"severity_thresholds": null,
"sample_size": 50,
"run": true
},
"randomize_pixels_with_mask": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"probability": 0.1
},
"contrast_increase": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"percent": 0.5
},
"contrast_decrease": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"percent": 0.5
},
"add_rain": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"alpha": 0.5
},
"add_snow": {
"severity_thresholds": null,
"sample_size": 50,
"run": true,
"alpha": 0.5
},
"vulnerability": {
"exclude_columns": [],
"run": false,
"severity_level_thresholds": null,
"sample_size": 10,
"search_count": 10
},
"sensitivity": {
"exclude_columns": [],
"run": true,
"severity_level_thresholds": null,
"linf_constraint": 0.01,
"sample_size": 10
},
"multi_feat_vulnerability": {
"exclude_columns": [],
"run": false,
"severity_level_thresholds": null,
"l0_constraint": 3,
"sample_size": 10,
"search_count": 10
},
"multi_feat_sensitivity": {
"exclude_columns": [],
"run": true,
"severity_level_thresholds": null,
"l0_constraint": 3,
"sample_size": 10,
"linf_constraint": 0.01
},
"tabular_hopskipjump_attack": {
"exclude_columns": [],
"severity_thresholds": null,
"run": false,
"sample_size": 10,
"attack_params": {
"targeted": false,
"norm": 2,
"max_iter": 50,
"max_eval": 1000,
"init_eval": 100,
"init_size": 100
}
},
"images_hopskipjump_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"targeted": false,
"norm": 2,
"max_iter": 30,
"max_eval": 50,
"init_eval": 20,
"init_size": 20
},
"run": true
},
"square_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"norm": "inf",
"max_iter": 100,
"eps": 0.02,
"p_init": 0.25,
"nb_restarts": 5
},
"run": true
},
"pixel_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"th": 10,
"max_iter": 5,
"targeted": false
},
"run": true
},
"text_invisible_chars_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 200,
"population_size": 15,
"max_unsuccessful_iters": 2,
"max_consecutive_unsuccessful": null
},
"mutator_params": {
"aug_char_p": 0.01
},
"max_str_len": 2000,
"run": true
},
"text_deletion_chars_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 200,
"population_size": 15,
"max_unsuccessful_iters": 2,
"max_consecutive_unsuccessful": null
},
"mutator_params": {
"aug_char_p": 0.01
},
"max_str_len": 2000,
"run": true
},
"text_intentional_homoglyph_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 200,
"population_size": 15,
"max_unsuccessful_iters": 2,
"max_consecutive_unsuccessful": null
},
"mutator_params": {
"aug_char_p": 0.01
},
"max_str_len": 2000,
"run": true
},
"text_confusable_homoglyph_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 200,
"population_size": 15,
"max_unsuccessful_iters": 2,
"max_consecutive_unsuccessful": null
},
"mutator_params": {
"aug_char_p": 0.01
},
"max_str_len": 2000,
"run": true
},
"text_hotflip_attack": {
"severity_thresholds": null,
"sample_size": 5,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 200,
"population_size": 15,
"max_unsuccessful_iters": 2,
"max_consecutive_unsuccessful": null
},
"mutator_params": {},
"max_str_len": 2000,
"run": true,
"constraint_params": {
"stopwords": [],
"include_default_stopwords": true,
"max_tokens_perturbed": 3,
"min_cos_sim": 0.2
}
},
"universal_triggers_attack": {
"severity_thresholds": null,
"sample_size": 3,
"attack_params": {
"max_iterations": 4,
"target_score": 0.0,
"max_queries": 500,
"population_size": 2,
"max_unsuccessful_iters": 200,
"max_consecutive_unsuccessful": 10
},
"mutator_params": {
"prefix_len": 10,
"aug_word_p": 0.2
},
"max_str_len": 2000,
"run": true,
"batch_size": 10
},
"numeric_outlier": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
},
"bound_multiplier": 3.0
},
"unseen_categorical": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"rare_categories": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
},
"include_columns": [],
"min_num_occurrences": 0,
"min_pct_occurrences": 0.0,
"min_ratio_rel_uniform": 0.005
},
"out_of_range": {
"exclude_columns": [],
"run": false,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
},
"std_factor": 3.0
},
"req_characters": {
"column_specific_params": {},
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"inconsistencies": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
},
"freq_ratio_threshold": 0.02,
"min_correlation": 0.1,
"max_pairwise_tests": 200,
"max_unique_pairs_for_firewall": 15
},
"capitalization": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"empty_string": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"embedding_anomalies": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
},
"distance_quantile": 0.995
},
"feature_type_check": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"null_check": {
"exclude_columns": [],
"run": true,
"performance_change_config": {
"performance_thresholds": null,
"min_num_samples": 1
}
},
"correlation_feat_drift": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"correlation_drift_thresholds": null,
"p_value_threshold": 0.05,
"min_correlation": 0.1,
"max_pairwise_tests": 200
},
"correlation_label_drift": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"correlation_drift_thresholds": null,
"p_value_threshold": 0.05
},
"mutual_information_feat_drift": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"mutual_information_thresholds": null,
"min_mutual_information": 0.1,
"max_pairwise_tests": 200,
"min_sample_size": 100
},
"mutual_information_label_drift": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"mutual_information_thresholds": null
},
"categorical_label_drift": {
"run": true,
"drift_statistic": "Population Stability Index",
"params": {
"psi_thresholds": null
}
},
"multiclass_pred_label_drift": {
"run": true,
"drift_statistic": "Population Stability Index",
"params": {
"psi_thresholds": null
}
},
"regression_label_drift": {
"run": true,
"drift_statistic": "Population Stability Index",
"params": {
"psi_thresholds": null,
"num_bins": 100
}
},
"feature_drift": {
"exclude_columns": [],
"run": true,
"min_sample_size": 100,
"performance_change_thresholds": null,
"numeric_feature_params": {
"drift_statistic": "Population Stability Index",
"min_num_quantiles": 1000,
"num_bins": 100,
"psi_thresholds": [
0.2,
0.6
],
"kl_divergence_thresholds": [
0.1,
0.2
],
"js_divergence_thresholds": [
0.05,
0.1
],
"ks_test_p_value_threshold": 0.05,
"ignore_nans": true
},
"categorical_feature_params": {
"drift_statistic": "Population Stability Index",
"chi_squared_p_value_threshold": 0.05,
"psi_thresholds": [
0.2,
0.6
],
"max_sample_size": null,
"ignore_nans": true
}
},
"prediction_drift": {
"run": true,
"drift_statistic": "Population Stability Index",
"params": {
"run": true,
"min_sample_size": 100,
"min_num_quantiles": 1000,
"psi_thresholds": null,
"num_bins": 100
}
},
"embedding_drift": {
"run": true,
"min_sample_size": 100,
"performance_change_thresholds": null,
"drift_statistic": "euclidean_distance",
"params": {
"distance_thresholds": [
0.1,
0.2
],
"normalize": true
}
},
"null_proportion": {
"exclude_columns": [],
"run": true,
"min_sample_size": 100,
"performance_change_thresholds": null,
"p_value_threshold": 0.05
},
"row_null_proportion": {
"exclude_columns": [],
"run": true,
"min_sample_size": 100,
"performance_change_thresholds": null,
"drift_statistic": "Population Stability Index",
"params": {
"psi_threshold": [
0.2,
0.6
]
}
},
"label_imbalance": {
"run": true,
"severity_thresholds": null,
"normalize": true
},
"required_features": {
"run": true,
"required_feats": null,
"allowed_feats": null,
"ordered": false,
"required_only": false
},
"duplicate_rows": {
"exclude_columns": [],
"run": false
},
"mi_decrease_feature_to_label": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_mutual_information_requirement": 0.2
},
"high_mi_feature_to_label": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true
},
"high_feature_correlation": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"max_pairwise_tests": 200,
"min_num_samples": 100
},
"protected_feature_drift": {
"exclude_columns": [],
"run": true,
"min_sample_size": 100,
"performance_change_thresholds": null,
"numeric_feature_params": {
"drift_statistic": "Population Stability Index",
"min_num_quantiles": 1000,
"num_bins": 100,
"psi_thresholds": [
0.2,
0.6
],
"kl_divergence_thresholds": [
0.1,
0.2
],
"js_divergence_thresholds": [
0.05,
0.1
],
"ks_test_p_value_threshold": 0.05,
"ignore_nans": true
},
"categorical_feature_params": {
"drift_statistic": "Population Stability Index",
"chi_squared_p_value_threshold": 0.05,
"psi_thresholds": [
0.2,
0.6
],
"max_sample_size": null,
"ignore_nans": true
}
},
"protected_proxies": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true
},
"class_imbalance": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true
},
"equalized_odds": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 20,
"performance_change_thresholds": null
},
"intersectional_group_fairness": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"group_fairness_avg_pred": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"group_fairness_avg_rank": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"demographic_parity": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 20,
"performance_change_thresholds": null
},
"demographic_parity_avg_pred": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 20,
"performance_change_thresholds": null
},
"demographic_parity_avg_rank": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 20,
"performance_change_thresholds": null
},
"chi_squared_independence": {
"run": true,
"p_value_thresholds": null,
"min_sample_size": 100,
"min_subset_sample_size": 10
},
"predict_protected_features": {
"run": false,
"accuracy_thresholds": null,
"min_sample_size": 100,
"non_protected_train_features": null,
"max_protected_feature_cardinality": 100,
"max_categorical_train_feature_cardinality": 100,
"ref_data_split": 0.8
},
"subset_sensitivity_pos_pred": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"performance_thresholds": null,
"min_sample_size": 20,
"num_samples_to_simulate": 100
},
"subset_sensitivity_avg_pred": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"performance_thresholds": null,
"min_sample_size": 20,
"num_samples_to_simulate": 100
},
"subset_sensitivity_avg_rank": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"performance_thresholds": null,
"min_sample_size": 20,
"num_samples_to_simulate": 100
},
"equal_opportunity": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"equal_opp_macro_recall": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"predictive_equality": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"min_sample_size": 10,
"performance_change_thresholds": null
},
"text_swap_masc_to_fem": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_swap_fem_to_masc": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_swap_masc_name_to_fem_name": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"text_swap_fem_name_to_masc_name": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"text_swap_masc_to_plural": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_swap_fem_to_plural": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000
},
"text_swap_high_country_to_low_country": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"text_swap_low_country_to_high_country": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"text_swap_maj_name_to_min_name": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"text_swap_min_name_to_maj_name": {
"severity_thresholds": null,
"sample_size": 20,
"run": true,
"max_str_len": 2000,
"source": null,
"target": null,
"case_invariant": true
},
"gendered_pronoun_distribution": {
"exclude_columns": [],
"run": true,
"performance_change_thresholds": [
0.1,
0.3
]
},
"fill_mask_invariance": {
"exclude_columns": [],
"run": true,
"performance_change_thresholds": [
0.4,
0.8
]
},
"exact_match_label_flipping": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true
},
"near_match_label_flipping": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"vectorization_params": {
"image_projection_dim": 1024,
"text_vector_dim": 256,
"max_categories": 100,
"min_frequency": 0.01
}
},
"stateful_black_box_detection": {
"exclude_columns": [],
"severity_thresholds": null,
"run": true,
"window_size": 1000,
"num_neighbors": 50,
"vectorization_params": {
"image_projection_dim": 1024,
"text_vector_dim": 256,
"max_categories": 100,
"min_frequency": 0.01
}
}
}
}