MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Methodtest_task_list_imperatives_are_clear_action_verbs
(self, tmp_path: Path, action: str)
tests/validators/test_quality_score.py:1493
Functiontest_task_modes_reject_custom_grader_symlink_before_copy
(tmp_path: Path, task_source: str)
tests/test_harbor_adapter_safety.py:485
Functiontest_task_selector_mapping_rejects_cross_platform_identity_ambiguity
( tmp_path: Path, expected_case_ids: list[str], mapping: dict[str, str], )
tests/test_harbor_collector_runtime_failures.py:2211
Functiontest_task_selector_mapping_rejects_duplicate_logical_case_ids
(tmp_path: Path)
tests/test_harbor_collector_runtime_failures.py:2193
Functiontest_task_staging_enospc_returns_structured_error_without_launch_and_cleans_artifacts
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure_arm: str, )
tests/test_tier3_progress.py:4957
Functiontest_task_staging_has_the_shape_native_staging_consumes
(tmp_path: Path, source)
tests/tier3/test_private_loose_source.py:484
Functiontest_task_status_words_survive_persistence
( content: str, operation: str | None, status: str, checkbox: str, tmp_path: Path )
tests/test_reference_skill_scripts.py:94
Functiontest_task_text_that_shares_the_skill_name_or_file_does_not_warn
(entry)
tests/tier3/test_dataset_baseline_fairness.py:165
Functiontest_task_timeout_plan_rejects_staged_timeout_product_overflow
( tmp_path: Path, task_toml: str, )
tests/test_harbor_runtime_preflight.py:789
Functiontest_task_timeout_plan_uses_largest_staged_timeout
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:766
Functiontest_task_verifier_env_forwards_the_judge_policies
()
tests/tier3/test_judge_policies.py:826
Functiontest_tautological_pair_checks_have_reasons
(field: str, reason: str)
tests/reporting/test_dashboard_data.py:280
Functiontest_temp_snapshot_inside_legacy_files_does_not_recurse_into_itself
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_skill_isolation.py:5002
Functiontest_template_anthropic_claude_5_5_sends_effort_and_server_side_fallback
(judge_env, model, claude_5_5)
tests/tier3/test_judge_parse_robustness.py:806
Functiontest_template_anthropic_gateway_gets_effort_without_server_side_fallback
(judge_env)
tests/tier3/test_judge_parse_robustness.py:822
Functiontest_template_anthropic_refusal_tries_the_configured_fallback_model
(judge_env)
tests/tier3/test_judge_parse_robustness.py:881
Functiontest_template_anthropic_reply_model_is_ignored_without_server_side_fallback
(judge_env)
tests/tier3/test_judge_parse_robustness.py:868
Functiontest_template_anthropic_request_uses_model_compatible_temperature
(monkeypatch, model, expected_temperature)
tests/tier3/test_judge_parse_robustness.py:565
Functiontest_template_baseline_trial_without_trajectory_keeps_skill_metrics_na
(tmp_path, monkeypatch)
tests/tier3/test_metric_applicability_producer.py:363
Functiontest_template_bedrock_classifier_refusal_retries_on_opus_4_8
(monkeypatch, model)
tests/tier3/test_judge_parse_robustness.py:926
Functiontest_template_bedrock_refusal_without_a_fallback_route_is_reported
(monkeypatch)
tests/tier3/test_judge_parse_robustness.py:945
Functiontest_template_bedrock_request_uses_model_compatible_temperature
(monkeypatch, model, expected_temperature)
tests/tier3/test_judge_parse_robustness.py:604
Functiontest_template_behavior_judge_matches_eval_core
(monkeypatch, responses)
tests/tier3/test_judge_parse_robustness.py:453
Functiontest_template_behavior_judge_max_tokens_matches_eval_core_constant
()
tests/tier3/test_judge_parse_robustness.py:467
Functiontest_template_benign_words_in_agent_response_are_not_secrets
(word)
tests/tier3/test_harbor_template_secret_patterns.py:82
Functiontest_template_benign_words_in_observation_are_not_secrets
(word)
tests/tier3/test_harbor_template_secret_patterns.py:74
Functiontest_template_budget_effort_and_fallback_constants_match_package
()
tests/tier3/test_judge_parse_robustness.py:521
Functiontest_template_bundles_match_shared_helper
()
tests/tier3/test_evidence_compiler.py:67
Functiontest_template_cases_keep_developer_behaviors
()
tests/tier3/test_dataset_baseline_fairness.py:432
Functiontest_template_cases_keep_the_grounding_order_and_stay_fair
(generate, grounding)
tests/tier3/test_dataset_baseline_fairness.py:443
Functiontest_template_cases_stay_fair_when_the_description_names_the_skill
(generate, description)
tests/tier3/test_dataset_baseline_fairness.py:878
Functiontest_template_cases_still_quote_a_description_that_names_other_files
(description)
tests/tier3/test_dataset_baseline_fairness.py:992
Functiontest_template_default_judge_models_match_provider_chat_defaults
()
tests/tier3/test_judge_parse_robustness.py:474
Functiontest_template_escapes_each_recorded_field_and_keeps_shared_execution_errors_accessible
()
tests/reporting/test_diagnostic_view.py:162
Functiontest_template_extract_json_matches_eval_core_on_rejected_shapes
(text)
tests/tier3/test_judge_parse_robustness.py:433
Functiontest_template_extract_json_matches_eval_core_on_tolerated_shapes
(text, expected)
tests/tier3/test_judge_parse_robustness.py:428
Functiontest_template_fallback_records_safe_reason
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:616
Functiontest_template_generator_cases_are_fair_to_the_baseline
(generate, eval_prompt)
tests/tier3/test_dataset_baseline_fairness.py:407
Functiontest_template_glued_id_or_hash_tokens_are_not_secrets
(token)
tests/tier3/test_harbor_template_secret_patterns.py:99
Functiontest_template_goal_preserves_provenance_on_success_and_error
(monkeypatch)
tests/tier3/test_judge_result_semantics.py:884
Functiontest_template_graded_text_does_not_quote_a_description_that_names_a_skill_script
(generate, description)
tests/tier3/test_dataset_baseline_fairness.py:977
Functiontest_template_last_resort_judge_follows_the_selected_provider
(judge_env, provider)
tests/tier3/test_judge_parse_robustness.py:479
Functiontest_template_main_scores_standard_negative_fail_closed
(tmp_path, monkeypatch, include_trusted_identity)
tests/tier3/test_negative_control_invocation_evidence.py:449
Functiontest_template_model_capabilities_match_provider_config
(model)
tests/tier3/test_judge_parse_robustness.py:509
Functiontest_template_model_fallback_isolates_schema_memoization_per_model
Verify memoizing primary-model as schema-unsupported does not strip schema from fallback-model.
tests/test_llm_judge.py:682
Functiontest_template_native_openai_gpt5_payload_matches_eval_core
(model)
tests/tier3/test_judge_parse_robustness.py:706
Functiontest_template_negative_case_is_unchanged
()
tests/tier3/test_dataset_baseline_fairness.py:465
Functiontest_template_no_reference_main_writes_numeric_reward_and_rich_na
(tmp_path, monkeypatch)
tests/tier3/test_metric_applicability_producer.py:287
Functiontest_template_non_string_openai_content_remains_a_provider_error
(monkeypatch, content)
tests/tier3/test_judge_parse_robustness.py:681
Functiontest_template_nonfinite_ragas_score_falls_back_to_custom
(monkeypatch)
tests/tier3/test_judge_result_semantics.py:906
Functiontest_template_openai_refusal_or_empty_reply_tries_the_fallback_model
(judge_env, first_reply)
tests/tier3/test_judge_parse_robustness.py:913
Functiontest_template_positive_cases_grade_the_task_outcome
()
tests/tier3/test_dataset_baseline_fairness.py:417
Functiontest_template_prompt_does_not_require_a_skill_script
()
tests/tier3/test_dataset_baseline_fairness.py:815
Functiontest_template_prompt_is_generic_when_description_requires_own_script
(generate)
tests/tier3/test_dataset_baseline_fairness.py:1720
Functiontest_template_prompt_is_generic_when_the_description_names_the_skill_directory
()
tests/tier3/test_dataset_baseline_fairness.py:1245
Functiontest_template_prompt_names_a_skill_md_in_the_description_as_the_users_file
(description, task)
tests/tier3/test_dataset_baseline_fairness.py:891
Functiontest_template_ragas_goal_scorer_skips_reasoning_judges
(judge_env, model, enabled)
tests/tier3/test_judge_parse_robustness.py:959
Functiontest_template_real_key_in_observation_is_flagged
(secret)
tests/tier3/test_harbor_template_secret_patterns.py:89
Functiontest_template_reference_check_counts_false_and_zero_but_not_empty_values
(value, expected)
tests/tier3/test_metric_applicability_producer.py:63
Functiontest_template_refine_drops_runs_of_a_shell_skill_script
(tmp_path: Path)
tests/tier3/test_dataset_baseline_fairness.py:1558
Functiontest_template_refine_drops_skill_use_and_keeps_task_work
(with_skill_context)
tests/tier3/test_dataset_baseline_fairness.py:587
Functiontest_template_refine_ignores_non_string_skill_fields
()
tests/tier3/test_dataset_baseline_fairness.py:1521
Functiontest_template_refine_treats_a_placeholder_expected_script_as_no_script
(placeholder, command)
tests/tier3/test_dataset_baseline_fairness.py:1544
Functiontest_template_refinement_deduplicates_standard_behaviors_and_keeps_trajectory_actions
()
tests/tier3/test_dataset_grounding.py:401
Functiontest_template_refusal_fallback_target_matches_package
(monkeypatch, value)
tests/tier3/test_judge_parse_robustness.py:547
Functiontest_template_refusal_without_a_fallback_is_a_judge_error
(judge_env)
tests/tier3/test_judge_parse_robustness.py:894
Functiontest_template_required_judge_accepts_only_verifier_authorized_na
()
tests/tier3/test_metric_applicability_producer.py:270
Functiontest_template_required_judge_records_the_model_that_answered
(judge_env, reply)
tests/tier3/test_judge_parse_robustness.py:852
Functiontest_template_safe_network_commands_match_eval_core
Verify that standalone Harbor template and eval_core both treat safe commands as safe.
tests/tier3/test_harbor_template_secret_patterns.py:246
Functiontest_template_salvage_matches_eval_core
()
tests/tier3/test_judge_parse_robustness.py:438
Functiontest_template_salvage_score_matches_eval_core_on_partial_recovery
(monkeypatch)
tests/tier3/test_judge_parse_robustness.py:739
Functiontest_template_schema_and_payload_match_eval_core
()
tests/tier3/test_judge_parse_robustness.py:722
Functiontest_template_structured_judges_retry_nullable_openai_content
( monkeypatch, judge_name, valid_response, )
tests/tier3/test_judge_parse_robustness.py:648
Functiontest_template_task_is_generic_when_the_skill_owns_the_skill_md_it_reads
(description)
tests/tier3/test_dataset_baseline_fairness.py:911
Functiontest_template_task_renames_a_skill_md_generic_wording_explains_when_the_skill_is_named_after_it
( name, description, task )
tests/tier3/test_dataset_baseline_fairness.py:947
Functiontest_template_task_still_quotes_a_skill_md_no_skill_name_owns
(description, task)
tests/tier3/test_dataset_baseline_fairness.py:930
Functiontest_template_trial_is_unscored_when_a_judge_claims_na_for_a_defined_reference
(tmp_path, monkeypatch, metric)
tests/tier3/test_metric_applicability_producer.py:380
Functiontest_template_trial_without_trajectory_keeps_reference_less_judges_na
(tmp_path, monkeypatch)
tests/tier3/test_metric_applicability_producer.py:339
Functiontest_template_unsafe_network_commands_match_eval_core
Verify that standalone Harbor template and eval_core both flag unsafe exfiltration commands.
tests/tier3/test_harbor_template_secret_patterns.py:284
Functiontest_template_unsupported_codex_wrapper_is_not_a_clean_security_result
()
tests/tier3/test_codex_tool_call_normalization.py:370
Functiontest_template_uses_current_command_aliases
(harbor_eval_template)
tests/tier3/test_shared_atif_staging.py:45
Functiontest_template_uses_shared_declarative_check_logic
(harbor_eval_template)
tests/tier3/test_declarative_checks.py:294
Functiontest_template_uses_the_shared_log_redaction
()
tests/tier3/test_harbor_template_secret_patterns.py:223
Functiontest_template_verified_facts_match_shared
()
tests/tier3/test_verified_facts.py:57
Functiontest_terminal_bad_or_empty_uploads_preserve_normalized_jobs_and_run_analysis
(uploads)
tests/reporting/test_evidence_views.py:330
Functiontest_terminal_control_removal_cannot_reassemble_a_credential_in_static_views
()
tests/reporting/test_private_static_reports.py:649
Functiontest_terminal_controls_are_stripped
(text: str, expected: str)
tests/utils/test_rich_markup.py:104
Functiontest_terminal_conversion_and_token_context
(tmp_path)
tests/test_hermes_agent.py:281
Functiontest_terminal_deduplicates_failure_reasons
(capsys)
tests/tier3/test_interactions.py:338
Functiontest_terminal_does_not_restore_dimension_lift_from_raw_scores
()
tests/reporting/test_skill_activation_reporting.py:345
Functiontest_terminal_failed_reads_cannot_prove_activation
(tmp_path, result)
tests/test_hermes_agent.py:296
Functiontest_terminal_findings_panel_never_shows_a_tolerated_agent
( tmp_path: Path, capsys: pytest.CaptureFixture[str], tolerated: tuple[str, ...], panel_agent:
tests/tier3/test_baseline_timeouts.py:1082
Functiontest_terminal_keeps_not_applicable_distinct_from_missing_evidence
()
tests/tier3/test_metric_applicability_display.py:13
Functiontest_terminal_lift_reads_na_when_either_arm_is_na
()
tests/tier3/test_metric_applicability_display.py:51
Functiontest_terminal_lift_reads_skipped_in_a_baseline_only_run_unless_the_row_is_na
()
tests/tier3/test_metric_applicability_display.py:90
Functiontest_terminal_lists_harbor_jobs_from_engine_uploads
()
tests/reporting/test_evidence_views.py:313
Functiontest_terminal_matching_upload_fills_only_missing_canonical_analysis
(existing_analysis, padded_url)
tests/reporting/test_evidence_views.py:409
Functiontest_terminal_overall_renormalizes_only_explicit_na_metrics
()
tests/tier3/test_metric_applicability_display.py:28
Functiontest_terminal_prints_coverage_notes_for_a_non_blocking_tier3_result
``validate`` runs Tier 3 non-blocking by default; the failed run has errors but no findings. The full terminal report (``validate --verbose`` or
tests/reporting/test_tier3_incomplete_arms.py:484
Functiontest_terminal_punctuation_does_not_hide_skill_use_or_match_metadata
(text, warns)
tests/tier3/test_dataset_baseline_fairness.py:1813
← previousnext →23,101–23,200 of 24,741, ranked by callers