MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_authored_feedback_with_malformed_optional_metadata_remains_renderable
()
tests/reporting/test_insights_view.py:294
Functiontest_authored_restore_does_not_treat_generic_standard_lines_as_authored
()
tests/tier3/test_dataset_grounding.py:440
Functiontest_authored_root_evals_content_stays_guarded
(tmp_path)
tests/tier3/test_plugin_runtime_identity.py:75
Functiontest_authoritative_default_aggregate_allows_agent_only_step_without_verifier_result
(tmp_path: Path)
tests/test_issue55_collector_truth.py:952
Functiontest_authoritative_default_aggregate_rejects_malformed_step_entry
( tmp_path: Path, invalid_step: object, )
tests/test_issue55_collector_truth.py:1020
Functiontest_authoritative_default_aggregate_rejects_malformed_step_results_container
( tmp_path: Path, invalid_steps: object, )
tests/test_issue55_collector_truth.py:1072
Functiontest_authoritative_default_aggregate_rejects_missing_or_empty_step_verifier_reward
( tmp_path: Path, verifier_result: dict[str, object], )
tests/test_issue55_collector_truth.py:930
Functiontest_authoritative_default_aggregate_rejects_present_empty_step_exception
(tmp_path: Path)
tests/test_issue55_collector_truth.py:999
Functiontest_authoritative_default_aggregate_rejects_real_harbor_step_exception_without_reward
(tmp_path: Path)
tests/test_issue55_collector_truth.py:970
Functiontest_authoritative_final_strategy_accepts_incomplete_intermediate_standard_reward
(tmp_path: Path)
tests/test_issue55_collector_truth.py:825
Functiontest_authoritative_mean_strategy_treats_missing_step_metric_keys_as_zero
(tmp_path: Path)
tests/test_issue55_collector_truth.py:846
Functiontest_authoritative_multistep_na_requires_every_constituent_sidecar
(tmp_path)
tests/tier3/test_metric_applicability_producer.py:408
Functiontest_authoritative_multistep_reward_honors_failed_step_sidecar
(tmp_path: Path)
tests/test_issue55_collector_truth.py:746
Functiontest_authoritative_multistep_reward_preserves_complete_custom_topologies
( tmp_path: Path, aggregate: dict[str, object], step_rewards: list[dict[str, object]], expecte
tests/test_issue55_collector_truth.py:2006
Functiontest_authoritative_multistep_reward_rejects_invalid_default_constituent_without_sidecar
( tmp_path: Path, invalid_step: dict[str, object], case_name: str, )
tests/test_issue55_collector_truth.py:797
Functiontest_authoritative_reward_fails_closed_on_symlinked_external_judge_sidecar
( tmp_path: Path, verifier_relative: Path, )
tests/test_issue55_collector_truth.py:2445
Functiontest_authoritative_root_failed_status_is_never_scored
(tmp_path: Path, status_location: str)
tests/test_issue55_collector_truth.py:896
Functiontest_authoritative_root_omits_unsafe_names_from_nested_harbor_reward_details
( tmp_path: Path, surface: str, )
tests/test_issue55_collector_truth.py:2079
Functiontest_authoritative_root_preserves_flat_numeric_custom_metric
(tmp_path: Path)
tests/test_issue55_collector_truth.py:2030
Functiontest_authoritative_root_rejects_invalid_constituent_custom_metric_contract
( tmp_path: Path, root_kind: str, )
tests/test_issue55_collector_truth.py:2116
Functiontest_authoritative_root_rejects_nonnumeric_reward_shapes
( tmp_path: Path, invalid_surface: dict[str, object], )
tests/test_issue55_collector_truth.py:2057
Functiontest_authoritative_root_rejects_reserved_constituent_custom_metric_name
( tmp_path: Path, )
tests/test_issue55_collector_truth.py:2176
Functiontest_authoritative_root_rejects_union_of_valid_constituent_custom_metrics
( tmp_path: Path, )
tests/test_issue55_collector_truth.py:2146
Functiontest_authoritative_strategies_accept_successful_empty_intermediate_verifier_rewards
( tmp_path: Path, strategy: str, aggregate_score: float, empty_rewards: dict[str, object] | No
tests/test_issue55_collector_truth.py:871
Functiontest_authoritative_validator_results_exclude_top_level_mirror
()
tests/reporting/test_dashboard_data.py:96
Functiontest_authorization_header_in_command_words_is_blocked
(config: dict)
tests/validators/test_mcp_static.py:1102
Functiontest_authorization_header_literal_under_any_scheme_is_blocked
(header: str, scheme: str)
tests/validators/test_mcp_static.py:1077
Functiontest_authorization_header_masking
(text: str, expected: str)
tests/tier3/test_security_finding_evidence.py:405
Functiontest_authorization_header_reference_in_command_words_passes
(config: dict)
tests/validators/test_mcp_static.py:1118
Functiontest_authorization_header_reference_is_allowed
(value: str)
tests/validators/test_mcp_static.py:1083
Functiontest_auto_approval_in_a_module_a_third_level_script_imports_is_found
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:1053
Functiontest_auto_approve_for_read_only_tools_is_recorded_only
(tmp_path: Path)
tests/validators/test_plugin_static_risk_review_fixes.py:622
Functiontest_auto_approve_for_write_fetch_or_mcp_tools_is_medium
(tmp_path: Path, matcher: str)
tests/validators/test_plugin_static_risk_review_fixes.py:614
Functiontest_auto_approve_with_a_powershell_launcher_condition_is_high
Regression: a PowerShell 'if' condition was read as a Bash rule, so its launchers looked narrow.
tests/validators/test_plugin_component_risk.py:375
Functiontest_auto_approve_with_a_scoped_shell_condition_is_not_flagged
( tmp_path: Path, matcher: str, condition: str )
tests/validators/test_plugin_component_risk.py:385
Functiontest_auto_approve_with_narrow_matcher_is_recorded_not_flagged
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:337
Functiontest_auto_changes_on_pr_and_mr_but_setup_changes_force_all
(launch, event)
tests/ci/test_run_tier1_image.py:234
Methodtest_auto_detect_error_uses_target_label_without_host_path
(self, _mock_detect, tmp_path: Path)
tests/validators/test_similarity.py:52
Methodtest_auto_detect_returns_error
When auto-detection returns CONTENT_TYPE_UNKNOWN, validation fails.
tests/validators/test_similarity.py:43
Functiontest_auto_does_not_treat_headings_alone_as_author_guidance
(tmp_path, monkeypatch, retained_trace)
tests/tier3/test_dataset_auto_grounding.py:287
Functiontest_auto_existing_empty_dataset_is_unchanged_without_force
(tmp_path)
tests/tier3/test_dataset_auto_grounding.py:542
Functiontest_auto_existing_matching_trace_requires_explicit_write_authorization
(tmp_path, monkeypatch, options)
tests/tier3/test_dataset_auto_grounding.py:825
Functiontest_auto_falls_back_to_matching_retained_trace_and_flags_observation
(tmp_path, monkeypatch, source_option)
tests/tier3/test_dataset_auto_grounding.py:350
Functiontest_auto_force_invalid_dataset_explains_the_explicit_starter_option
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:300
Functiontest_auto_force_invalid_existing_dataset_gives_auto_recovery
(tmp_path, original)
tests/tier3/test_dataset_auto_grounding.py:877
Functiontest_auto_ignores_malformed_tool_call_steps
(tmp_path, monkeypatch, tool_calls)
tests/tier3/test_dataset_auto_grounding.py:533
Functiontest_auto_intent_refine_retains_collection_compatibility
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:452
Functiontest_auto_keeps_existing_dataset_without_explicit_refinement
(tmp_path, monkeypatch, entrypoint)
tests/tier3/test_dataset_auto_grounding.py:82
Functiontest_auto_loads_native_clients_natively_and_keeps_hermes_on_the_wrapper
()
tests/tier3/test_plugin_load_decisions.py:20
Functiontest_auto_missing_input_fails_before_provider_collection_or_writes
(tmp_path, monkeypatch, options)
tests/tier3/test_dataset_auto_grounding.py:334
Functiontest_auto_missing_input_preserves_existing_dataset_when_forced
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:486
Functiontest_auto_partial_trace_checks_unmatched_inferred_outcomes
(tmp_path, monkeypatch, proposed, authored)
tests/tier3/test_dataset_auto_grounding.py:218
Functiontest_auto_prefers_nonblank_author_intent_to_retained_trace
(tmp_path, monkeypatch, explicit_auto)
tests/tier3/test_dataset_auto_grounding.py:51
Functiontest_auto_prompt_overrides_eval_guidance_and_records_effective_source
(tmp_path)
tests/tier3/test_dataset_auto_grounding.py:67
Functiontest_auto_records_agent_fallback_and_native_refuses_before_engine
(tmp_path)
tests/test_private_runtime_review.py:168
Functiontest_auto_refine_preserves_typed_authored_expected_outputs
(tmp_path, monkeypatch, authored_output)
tests/tier3/test_dataset_auto_grounding.py:315
Functiontest_auto_rejects_empty_or_non_agent_trace_evidence
(tmp_path, monkeypatch, steps)
tests/tier3/test_dataset_auto_grounding.py:383
Functiontest_auto_rejects_foreign_explicit_results_even_with_matching_case_id
(tmp_path)
tests/tier3/test_dataset_auto_grounding.py:404
Functiontest_auto_rejects_heading_and_comment_only_guidance
(tmp_path, monkeypatch, source, guidance)
tests/tier3/test_dataset_auto_grounding.py:903
Functiontest_auto_rejects_incomplete_run_and_conflicting_skill_identity
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:518
Functiontest_auto_rejects_linked_case_identity_before_reading
(tmp_path, monkeypatch, relative)
tests/tier3/test_dataset_auto_grounding.py:692
Functiontest_auto_rejects_nonregular_trace_sources_before_reading
(tmp_path, monkeypatch, filename)
tests/tier3/test_dataset_auto_grounding.py:668
Functiontest_auto_rejects_unmatched_case_ids_before_generation
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:393
Functiontest_auto_reporter_selects_terminal_appropriate_presentation
(monkeypatch: pytest.MonkeyPatch)
tests/test_tier3_progress.py:108
Functiontest_auto_reporter_uses_immediate_plain_output_for_non_interactive_tty
( monkeypatch: pytest.MonkeyPatch, term: str, )
tests/test_tier3_progress.py:216
Functiontest_auto_requires_text_or_tool_actions_beyond_placeholder_messages
(tmp_path, monkeypatch, message)
tests/tier3/test_dataset_auto_grounding.py:555
Functiontest_auto_searches_past_an_agent_without_usable_matching_evidence
(tmp_path, monkeypatch, higher_case_id)
tests/tier3/test_dataset_auto_grounding.py:565
Functiontest_auto_tool_only_trace_checks_nested_inferred_outcomes
(tmp_path, monkeypatch, proposed, reason, authored)
tests/tier3/test_dataset_auto_grounding.py:163
Functiontest_auto_trace_does_not_regenerate_ids_before_llm_refinement
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:499
Functiontest_auto_trace_filters_only_inferred_skill_use_ground_truth
(tmp_path, monkeypatch, use_llm, authored_truth)
tests/tier3/test_dataset_auto_grounding.py:944
Functiontest_auto_trace_keeps_skill_use_out_of_new_expected_outputs
(tmp_path, monkeypatch, use_llm, authored)
tests/tier3/test_dataset_auto_grounding.py:123
Functiontest_auto_trace_preserves_authored_cases_and_flags_unmatched_coverage
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:413
Functiontest_auto_trace_preserves_authored_prompt_metadata_and_review_history
(tmp_path, monkeypatch, use_llm, wrapper)
tests/tier3/test_dataset_auto_grounding.py:592
Functiontest_auto_unchanged_dataset_does_not_parse_the_skill
(tmp_path, monkeypatch)
tests/tier3/test_dataset_auto_grounding.py:108
Functiontest_auto_with_both_task_sources_uses_evals_json_and_probes_the_standard_judge
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:3719
Functiontest_auto_with_native_only_custom_grading_skips_the_standard_judge_probe
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:3776
Functiontest_automatic_setup_supports_any_github_owner
(repository, monkeypatch, owner)
tests/test_ci_onboarding.py:450
Functiontest_autopilot_llm_case_does_not_grade_skill_use
(tmp_path: Path, llm_reply)
tests/tier3/test_dataset_baseline_fairness.py:1306
Functiontest_autopilot_llm_case_keeps_a_task_check_that_mentions_secrets_or_destructive_changes
( tmp_path: Path, llm_reply, task_check )
tests/tier3/test_dataset_baseline_fairness.py:1431
Functiontest_autopilot_llm_case_replaces_skill_use_with_text_that_names_no_skill_script
(tmp_path: Path, llm_reply)
tests/tier3/test_dataset_baseline_fairness.py:1341
Functiontest_autopilot_llm_case_with_only_its_own_security_check_left_gets_outcome_checks
(tmp_path: Path, llm_reply)
tests/tier3/test_dataset_baseline_fairness.py:1393
Functiontest_autopilot_llm_case_with_only_skill_use_checks_gets_outcome_checks
(tmp_path: Path, llm_reply)
tests/tier3/test_dataset_baseline_fairness.py:1367
Functiontest_autopilot_llm_generator_surfaces_model_error_without_printing_raw_detail
( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], )
tests/test_evaluation_service.py:702
Functiontest_autopilot_missing_docker_does_not_spend_generation_request
(monkeypatch, tmp_path, command)
tests/test_evaluation_service.py:419
Functiontest_autopilot_records_llm_and_template_fallback
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:657
Functiontest_autopilot_rejects_invalid_run_before_paid_generation
(monkeypatch, tmp_path, command, flags, diagnostic)
tests/test_evaluation_service.py:382
Functiontest_autopilot_skips_harbor_agent_preflight_for_nemo_gym
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/test_nemo_gym_backend.py:2990
Functiontest_autopilot_starter_leaves_out_eval_md_skill_use_behaviors
(tmp_path: Path)
tests/tier3/test_dataset_baseline_fairness.py:1459
Functiontest_autopilot_template_case_does_not_grade_skill_use
(tmp_path: Path)
tests/tier3/test_dataset_baseline_fairness.py:486
Functiontest_autopilot_validates_setup_commands_against_evaluation_arms
( monkeypatch, tmp_path, command, arm_flags, compatible, setup_field, setup_value )
tests/test_evaluation_service.py:511
Functiontest_available_trend_skips_unavailable_rows_in_same_segment
(tmp_path)
tests/test_tier3_history.py:363
Functiontest_average_metrics_omits_unavailable_canonical_metrics
()
tests/test_harbor_metrics_truth.py:190
Functiontest_backend_agent_wrappers_are_the_only_extra_agent_paths_allowed
(acme: types.ModuleType)
tests/test_custom_execution_backends.py:151
Functiontest_backend_aliases_cannot_bypass_sandbox_runtime_policy
( env_mode: str, name: str, value: object, )
tests/test_tier3_public_runtime.py:345
Functiontest_backend_compose_support_gates_staged_tasks
( acme: types.ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/test_custom_execution_backends.py:276
Functiontest_backend_defaults_to_harbor_and_is_forwarded
()
tests/test_nemo_gym_backend.py:41
Functiontest_backend_descriptors_are_validated
(overrides: dict[str, object], message: str)
tests/test_custom_execution_backends.py:307
Functiontest_backend_dns_resolution_obeys_deadline_without_late_request_or_resolver_growth
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/test_nvidia_build_bridge.py:759
Functiontest_backend_error_is_safe_and_logs_never_expose_api_key
(bridge_services: _BridgeServices)
tests/test_nvidia_build_bridge.py:1566
Functiontest_backend_host_variables_reach_harbor_and_are_protected_from_skills
(acme: types.ModuleType)
tests/test_custom_execution_backends.py:232
← previousnext →12,901–13,000 of 24,741, ranked by callers