MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_native_atif_error_is_matched_to_its_own_call
(grader: tuple[ModuleType, ModuleType])
tests/tier3/test_failed_skill_reads.py:265
Functiontest_native_atif_extras_win_over_inference
()
tests/tier3/test_trajectory_timing.py:156
Functiontest_native_baseline_ignores_target_payload_in_unstaged_results
(tmp_path: Path, payload_kind: str)
tests/test_harbor_runtime_skill_isolation.py:3505
Functiontest_native_baseline_rejects_aliased_authored_skill
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:3465
Functiontest_native_baseline_rejects_authored_agent_phase_healthchecks
( tmp_path: Path, healthcheck_scope: str, )
tests/test_harbor_runtime_skill_isolation.py:2809
Functiontest_native_baseline_rejects_effective_task_shipped_prior_trajectory_before_output_mutation
( tmp_path: Path, trajectory_scope: str, relative_path: Path, )
tests/test_harbor_runtime_skill_isolation.py:2851
Functiontest_native_baseline_rejects_every_configured_step_workdir_projection
( tmp_path: Path, overlay_relative: Path, )
tests/test_harbor_runtime_skill_isolation.py:3228
Functiontest_native_baseline_rejects_renamed_target_manifest_payload
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:3483
Functiontest_native_bash_exit_is_attributable
(agent_class, exit_code, native_events, tmp_path)
tests/tier3/test_opencode_exit_status.py:50
Functiontest_native_body_uses_readable_sixteen_pixel_default
()
tests/reporting/test_report_redesign.py:474
Functiontest_native_bootstrap_avoids_system_node_and_keeps_vendor_installer
( tmp_path: Path, agent_class: type, manager: str )
tests/tier3/test_installed_agent_bootstrap.py:125
Functiontest_native_bootstrap_preserves_other_system_dependencies
( tmp_path: Path, agent_class: type, missing_command: str, expected_package: str )
tests/tier3/test_installed_agent_bootstrap.py:197
Functiontest_native_bootstrap_preserves_unknown_dependency_validation
(tmp_path: Path, agent_class: type)
tests/tier3/test_installed_agent_bootstrap.py:277
Functiontest_native_brand_header_replaces_sidebar_and_decorative_logo
()
tests/reporting/test_report_redesign.py:465
Functiontest_native_case_alias_cannot_overwrite_an_input
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, destination: str )
tests/tier3/test_decoy_commands.py:252
Functiontest_native_claude_code_loads_only_the_inventoried_member_skills
(tmp_path: Path)
tests/tier3/test_openclaw_prepare.py:169
Functiontest_native_claude_mcp_applies_the_task_credential_policy
(tmp_path: Path, server: dict)
tests/tier3/test_private_runtime_regressions.py:72
Functiontest_native_claude_mcp_keeps_operator_approved_values_and_plugin_root
(tmp_path: Path, monkeypatch)
tests/tier3/test_private_runtime_regressions.py:78
Functiontest_native_claude_model_resolution_is_unchanged
(tmp_path, model, extra_env, expected)
tests/tier3/test_claude_gateway_model_handoff.py:91
Functiontest_native_codex_exec_accepts_one_first_line_pragma
(extractor)
tests/tier3/test_codex_tool_call_normalization.py:100
Functiontest_native_codex_exec_evidence_refs_resolve_to_the_outer_call
()
tests/tier3/test_codex_tool_call_normalization.py:967
Functiontest_native_codex_exec_keeps_unsupported_wrappers_atomic
(source, extractor)
tests/tier3/test_codex_tool_call_normalization.py:200
Functiontest_native_codex_exec_maps_a_rendered_retry_observation_only_to_its_owner
(extractor)
tests/tier3/test_codex_tool_call_normalization.py:696
Functiontest_native_codex_exec_parser_limits_fail_closed
(source, extractor)
tests/tier3/test_codex_tool_call_normalization.py:682
Functiontest_native_codex_exec_unwraps_static_tools_in_source_order
(extractor)
tests/tier3/test_codex_tool_call_normalization.py:76
Functiontest_native_comparison_numeric_cells_are_typed_finite_and_bounded
(invalid)
tests/reporting/test_native_overview_comparison.py:143
Functiontest_native_comparison_text_uses_a_parser_without_changing_visible_reason_text
()
tests/reporting/test_report_markup_assertions.py:47
Functiontest_native_completeness_retains_recorded_ignored_mcp_config_details
()
tests/reporting/test_native_evidence.py:304
Functiontest_native_contamination_scan_uses_shared_ignore_callback
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_skill_isolation.py:3544
Functiontest_native_controls_have_readable_computed_contrast_in_each_state
(page, reports, theme, width)
tests/reporting/test_report_browser.py:863
Functiontest_native_copy_input_without_fixtures_gets_empty_build_context_directory
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:2447
Functiontest_native_custom_only_accepts_step_local_test_scripts_without_top_level_tests
( tmp_path: Path, )
tests/test_harbor_runtime_skill_isolation.py:3017
Functiontest_native_custom_only_accepts_top_level_fallback_for_separate_step_without_step_context
( tmp_path: Path, )
tests/test_harbor_runtime_skill_isolation.py:3093
Functiontest_native_custom_only_native_test_preserves_authored_skill_evaluator_package
( tmp_path: Path, )
tests/test_harbor_runtime_skill_isolation.py:3136
Functiontest_native_custom_only_preserves_authored_judge_controls_in_verifier_env
( tmp_path: Path, name: str, )
tests/test_harbor_runtime_skill_isolation.py:385
Functiontest_native_custom_only_preserves_task_and_step_collect_hooks
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:2987
Functiontest_native_custom_only_rejects_top_level_fallback_hidden_by_existing_separate_step_context
( tmp_path: Path, )
tests/test_harbor_runtime_skill_isolation.py:3046
Functiontest_native_dataset_writer_quotes_hostile_directory_names
(tmp_path: Path)
tests/test_harbor_scaffold_transactions.py:473
Functiontest_native_dedicated_verifier_image_without_bundled_test_is_incomplete
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:5201
Functiontest_native_default_environment_installs_declared_apt_packages
( tmp_path: Path, apt_packages: str | None, expected: list[str] )
tests/test_harbor_apt_install_lines.py:107
Functiontest_native_default_without_eval_metadata_preserves_existing_output
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:438
Functiontest_native_different_condition_models_cannot_pair
(tmp_path: Path)
tests/reporting/test_dashboard_data.py:1118
Functiontest_native_duplicate_reward_rows_require_authoritative_step_scope
(tmp_path: Path)
tests/reporting/test_dashboard_data.py:1214
Functiontest_native_environment_hidden_harbor_022_kwargs_remain_usable
( env_mode: str, name: str, value: object, )
tests/test_tier3_public_runtime.py:212
Functiontest_native_environment_install_hints_use_real_harbor_022_extra_names
()
tests/test_tier3_public_runtime.py:578
Functiontest_native_environment_is_forwarded_to_harbor
()
tests/test_tier3_public_runtime.py:119
Functiontest_native_environment_kwargs_reject_unknown_harbor_022_names
(env_mode: str)
tests/test_tier3_public_runtime.py:193
Functiontest_native_environment_kwargs_resolve_real_harbor_ec2_constructor
(tmp_path: Path)
tests/test_tier3_public_runtime.py:228
Functiontest_native_environment_kwargs_round_trip_through_real_harbor_parser
()
tests/test_tier3_public_runtime.py:171
Functiontest_native_environment_required_kwargs_accept_valid_ec2_attach_configuration
()
tests/test_tier3_public_runtime.py:484
Functiontest_native_environment_required_kwargs_fail_before_mutation
( env_mode: str, environment_kwargs: dict[str, object], expected: str, )
tests/test_tier3_public_runtime.py:444
Functiontest_native_environment_required_kwargs_reject_non_string_values_without_crashing
( env_mode: str, environment_kwargs: dict[str, object], expected: str, )
tests/test_tier3_public_runtime.py:473
Functiontest_native_environment_required_kwargs_reject_whitespace_padded_ec2_launch_mode
()
tests/test_tier3_public_runtime.py:569
Functiontest_native_evidence_is_read_before_cleanup_and_never_inferred_from_staging
(tmp_path: Path)
tests/tier3/test_plugin_execution.py:184
Methodtest_native_exec_inner_calls_without_proven_observation_are_unknown
(self)
tests/tier3/test_plugin_signals.py:224
Methodtest_native_exec_wrapper_is_normalized_and_mapped_observation_is_used
(self)
tests/tier3/test_plugin_signals.py:215
Functiontest_native_extras_on_copied_context_only_do_not_block_inference
()
tests/tier3/test_trajectory_timing.py:433
Functiontest_native_fallback_verifier_compose_uses_the_verifier_allowlist
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:5156
Functiontest_native_generated_stop_on_pass_names_preserve_attempt_sets
(tmp_path: Path)
tests/reporting/test_dashboard_data.py:1093
Functiontest_native_groups_display_recorded_evidence_without_changing_input
()
tests/reporting/test_native_evidence.py:36
Functiontest_native_harbor_14_task_stages_without_rewriting_its_schema
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:244
Functiontest_native_harbor_compose_uses_the_same_escape_policy
(tmp_path: Path)
tests/test_harbor_custom_compose_safety.py:389
Functiontest_native_harbor_metadata_entry_id_seam_across_runner_collector_and_dataset_snapshot
Verify native custom-only tasks with [metadata].entry_id != directory name align across runner, collector, and dataset_snapshot.json.
tests/test_tier3_public_runtime.py:2596
Functiontest_native_harbor_source_reuses_the_reviewed_tasks
(monkeypatch, tmp_path)
tests/tier3/test_interactions.py:495
Functiontest_native_harbor_task_source_resolves_like_harbor
(tmp_path: Path)
tests/test_nemo_gym_backend.py:1495
Functiontest_native_header_and_page_geometry_replace_sidebar_workspace
()
tests/reporting/test_native_report_parity.py:31
Functiontest_native_header_scale_is_not_replaced_by_the_compact_override
()
tests/reporting/test_native_feature_restoration.py:123
Functiontest_native_header_versions_and_visible_overview_comparison_are_restored
(page, reports, width)
tests/reporting/test_report_browser.py:532
Functiontest_native_hermes_keeps_the_wrapper_task_under_its_own_home
()
tests/tier3/test_plugin_load_decisions.py:31
Functiontest_native_hooks_stage_only_the_with_arm_of_native_agents
(tmp_path: Path)
tests/tier3/test_plugin_execution.py:161
Functiontest_native_host_override_does_not_launder_authored_retry_settings
Verify an operator override cannot make authored retry literals acceptable for standard grading.
tests/test_harbor_task_toml_retry_env.py:272
Functiontest_native_html_places_each_group_once_and_preserves_canonical_exports
()
tests/reporting/test_native_evidence.py:238
Functiontest_native_html_places_each_plugin_section_once_and_exports_exact_snapshots
()
tests/reporting/test_plugin_evidence_report.py:186
Functiontest_native_identity_helper_rejects_non_case_id_metadata_scalars
( tmp_path: Path, entry_id: object, )
tests/test_harbor_runtime_skill_isolation.py:5101
Functiontest_native_identity_helpers_preserve_numeric_zero_entry_id
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:5087
Functiontest_native_inconclusive_judge_probe_does_not_skip_the_same_route_used_by_an_agent
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:3896
Functiontest_native_input_is_preserved_without_eval_fixture_declaration
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:2474
Functiontest_native_invalid_coverage_counts_cannot_be_hidden_by_canonical_normalization
( tmp_path: Path, field: str, value: object )
tests/reporting/test_dashboard_data.py:1132
Functiontest_native_invocation_exports_session_even_on_failure
(tmp_path, monkeypatch, failure)
tests/test_hermes_agent.py:203
Functiontest_native_jobs_forward_gateway_settings_only_at_protected_execution_steps
()
tests/ci/test_gateway_benchmark_workflows.py:41
Functiontest_native_judge_provenance_uses_only_scoring_steps
( tmp_path: Path, strategy: str | None, equal_scores: bool, final_only: bool )
tests/test_harbor_collector_judge_strategy.py:91
Functiontest_native_load_without_agents_is_refused
(tmp_path: Path)
tests/tier3/test_plugin_execution.py:178
Functiontest_native_macro_distinguishes_omitted_rows_from_shortened_text
()
tests/reporting/test_plugin_evidence_report.py:164
Functiontest_native_main_image_replaces_authored_healthcheck_with_constant_readiness
(tmp_path: Path)
tests/test_harbor_compose_readiness.py:49
Functiontest_native_mcp_keeps_agent_grants_and_legacy_verifier_defaults
( tmp_path: Path, scope: str, verifier_env: dict[str, str] | None )
tests/test_harbor_runtime_skill_isolation.py:5345
Functiontest_native_mcp_preserves_recorded_resolution_detail_and_ignores_shaped_status
()
tests/reporting/test_native_evidence.py:209
Functiontest_native_mcp_uses_each_tasks_runtime_environment_even_with_cached_bundle
(tmp_path)
tests/test_private_runtime_rereview.py:126
Functiontest_native_mcp_validation_hints_do_not_echo_authored_values
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, declaration: str, hint: str )
tests/test_harbor_runtime_skill_isolation.py:5375
Functiontest_native_missing_attempt_does_not_publish_partial_usage_as_total
(tmp_path: Path)
tests/reporting/test_dashboard_data.py:1079
Functiontest_native_mode_canonicalizes_numeric_eval_id_and_stages_task
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:513
Functiontest_native_mode_validates_eval_ids_before_replacing_output
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:499
Functiontest_native_multi_step_timeouts_are_refused_by_name
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:600
Functiontest_native_multistep_rewards_count_as_one_logical_attempt
(tmp_path: Path)
tests/test_harbor_collector_runtime_failures.py:1205
Functiontest_native_multistep_root_requires_every_authoritative_trajectory
(tmp_path: Path, complete: bool)
tests/reporting/test_activation_validation.py:392
Functiontest_native_openai_completion_token_url_accepts_only_canonical_variants
(request_url: str)
tests/test_llm_judge.py:302
Functiontest_native_openai_gpt5_uses_max_completion_tokens_without_temperature
Verify native OpenAI gpt-5 requests use max_completion_tokens without temperature.
tests/test_llm_judge.py:27
Functiontest_native_openai_non_gpt5_keeps_max_tokens
()
tests/test_llm_judge.py:245
Functiontest_native_openai_opencode_keeps_its_default_unless_a_chat_model_is_set
( monkeypatch: pytest.MonkeyPatch, configured_model: str | None, expected: tuple[str, str], )
tests/test_provider_agent_routing.py:83
Functiontest_native_overview_displays_quality_threshold_failure_not_an_advisory
(tmp_path)
tests/ci/test_native_reports.py:285
Functiontest_native_parallel_tool_intervals_are_counted_once
()
tests/tier3/test_trajectory_timing.py:167
← previousnext →19,001–19,100 of 24,741, ranked by callers