MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_load_evals_config_yaml_errors_give_position_without_source_text
(tmp_path)
tests/tier3/test_evals_config.py:224
Functiontest_load_evals_config_yaml_errors_keep_the_pyyaml_context
(tmp_path, document, detail)
tests/tier3/test_evals_config.py:256
Methodtest_load_fails_closed_if_path_changes_after_descriptor_open
(self, tmp_path: Path, monkeypatch)
tests/embedding/test_catalog.py:381
Functiontest_load_failure_is_excluded_from_each_affected_pair
()
tests/tier3/test_private_comparison_contracts.py:262
Functiontest_load_failures_exclude_the_case_from_the_ladder_and_the_glue
()
tests/tier3/test_private_ladder_stats.py:330
Functiontest_load_falls_back_to_cursor_txt
(tmp_path: Path)
tests/tier3/test_log_converters.py:70
Functiontest_load_falls_back_to_opencode_txt
(tmp_path: Path)
tests/tier3/test_log_converters.py:422
Functiontest_load_inventory_bounds_report_all_withheld_rows
()
tests/reporting/test_private_runtime_projection.py:266
Functiontest_load_prefers_claude_log_over_cursor_when_both
(tmp_path: Path)
tests/tier3/test_log_converters.py:80
Functiontest_load_prefers_trajectory_json
(tmp_path: Path)
tests/tier3/test_log_converters.py:57
Methodtest_load_rejects_device_catalog
(self)
tests/embedding/test_catalog.py:579
Methodtest_load_rejects_duplicate_ids
(self, tmp_path: Path)
tests/embedding/test_catalog.py:268
Methodtest_load_rejects_duplicate_json_keys
(self, tmp_path: Path)
tests/embedding/test_catalog.py:275
Methodtest_load_rejects_fifo_without_blocking
(self, tmp_path: Path)
tests/embedding/test_catalog.py:550
Methodtest_load_rejects_incompatible_metadata
(self, tmp_path: Path, field: str, value: object, message: str)
tests/embedding/test_catalog.py:237
Methodtest_load_rejects_malformed_entries
(self, tmp_path: Path, mutation, message: str)
tests/embedding/test_catalog.py:260
Methodtest_load_rejects_overflow_magnitude_vector
(self, tmp_path: Path)
tests/embedding/test_catalog.py:347
Methodtest_load_rejects_socket_catalog
(self)
tests/embedding/test_catalog.py:584
Methodtest_load_rejects_surrogate_and_control_strings
(self, tmp_path: Path, unsafe: str)
tests/embedding/test_catalog.py:296
Methodtest_load_rejects_symlinked_intermediate_directory
(self, tmp_path: Path)
tests/embedding/test_catalog.py:481
Methodtest_load_rejects_unknown_schema_fields
(self, tmp_path: Path, scope: str)
tests/embedding/test_catalog.py:285
Methodtest_load_save_cache_roundtrip
(self, tmp_path: Path)
tests/embedding/test_registry.py:531
Functiontest_load_utf8_dataset_under_non_utf8_locale
(tmp_path: Path, suffix: str)
tests/tier3/test_dataset_utils.py:21
Functiontest_loaded_catalog_rejects_over_budget_work_before_normalizing
( tmp_path: Path, monkeypatch, operation: str )
tests/embedding/test_similarity_limits.py:271
Functiontest_loaded_with_skill_only_report_has_no_phantom_baseline
(tmp_path, monkeypatch, baseline_status)
tests/reporting/test_tier3_expectations.py:264
Functiontest_loader_accepts_an_explicit_absolute_catalog_root
(selection: SimpleNamespace, tmp_path: Path)
tests/tier3/test_decoys.py:330
Functiontest_loader_builds_context_from_a_real_run
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:1366
Functiontest_loader_diagnostics_do_not_echo_secret_values_or_keys
(payload, tmp_path)
tests/tier3/test_external_metrics.py:350
Functiontest_loader_diagnostics_hide_metric_definition_keys
(payload, tmp_path)
tests/tier3/test_external_metrics.py:370
Functiontest_loader_diagnostics_include_observation_field_path
(payload, tmp_path)
tests/tier3/test_external_metrics.py:362
Functiontest_loader_keeps_usage_of_attempts_that_scores_never_read
(tmp_path: Path)
tests/reporting/test_tier3_token_usage_reports.py:204
Functiontest_loader_normalizes_author_fields
(skill)
tests/tier3/test_dataset_coverage.py:79
Functiontest_loader_preserves_hash_and_rejects_duplicate_keys
(payload, tmp_path)
tests/tier3/test_external_metrics.py:216
Functiontest_loader_reads_frontmatter_as_selection_does
( selection: SimpleNamespace, tmp_path: Path, changed: str, frontmatter: str, match: str )
tests/tier3/test_decoys.py:671
Functiontest_loader_rejects_a_hand_edited_nested_target_selection
(selection: SimpleNamespace, tmp_path: Path)
tests/tier3/test_decoys.py:188
Functiontest_loader_rejects_a_renamed_target_alias_with_a_correct_source_digest
( selection: SimpleNamespace, tmp_path: Path, )
tests/tier3/test_decoys.py:542
Functiontest_loader_rejects_malformed_or_unbounded_input
(tmp_path, raw)
tests/tier3/test_external_metrics.py:228
Functiontest_loader_rejects_removing_the_unsafe_action_even_when_quoted_evidence_remains
(pilot_copy)
tests/tier3/test_grader_pilot.py:75
Functiontest_loader_rejects_target_payload_with_a_correct_source_digest
( selection: SimpleNamespace, tmp_path: Path, )
tests/tier3/test_decoys.py:560
Functiontest_loader_reuses_only_run_bound_feedback_without_new_judge_calls
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, run_id: str, schema_version: str, restored: bool )
tests/evaluation/test_tier3_feedback_chat.py:1334
Functiontest_loader_truncation_reaches_the_canonical_payload
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_report_data_bounds.py:755
Functiontest_local_agent_credentials_map_provider_to_agent_env
()
tests/test_harbor_local_mode.py:1686
Functiontest_local_agents_do_not_retry_a_missing_host_cli
(tmp_path: Path, agent_class: type, sleeps: list[float])
tests/tier3/test_installed_agent_install_retry.py:281
Functiontest_local_agents_keep_existing_version_check_without_install
(tmp_path: Path, agent_class: type)
tests/tier3/test_installed_agent_bootstrap.py:348
Functiontest_local_anthropic_opencode_is_rejected_explicitly
()
tests/test_harbor_runner_environment.py:1108
Functiontest_local_artifact_collection_retains_source_and_separate_step_snapshots
( local_output_environment: SkillEvaluatorLocalEnvironment, )
tests/test_harbor_local_environment.py:273
Functiontest_local_artifact_parent_paths_publish_and_collect_step_snapshots
Computing the artifact child from /logs must reach the publish directory.
tests/test_harbor_local_environment.py:139
Functiontest_local_autopilot_offers_missing_runtime_before_environment_check
( monkeypatch, tmp_path, command, invalid_options )
tests/test_evaluation_service.py:454
Functiontest_local_backend_keeps_private_local_agents
(tmp_path, agent, path)
tests/test_private_runtime_rereview.py:74
Functiontest_local_bedrock_agent_is_rejected_explicitly
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_runner_environment.py:1119
Functiontest_local_bootstrap_installs_only_the_selected_native_runtime
(monkeypatch)
tests/ci/test_all_tier_preflight.py:58
Functiontest_local_bootstrap_installs_the_opencode_release_docker_runs_pin
(monkeypatch)
tests/ci/test_all_tier_preflight.py:88
Functiontest_local_bootstrap_runs_the_pinned_install_that_local_setup_offers
(monkeypatch)
tests/ci/test_all_tier_preflight.py:101
Functiontest_local_bridge_command_uses_custom_agent_import_path
()
tests/test_tier3_public_runtime.py:658
Functiontest_local_callback_error_message_receives_only_redacted_output
(tmp_path: Path)
tests/test_harbor_local_mode.py:2607
Functiontest_local_callback_error_remains_primary_when_cleanup_reports_failure
(tmp_path: Path)
tests/test_harbor_local_mode.py:2558
Functiontest_local_callback_exception_is_propagated_after_process_reap
( tmp_path: Path, error_type: type[BaseException], )
tests/test_harbor_local_mode.py:2502
Functiontest_local_callback_receives_complete_line_before_process_exit
(tmp_path: Path)
tests/test_harbor_local_mode.py:2355
Functiontest_local_callback_streams_safe_partial_line_before_process_exit
(tmp_path: Path)
tests/test_harbor_local_mode.py:2377
Functiontest_local_claude_code_runs_keep_effort_without_a_cli_version
()
tests/test_tier3_public_runtime.py:2883
Functiontest_local_claude_does_not_rewrite_instruction_permission_text
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:399
Functiontest_local_claude_instruction_env_preserves_literal_prompt_paths
(tmp_path: Path)
tests/test_harbor_local_mode.py:1962
Functiontest_local_claude_uses_managed_permissions_and_trial_temp_dir
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_mode.py:654
Functiontest_local_claude_uses_managed_policy_permission_mode
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:378
Functiontest_local_codex_final_config_and_launcher_preserve_explicit_gateway_routing
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/tier3/test_harbor_local_agents.py:200
Functiontest_local_codex_final_config_preserves_user_and_harbor_mcp_servers
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/tier3/test_harbor_local_agents.py:226
Functiontest_local_codex_keeps_short_model_without_gateway
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:744
Functiontest_local_codex_preserves_full_gateway_model_name
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:720
Functiontest_local_codex_preserves_instruction_text_during_launcher_rewrites
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:767
Functiontest_local_codex_rewrites_upstream_tmp_secrets_dir
(monkeypatch, tmp_path)
tests/tier3/test_harbor_local_agents.py:352
Functiontest_local_codex_runtime_provider_fields_win_without_losing_unrelated_user_config
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/tier3/test_harbor_local_agents.py:263
Functiontest_local_codex_uploads_openai_responses_config_and_creates_per_trial_state_dirs
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/tier3/test_harbor_local_agents.py:167
Functiontest_local_codex_uses_per_trial_codex_home
()
tests/tier3/test_harbor_local_agents.py:160
Functiontest_local_concurrent_callback_contexts_are_isolated
(tmp_path: Path)
tests/test_harbor_local_mode.py:2298
Functiontest_local_environment_rejects_missing_or_unsupported_runtime_agent
( monkeypatch: pytest.MonkeyPatch, runtime_agent: str | None, )
tests/test_harbor_local_mode.py:5006
Functiontest_local_environment_requires_runtime_agent_keyword
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_local_mode.py:5016
Functiontest_local_environment_rewrites_read_only_uploaded_script_once
A read-only template is made writable without rewriting its local path twice.
tests/test_harbor_local_environment.py:373
Functiontest_local_exec_without_callback_uses_bounded_stream_collector
(tmp_path: Path)
tests/test_harbor_local_mode.py:2441
Functiontest_local_exec_writes_and_reads_logical_artifact_paths
( local_output_environment: SkillEvaluatorLocalEnvironment, )
tests/test_harbor_local_environment.py:115
Functiontest_local_git_operations_do_not_enable_network_or_credentials
(monkeypatch, tmp_path)
tests/test_ci_onboarding_git.py:90
Functiontest_local_host_env_excludes_provider_keys_by_default
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_local_mode.py:1744
Functiontest_local_host_env_inherits_on_opt_in
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_local_mode.py:1753
Functiontest_local_install_records_a_cli_version_that_supports_the_model
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, agent_class: type, model: str, stdout:
tests/tier3/test_harbor_local_agents.py:454
Functiontest_local_install_rejects_a_cli_too_old_for_the_model
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, agent_class: type, model: str, stdout:
tests/tier3/test_harbor_local_agents.py:493
Functiontest_local_instruction_env_does_not_allow_unrecognized_harbor_values
(tmp_path: Path, name: str)
tests/test_harbor_local_mode.py:1956
Functiontest_local_instruction_env_preserves_protected_paths
(tmp_path: Path, name: str)
tests/test_harbor_local_mode.py:1917
Functiontest_local_instruction_env_rejects_malformed_claude_channels
(tmp_path: Path, name: str)
tests/test_harbor_local_mode.py:1987
Functiontest_local_is_a_registered_env_mode
()
tests/test_harbor_local_mode.py:500
Functiontest_local_json_over_limit_is_a_notice_and_other_reports_load
(tmp_path: Path, monkeypatch)
tests/reporting/test_dashboard_data.py:992
Methodtest_local_markdown_link_remains_a_nested_reference
(self, tmp_path: Path)
tests/validators/test_quality_score.py:1375
Functiontest_local_mode_agent_commands_keep_background_tasks_disabled
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/test_harbor_claude_background_tasks.py:91
Functiontest_local_mode_agent_commands_keep_refusal_fallback_disabled
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/test_harbor_claude_refusal_fallback.py:491
Functiontest_local_mode_allows_other_agents_when_vertex_flag_is_present
Allow non-Claude agents in local mode even if CLAUDE_CODE_USE_VERTEX=1 is set in environment.
tests/test_harbor_gke_mode.py:380
Functiontest_local_mode_notice_precedes_progress_configuration
(monkeypatch: pytest.MonkeyPatch)
tests/test_evaluation_service.py:778
Functiontest_local_mode_strictly_rejects_vertex_ai
Following Bedrock precedent, local mode with Vertex AI is strictly rejected.
tests/test_harbor_gke_mode.py:320
Functiontest_local_mode_strictly_rejects_vertex_ai_under_nv_build
Enforce strict rejection of Vertex AI in local mode even when using nv_build provider.
tests/test_harbor_gke_mode.py:366
Functiontest_local_nested_callbacks_run_in_scope_order
(tmp_path: Path)
tests/test_harbor_local_mode.py:2327
Functiontest_local_nonzero_exit_redacts_detached_proxy_components_from_callback_and_result
( tmp_path: Path, proxy_uri: str, )
tests/test_harbor_local_mode.py:2103
Functiontest_local_nvidia_build_bridge_cleanup_preserves_cancellation
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_mode.py:848
Functiontest_local_nvidia_build_bridge_closes_if_cancelled_during_startup
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_mode.py:789
← previousnext →18,101–18,200 of 24,741, ranked by callers