MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_claude_code_task_list_calls_are_left_out_of_the_score
(module)
tests/tier3/test_tool_productivity.py:190
Functiontest_claude_code_vertex_exec_as_agent_unsets_anthropic_vars
Prepend defensive unset for Anthropic direct-API vars only when Vertex auth is active.
tests/tier3/test_harbor_local_agents.py:1139
Functiontest_claude_code_vertex_model_connection_and_auth_env
Suppress inferred Anthropic endpoint and forward Vertex AI env vars when CLAUDE_CODE_USE_VERTEX=1.
tests/tier3/test_harbor_local_agents.py:1096
Functiontest_claude_collects_every_member_kind_and_preserves_sources
(tmp_path)
tests/test_plugin_discovery.py:101
Functiontest_claude_effort_accepts_whole_number_floats_only
(tmp_path: Path)
tests/harness/test_overlay.py:1084
Functiontest_claude_error_marker_allows_quiet_negative_probe
(checker, claude_error_observation, command, line_ending)
tests/tier3/test_error_recovery.py:852
Functiontest_claude_error_marker_does_not_hide_failure
( checker, claude_error_observation, command, exit_code, extra, line_ending )
tests/tier3/test_error_recovery.py:880
Functiontest_claude_failed_read_cannot_receive_perfect_execution_credit
(grader: tuple[ModuleType, ModuleType])
tests/tier3/test_failed_skill_reads.py:70
Functiontest_claude_format_acceptance_matrix
(tmp_path: Path)
tests/harness/test_plugin_compat.py:59
Functiontest_claude_harbor_error_flags_mark_the_call_failed
()
tests/tier3/test_f90_mcp_execution.py:77
Functiontest_claude_ignored_field_values_are_noted
(tmp_path: Path)
tests/harness/test_overlay.py:1070
Functiontest_claude_init_event_confirms_listed_plugin_skills
(tmp_path: Path)
tests/tier3/test_plugin_execution.py:237
Functiontest_claude_inline_command_declarations_keep_packaged_commands
(tmp_path)
tests/test_plugin_discovery.py:482
Functiontest_claude_inline_plugin_needs_default_folders
(tmp_path: Path)
tests/harness/test_plugin_compat.py:91
Functiontest_claude_install_variable_defaults_are_paths_but_cannot_hide_credentials
(tmp_path: Path)
tests/tier3/test_private_runtime_regressions.py:259
Functiontest_claude_launch_keeps_pipe_and_gets_plugin_dir
()
tests/tier3/test_native_launch_insertion.py:53
Functiontest_claude_like_trajectory_without_wall_time_is_not_inferred
()
tests/tier3/test_trajectory_timing.py:87
Functiontest_claude_listing_cut_counts_when_to_use
(tmp_path: Path)
tests/harness/test_overlay.py:614
Functiontest_claude_markdown_declarations_supplement_packaged_defaults
(tmp_path, field_name, component_type, default_dir)
tests/test_plugin_discovery.py:439
Functiontest_claude_marketplace_missing_and_external_dependencies
(tmp_path)
tests/test_plugin_dependency_records.py:119
Functiontest_claude_mcp_servers_command_fallback_when_no_mcp_json
Fall back to base MCP registration command when no task mcp_servers.json exists.
tests/tier3/test_harbor_local_agents.py:1023
Functiontest_claude_mcp_servers_command_nested_logs_dir_layout
Resolve task config.json from trial root when logs_dir is nested as <trial>/logs/agent.
tests/tier3/test_harbor_local_agents.py:1044
Functiontest_claude_mcp_servers_command_resilient_to_missing_config_json
Handle missing or malformed config.json without raising errors.
tests/tier3/test_harbor_local_agents.py:1035
Functiontest_claude_mcp_servers_command_stdio
Build MCP configuration command with stdio transport, env, and command arguments.
tests/tier3/test_harbor_local_agents.py:959
Functiontest_claude_mcp_servers_command_with_mcp_json
Build MCP configuration command with streamable-http and headers from task mcp_servers.json.
tests/tier3/test_harbor_local_agents.py:917
Functiontest_claude_mcp_servers_preserves_task_runtime_env_args
Verify SkillEvaluatorClaudeCode extracts runtime_env from task.toml and validates MCP args.
tests/tier3/test_harbor_local_agents.py:1062
Functiontest_claude_mcp_servers_relative_task_path
Anchor relative task path to trial directory instead of host process CWD.
tests/tier3/test_harbor_local_agents.py:997
Methodtest_claude_opus_5_keeps_model_default_effort_and_no_fallback
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:1109
Functiontest_claude_orphan_result_flag_on_the_step_extra_marks_failure
()
tests/tier3/test_f90_mcp_execution.py:108
Functiontest_claude_placeholders_reach_other_harnesses_literally
(tmp_path: Path)
tests/harness/test_overlay.py:684
Functiontest_claude_quiet_negative_probe_is_not_successful_retry
(checker, claude_error_observation)
tests/tier3/test_error_recovery.py:892
Functiontest_claude_shared_slash_name_does_not_confirm_two_distinct_components
(has_skill_field: bool)
tests/tier3/test_native_review_fixes.py:115
Functiontest_claude_skill_argument_echo_is_not_the_task_prompt
()
tests/tier3/test_f90_signals_handoff_prompt.py:92
Functiontest_claude_stream_jsonl_tool_use_and_result
()
tests/tier3/test_log_converters.py:22
Functiontest_claude_user_config_reference_is_not_an_inline_credential
Regression: Claude Code fills '${user_config.KEY}' from the plugin's userConfig; it was read as a literal secret.
tests/validators/test_mcp_static.py:593
Functiontest_clean_actual_tier1_validators_produce_publication_execution_evidence
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_publication_evidence.py:145
Functiontest_clean_and_partial_scans_have_distinct_count_certainty
(incomplete)
tests/ci/test_summary_metrics.py:41
Methodtest_clean_file_passes
(self, tmp_path: Path)
tests/validators/test_unicode_smuggle.py:37
Methodtest_clean_mdc_has_no_pii
(self, tmp_path)
tests/validators/test_pii_grouping.py:49
Methodtest_clean_pass_is_not_flagged
(self)
tests/reporting/test_incomplete_scans.py:120
Functiontest_clean_run_with_enough_cases_and_a_negative_control_needs_no_next_step
(tmp_path: Path)
tests/test_tier3_next_steps.py:46
Methodtest_clean_scan_is_not_marked
(self, mock_run, skill_dir)
tests/validators/test_scan_incomplete.py:135
Methodtest_clean_script_no_findings
(self, skill_with_scripts)
tests/validators/test_script_lint.py:196
Methodtest_clean_skill_has_no_portability_findings
(self, tmp_path: Path)
tests/validators/test_schema.py:1570
Functiontest_clean_skill_loads_everywhere
(tmp_path: Path)
tests/harness/test_overlay.py:64
Functiontest_clean_skill_panel_is_one_line
(tmp_path: Path)
tests/harness/test_overlay.py:285
Methodtest_clean_skill_passes
Test validation passes for skill without security issues.
tests/validators/test_security.py:393
Methodtest_clean_skill_scanned_in_place
(self, mock_run, tmp_path)
tests/validators/test_artifact_dir_exclusion.py:156
Functiontest_clean_skip_is_distinct_from_pass
()
tests/reporting/test_shared_run_status.py:55
Functiontest_clean_stdio_command_passes
()
tests/validators/test_mcp_static.py:311
Functiontest_clean_tier2_skip_is_explicit_in_combined_outputs
()
tests/reporting/test_advisory_skip_verdict.py:378
Functiontest_cleanup_failure_degrades_terminal_progress_after_retention_finalizes
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:5183
Functiontest_cleanup_failure_leaves_only_private_exact_backup
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_secure_copy.py:1366
Functiontest_cleanup_failure_reports_actual_retention
(monkeypatch, tmp_path: Path)
tests/test_harbor_artifact_retention.py:40
Functiontest_cli_adds_json_block_and_keeps_exit_code
(tmp_path: Path)
tests/harness/test_overlay.py:227
Functiontest_cli_and_markdown_distinguish_explicit_na_from_missing_scores
()
tests/reporting/test_shared_run_status.py:196
Functiontest_cli_auth_diagnostic_reports_only_finite_wrapped_http_code
(monkeypatch, capsys)
tests/ci/test_feedback.py:1637
Functiontest_cli_auth_diagnostic_whitelists_classes_without_messages
(monkeypatch, capsys, exception_name, category)
tests/ci/test_feedback.py:1614
Functiontest_cli_auto_trace_fallback_records_template_mode_and_preserves_sources
( tmp_path, monkeypatch, existing_authored, provider_behavior, expected_reason, )
tests/tier3/test_dataset_auto_grounding.py:727
Functiontest_cli_autopilot_rejects_symlinked_evals_directory
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:755
Functiontest_cli_autopilot_reuses_existing_dataset_unchanged_and_preserves_skip_baseline
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:563
Functiontest_cli_autopilot_uses_deterministic_one_case_generation_without_provider_key
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:646
Functiontest_cli_autopilot_uses_keyed_llm_one_case_generation_and_forwards_options
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:592
Functiontest_cli_autopilot_warns_and_falls_back_when_llm_generation_errors
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:672
Functiontest_cli_checks_gym_settings_before_autopilot_writes_a_dataset
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/test_nemo_gym_backend.py:2088
Functiontest_cli_classifies_a_real_fern_only_diff
(git_repo: tuple[Path, str], tmp_path: Path)
tests/test_ci_change_classifier.py:398
Functiontest_cli_enforcement_flag_preserves_failed_evidence
(tmp_path, mode, expected)
tests/ci/test_run_tier1.py:206
Functiontest_cli_entry_limit_fails_without_embedding_or_partial_catalog
(tmp_path: Path, monkeypatch)
tests/embedding/test_similarity_limits.py:146
Functiontest_cli_environment_drops_provider_cloud_and_proxy_auth
(tmp_path)
tests/validators/test_private_format_boundaries.py:127
Functiontest_cli_environment_kwargs_allow_kubernetes_secret_object_reference
()
tests/tier3/test_evals_config.py:501
Functiontest_cli_environment_kwargs_allow_non_secret_key_names
()
tests/tier3/test_evals_config.py:497
Functiontest_cli_environment_kwargs_allow_provider_secret_references
(env_mode, entry, expected)
tests/tier3/test_evals_config.py:588
Functiontest_cli_environment_kwargs_malformed_syntax_exits_code_2_without_traceback
Verify malformed --ek arguments produce clean Click BadParameter exit code 2 without tracebacks.
tests/test_cli.py:1434
Functiontest_cli_environment_kwargs_reject_extreme_json_nesting_without_traceback
()
tests/tier3/test_evals_config.py:546
Functiontest_cli_environment_kwargs_reject_image_pull_secret_outside_ack_or_invalid_kubernetes_names
( env_mode: str, reference: str, )
tests/tier3/test_evals_config.py:518
Functiontest_cli_environment_kwargs_reject_secret_reference_fields_outside_exact_harbor_shapes
(env_mode, entry)
tests/tier3/test_evals_config.py:601
Functiontest_cli_environment_kwargs_reject_secret_value_patterns_without_echoing
(entry, secret)
tests/tier3/test_evals_config.py:649
Functiontest_cli_environment_kwargs_reject_secrets_without_echoing_values
(entry, secret)
tests/tier3/test_evals_config.py:490
Functiontest_cli_environment_kwargs_sensitive_credential_exits_code_2_without_traceback
Verify credential keys passed to --ek produce clean Click BadParameter exit code 2.
tests/test_cli.py:1459
Functiontest_cli_evaluate_accepts_harbor_native_cloud_environment
(monkeypatch: pytest.MonkeyPatch)
tests/test_evaluation_service.py:819
Functiontest_cli_evaluate_fails_closed_on_unusable_engine_result
( monkeypatch: pytest.MonkeyPatch, engine_result: object, expected: str, )
tests/test_evaluation_service.py:945
Functiontest_cli_evaluate_rejects_an_empty_explicit_agent_list
(monkeypatch: pytest.MonkeyPatch)
tests/test_evaluation_service.py:123
Functiontest_cli_evaluate_renders_persisted_score_and_report_truth
(monkeypatch: pytest.MonkeyPatch, tmp_path: Path)
tests/test_evaluation_service.py:839
Functiontest_cli_evaluate_renders_real_command_failure_before_nonzero_exit
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:869
Functiontest_cli_evaluate_uses_shared_service
(monkeypatch: pytest.MonkeyPatch)
tests/test_evaluation_service.py:157
Functiontest_cli_evaluate_validates_cost_price_pair_after_config_merge
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:342
Functiontest_cli_exports_and_read_only_results
(tmp_path, suffix)
tests/test_tier3_history.py:184
Functiontest_cli_failure_diagnostic_uses_only_trusted_line_and_finite_category
( gitlab, monkeypatch, capsys, failure, category )
tests/ci/test_feedback.py:1550
Functiontest_cli_failure_keeps_compact_artifacts_as_absolute_footer
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_evaluation_service.py:902
Functiontest_cli_findings_body_keeps_same_source_path_only_refs_distinct
()
tests/tier3/test_report_renders_refs.py:112
Functiontest_cli_findings_body_renders_evidence_pointer
()
tests/tier3/test_report_renders_refs.py:82
Functiontest_cli_findings_body_renders_normalized_tool_compact_identity
()
tests/tier3/test_report_renders_refs.py:88
Functiontest_cli_findings_include_custom_metric_details
()
tests/tier3/test_report_renders_refs.py:139
Functiontest_cli_formats_dataset_generation_failure_without_raw_exception
( tmp_path: Path, command: tuple[str, ...], )
tests/test_evaluation_service.py:48
Functiontest_cli_formats_dataset_option_dependency_failure
( tmp_path: Path, dependent_option: str, )
tests/test_evaluation_service.py:63
Functiontest_cli_forwards_skip_with_skill
(skill: Path, captured_options: list[Any])
tests/tier3/test_skip_with_skill.py:115
Functiontest_cli_grading_mode_accepts_legacy_aliases
()
tests/test_tier3_config_key_parity.py:459
Functiontest_cli_harbor_job_labels_remain_neutral_and_literal
(colored)
tests/reporting/test_reporters.py:3124
Functiontest_cli_harness_version_flag
(tmp_path: Path)
tests/harness/test_overlay.py:1126
Functiontest_cli_incomplete_is_nonzero_and_machine_readable
(monkeypatch, tmp_path, failure)
tests/tier3/test_interactions.py:713
← previousnext →13,901–14,000 of 24,741, ranked by callers