MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Methodtest_strict_reads_allow_home_descendant_but_reject_home_ancestor
( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_sandbox.py:966
Methodtest_strict_reads_allow_selected_symlinked_runtime
(self, run_root: Path, tmp_path: Path)
tests/test_harbor_local_sandbox.py:1525
Methodtest_strict_reads_block_host_tmp
(self, run_root: Path)
tests/test_harbor_local_sandbox.py:1481
Methodtest_strict_reads_filter_alias_to_shared_tree_but_allow_alias_to_narrow_runtime
( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_sandbox.py:948
Methodtest_strict_reads_filter_shared_tree_roots_but_keep_narrow_descendants
( self, monkeypatch: pytest.MonkeyPatch, broad_root: str, )
tests/test_harbor_local_sandbox.py:934
Methodtest_strict_reads_rejects_root_that_contains_host_home
( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_sandbox.py:895
Functiontest_strict_reads_stop_words_only_at_the_start_of_an_error_page
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:406
Methodtest_strict_reads_use_granular_usr_roots
(self, tmp_path: Path)
tests/test_harbor_local_sandbox.py:682
Functiontest_strict_refusal_never_switches_models
(verifier, monkeypatch, reply)
tests/tier3/test_judge_policies.py:602
Functiontest_strict_rejection_is_not_retried_or_handed_on
(verifier, monkeypatch, code, body)
tests/tier3/test_judge_policies.py:384
Functiontest_strict_retries_capacity_once_then_records_the_model_that_answered
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:310
Functiontest_strict_retry_after_above_the_maximum_delay_moves_on_without_waiting
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:659
Functiontest_strict_retry_after_never_waits_past_the_judge_budget
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:644
Methodtest_strict_runtime_outside_home_does_not_expose_ancestor_siblings
(self, run_root: Path)
tests/test_harbor_local_sandbox.py:1583
Functiontest_strict_runtime_ro_binds_use_exact_interpreter_and_python_library_roots
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_mode.py:5082
Functiontest_strict_schema_downgrade_counts_toward_the_request_limit
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:669
Functiontest_strict_stop_keeps_the_errors_of_earlier_models
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:394
Functiontest_strict_stop_words_in_a_connection_error_do_not_end_the_judge
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:565
Functiontest_strict_stop_words_in_an_echoed_model_id_do_not_end_the_judge
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:555
Functiontest_strict_structured_stop_errors_end_the_judge
(verifier, monkeypatch, body)
tests/tier3/test_judge_policies.py:585
Functiontest_strict_tries_at_most_three_models_twice_each
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:322
Methodtest_strict_usr_local_style_npm_symlink_keeps_alias_and_package_exact
( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_sandbox.py:529
Functiontest_strict_validation_uses_the_directory_and_frontmatter_identity
(tmp_path, declared_name, identity)
tests/tier3/test_dataset_baseline_fairness.py:1653
Functiontest_strict_whole_object_keeps_the_completed_verdict
(verifier, monkeypatch, achieved)
tests/tier3/test_judge_policies.py:706
Functiontest_string_and_numeric_error_flags_mark_failure
(flag: Any)
tests/tier3/test_f90_mcp_execution.py:121
Functiontest_string_and_selector_refs_share_one_canonical_id
(refs: list, canonical: str)
tests/test_plugin_refs.py:42
Methodtest_string_arguments_are_decoded
(self)
tests/tier3/test_plugin_signals.py:569
Methodtest_string_extra_frontmatter_key_is_preserved
(self, validator, valid_rule_file)
tests/validators/test_rules_schema.py:110
Methodtest_string_is_wrapped
(self)
tests/test_harbor_input_staging.py:72
Methodtest_string_returns_single_item_list
Test that string input returns list with one item.
tests/models/test_field_validators.py:19
Methodtest_string_severity_is_coerced
(self)
tests/models/test_result.py:57
Functiontest_string_skip_reasons_are_bounded_across_reporters
()
tests/reporting/test_advisory_skip_verdict.py:353
Functiontest_strip_arm_and_attempt_suffixes
Verify _strip_arm_and_attempt_suffixes removes both suffixes regardless of ordering.
tests/test_tier3_public_runtime.py:2113
Functiontest_strip_arm_suffix
Verify _strip_arm_suffix trims exact dual-arm suffixes.
tests/test_tier3_public_runtime.py:2093
Methodtest_strips_frontmatter_from_markdown
(self, skill_root: Path)
tests/deduplication/utils/test_skill_collector.py:95
Methodtest_strips_markdown_fences
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:822
Functiontest_structural_reward_fallback_bounds_identity_and_model_before_publication
(tmp_path: Path)
tests/test_harbor_collector_multistep_trajectory.py:1281
Functiontest_structured_action_keeps_its_evidence_and_verification_check
()
tests/reporting/test_insights_view.py:216
Functiontest_structured_config_and_directory_permissions_are_part_of_capability_parity
(tmp_path, kind, mode, suffix)
tests/tier3/test_private_comparison_staging.py:57
Methodtest_structured_error_flag_marks_failure
(self)
tests/tier3/test_plugin_signals.py:421
Methodtest_structured_evidence_names_every_reason_and_keeps_the_scanner_remedy
(self)
tests/validators/test_skillspector_reference_diagnostics.py:165
Functiontest_structured_failure_logs_and_annotations_are_escaped
(capsys)
tests/ci/test_run_tier1.py:400
Functiontest_structured_judges_pass_anthropic_output_config_schema
Verify Anthropic judge calls pass output_config.format.json_schema instead of response_format.
tests/test_llm_judge.py:429
Functiontest_structured_judges_pass_openai_response_format_schema
Verify OpenAI/compatible judge calls pass strict json_schema in both shared judge and template.
tests/test_llm_judge.py:384
Functiontest_structured_opencode_resource_exhaustion_invalidates_no_trajectory_reward
A provider error event is an agent failure even when Harbor exits cleanly.
tests/test_harbor_collector_runtime_failures.py:2486
Methodtest_structured_target_outside_the_scan_is_dropped
(self, target: str)
tests/validators/test_skillspector_reference_diagnostics.py:485
Functiontest_structured_tool_and_provenance_secrets
()
tests/test_trace_import.py:485
Functiontest_subagent_prompt_written_by_the_parent_is_not_the_task_prompt
check-18 e04 (Harbor 0.20+ shape): a sidechain user step is the parent's own words.
tests/tier3/test_f90_signals_handoff_prompt.py:139
Functiontest_subagent_tools_bash_is_advisory_and_not_more_severe_than_omitting_tools
(tmp_path: Path)
tests/validators/test_plugin_static_risk_review_fixes.py:780
Methodtest_submitter_detected_from_environment
The submitter identity is auto-detected from the environment (e.g. CI).
tests/validators/test_security.py:1165
Methodtest_submitter_home_directory_flagged
A /home/<submitter>/ path is flagged because it leaks the submitter's identity.
tests/validators/test_security.py:1141
Functiontest_subprocess_file_not_found_is_missing_executable_but_open_is_not
(checker)
tests/tier3/test_error_recovery.py:132
Functiontest_subshell_cd_option_or_attached_semicolon_still_reads_the_script
Regression: '(cd', 'cd --', and a ';' attached to a word hid the plugin script, and the relative script was reported as resolving against the user
tests/validators/test_plugin_component_risk.py:1023
Functiontest_subtabs_and_unusual_agent_names_have_complete_relationships
()
tests/reporting/test_report_navigation.py:94
Functiontest_succeeded_agent_comparison_withholds_failed_arm
(failed_arm: str)
tests/tier3/test_baseline_timeouts.py:298
Functiontest_succeeded_arm_aggregates_are_unchanged
(tmp_path: Path, status: str)
tests/reporting/test_failed_arm_trials.py:300
Functiontest_succeeded_arm_rewards_without_trial_id_load_unchanged
Only a withheld arm needs a stand-in identity; a succeeded arm's legacy rewards stay identity-less.
tests/reporting/test_failed_arm_trials.py:184
Functiontest_succeeded_baseline_condition_with_complete_counts_can_certify_publication
()
tests/reporting/test_advisory_skip_verdict.py:3087
Functiontest_succeeded_baseline_requires_score_evidence
()
tests/reporting/test_advisory_skip_verdict.py:3157
Functiontest_succeeded_evidence_requires_explicit_empty_execution_errors
( tmp_path: Path, error_location: str, )
tests/reporting/test_advisory_skip_verdict.py:2952
Functiontest_succeeded_evidence_with_invalid_execution_errors_cannot_certify_publication
( tmp_path: Path, error_location: str, invalid_errors: object, )
tests/reporting/test_advisory_skip_verdict.py:2934
Functiontest_succeeded_under_scored_attempts_cannot_certify_publication
(tmp_path: Path)
tests/reporting/test_advisory_skip_verdict.py:2845
Methodtest_success_metadata_uses_default_text_color
(self, colored: bool)
tests/reporting/test_reporters.py:380
Functiontest_success_output_has_scores_report_and_elapsed
(tmp_path: Path)
tests/test_tier3_result_display.py:710
Functiontest_success_publishes_before_cleanup_and_activates_hooks_only_inside_the_run
(tmp_path: Path)
tests/tier3/test_plugin_execution.py:48
Methodtest_success_rate_excludes_unknown_and_covers_every_server
(self)
tests/tier3/test_plugin_signals.py:854
Methodtest_success_result
ToolResult captures successful execution.
tests/utils/test_tool_runner.py:675
Functiontest_success_summary_keeps_valid_numeric_zero_quality_and_lift
()
tests/reporting/test_terminal_polish.py:654
Functiontest_success_summary_omits_unusable_lift_evidence_without_changing_payload
(value)
tests/reporting/test_terminal_polish.py:636
Functiontest_success_summary_omits_unusable_quality_evidence_without_changing_payload
(value)
tests/reporting/test_terminal_polish.py:613
Functiontest_successful_agent_with_legitimate_zero_score_renders_numeric_zero
(tmp_path: Path)
tests/test_harbor_html_report_failures.py:575
Functiontest_successful_best_agent_does_not_import_failed_peer_configuration
()
tests/reporting/test_native_overview_comparison.py:107
Functiontest_successful_build_replaces_partial_index_completely
(tmp_path: Path, selection: str)
tests/embedding/test_collection_coverage.py:237
Functiontest_successful_collection_does_not_materialize_job_result_as_a_trial
(tmp_path: Path)
tests/test_issue55_collector_truth.py:4128
Functiontest_successful_custom_only_run_keeps_custom_reward_labels
(tmp_path: Path)
tests/test_harbor_html_report_failures.py:611
Functiontest_successful_delivery_and_zero_refusal_counts_do_not_emit_active_warnings
()
tests/reporting/test_native_evidence.py:181
Methodtest_successful_evaluation
(self, tmp_path)
tests/validators/test_rubric_eval.py:766
Functiontest_successful_execution_with_missing_evidence_is_not_a_complete_report
(scope, expected)
tests/reporting/test_private_runtime_projection.py:154
Functiontest_successful_explicit_rebuild_replaces_incomplete_selection
(tmp_path: Path)
tests/embedding/test_collection_coverage.py:159
Functiontest_successful_gym_teardown_completes_cleanup_progress
(tmp_path: Path, interrupt: bool)
tests/test_nemo_gym_backend.py:1559
Methodtest_successful_llm_enrichment_cannot_erase_deterministic_findings
( self, mock_tools, sample_skill_dir: Path, )
tests/validators/test_security.py:1827
Functiontest_successful_or_unobserved_skill_lookup_is_preserved
(grader, content)
tests/tier3/test_failed_skill_reads.py:215
Functiontest_successful_output_is_not_failure
(checker, observation)
tests/tier3/test_error_recovery.py:524
Functiontest_successful_positive_routing_does_not_hide_unmet_script_requirement
()
tests/reporting/test_tier3_expectations.py:171
Functiontest_successful_rollup_preserves_all_counts
(tmp_path)
tests/ci/test_run_tier1.py:530
Functiontest_successful_run_with_failed_summary_still_rejected_without_advisory_downgrade_marker
(tmp_path)
tests/ci/test_publish_reports.py:1154
Functiontest_successful_scoring_does_not_turn_missing_runtime_evidence_into_complete_report
(tmp_path, monkeypatch, missing)
tests/tier3/test_private_runtime_report_acceptance.py:291
Functiontest_successful_skill_content_can_discuss_errors
(grader: tuple[ModuleType, ModuleType])
tests/tier3/test_failed_skill_reads.py:83
Methodtest_successful_validator_sends_source_and_complete_guidance_to_provider
(self, tmp_path, line_ending)
tests/validators/test_rubric_eval.py:568
Functiontest_suggestion_prompt_and_citation_keep_same_case_role_variants
(monkeypatch, make_reward, cited)
tests/tier3/test_suggestion_grounding.py:276
Functiontest_suggestion_prompt_guards_requirements_and_labels_hypotheses
(monkeypatch)
tests/tier3/test_suggestion_grounding.py:792
Functiontest_suggestion_reader_accepts_old_and_new_artifacts
(tmp_path, include_thresholds)
tests/tier3/test_findings_severity.py:80
Functiontest_suggestions_drop_evidence_refs_the_run_did_not_record
A cited ref that the run did not record is dropped instead of rendered as evidence.
tests/tier3/test_suggestion_grounding.py:728
Functiontest_suggestions_evidence_lookup_prefers_custom_detail_for_custom_metric_collision
(monkeypatch, metric_set)
tests/tier3/test_suggestion_grounding.py:826
Functiontest_suggestions_evidence_refs_lookup_uses_all_metrics
Lookup is built from ALL metrics' evidence_refs, not just goal_accuracy.
tests/tier3/test_suggestion_grounding.py:806
Functiontest_suggestions_evidence_refs_resolved_to_full_dict
LLM returns string ref; function resolves it to the full dict from rewards.
tests/tier3/test_suggestion_grounding.py:701
Functiontest_suggestions_keep_one_copy_of_a_repeated_suggestion
(monkeypatch)
tests/tier3/test_suggestion_grounding.py:774
Functiontest_suggestions_resolve_legacy_evidence_ref_shapes
(monkeypatch, raw_refs)
tests/tier3/test_suggestion_grounding.py:754
Functiontest_suggestions_structured_evidence_refs_are_dicts_not_strings
End-to-end: suggestions_v2 artifacts must have dict refs, never plain strings.
tests/tier3/test_suggestion_grounding.py:965
Functiontest_suite_accepts_declarative_entry_checks
(suite_dir)
tests/tier3/test_grader_validation.py:79
← previousnext →22,901–23,000 of 24,741, ranked by callers