MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_error_recovery_is_not_clean_when_codex_observation_evidence_is_untrusted
( extractor, checker, source, unsupported_evidence )
tests/tier3/test_codex_tool_call_normalization.py:856
Functiontest_error_recovery_penalties_match_the_check
The graduated error_recovery ladder quotes real floors and coefficients.
tests/test_metrics_reference_docs.py:172
Functiontest_error_row_with_rejected_json_gets_a_native_unavailable_report
(tmp_path, monkeypatch)
tests/ci/test_publish_reports.py:529
Functiontest_error_summary_keeps_distinct_trace_associations_across_repeated_projection
()
tests/reporting/test_insights_view.py:304
Methodtest_error_terms_still_record_error_handling
(self, tmp_path: Path)
tests/validators/test_quality_score.py:1014
Functiontest_errored_count_without_matching_trials_is_refused
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:640
Functiontest_errored_job_stats_suppress_rewards_without_trial_exception
Aggregate Harbor failure state wins even when a trial reward looks valid.
tests/test_harbor_collector_runtime_failures.py:771
Functiontest_errored_trial_has_unknown_activation
(tmp_path: Path, artifact: str)
tests/reporting/test_failed_arm_trials.py:197
Functiontest_errors_abort_all_candidates_before_copying
(workspace)
tests/test_trace_import.py:286
Methodtest_errors_stay_only_for_an_incomplete_scan_and_a_gate_failure_stays_once
(self, incomplete: bool)
tests/models/test_result.py:228
Functiontest_escape_decoding_is_linear_on_a_long_backslash_run
()
tests/validators/test_plugin_hook_risk_bounds.py:93
Functiontest_escape_heavy_payload_has_bounded_html_and_one_recoverable_canonical_copy
()
tests/reporting/test_unified_tier3_report.py:2640
Functiontest_escape_named_folders_after_a_folder_named_n_r_or_t_are_folders
(text: str)
tests/evaluation/test_tier3_feedback_chat.py:764
Functiontest_escaped_space_destination_does_not_produce_link
()
tests/validators/test_markdown.py:269
Functiontest_escaped_text_renders_literally_between_plain_text
()
tests/utils/test_rich_markup.py:67
Functiontest_escaped_text_renders_literally_between_tags
(text: str)
tests/utils/test_rich_markup.py:63
Functiontest_escaping_declared_roots_fail_closed
(tmp_path: Path, declared: str, problem: str)
tests/test_openclaw_plugin_discovery.py:228
Functiontest_escaping_does_not_disturb_trusted_status_markup
()
tests/reporting/test_rich_markup_escaping.py:234
Functiontest_estimate_budget_line_remains_honest_with_missing_reservation_details
(reservation)
tests/tier3/test_budget_reservation.py:194
Functiontest_eval_and_inline_code_may_expand_a_quoted_variable
``eval`` reads its words again in this shell, with the command's prefix in effect, so a ``$f`` quoted from the first reading names the script ther
tests/tier3/test_script_invocation_evidence.py:2074
Functiontest_eval_dataset_file_must_be_regular_and_unlinked
(tmp_path: Path, kind: str)
tests/test_harbor_runtime_skill_isolation.py:4530
Functiontest_eval_program_held_in_a_variable_is_read_after_this_shell_expands_it
This shell expands eval's words before eval reads them, so ``eval "$code"`` runs what ``code`` holds: a name it reassigns, unsets or unexports is
tests/tier3/test_script_invocation_evidence.py:2134
Functiontest_eval_prompt_that_restates_the_description_is_flagged_as_an_eval_problem
(tmp_path: Path)
tests/test_tier3_case_summary.py:1027
Methodtest_evals_directory_not_flagged_as_unexpected
A skill that only ships an evals/ subdir for Tier 3 must not warn.
tests/validators/test_folder_validation.py:338
Functiontest_evaluate_autopilot_without_tier3_extra_names_it_before_generation
(monkeypatch, tmp_path, env_mode)
tests/test_cli.py:1048
Functiontest_evaluate_forwards_claude_alias_as_canonical_agent
(monkeypatch, tmp_path: Path)
tests/test_tier3_public_runtime.py:755
Functiontest_evaluate_forwards_max_cases
(monkeypatch: pytest.MonkeyPatch, command: list[str])
tests/test_tier3_max_cases.py:270
Functiontest_evaluate_forwards_native_environment_without_legacy_sandbox_configuration
(monkeypatch, tmp_path)
tests/test_tier3_public_runtime.py:710
Functiontest_evaluate_keeps_an_unrelated_import_error
(monkeypatch: pytest.MonkeyPatch)
tests/test_cli.py:1071
Functiontest_evaluate_preserves_an_installed_dependency_import_failure
(monkeypatch, error)
tests/test_cli.py:1031
Functiontest_evaluate_rejects_a_non_positive_max_cases
(monkeypatch: pytest.MonkeyPatch)
tests/test_tier3_max_cases.py:286
Functiontest_evaluate_rejects_repeated_model_override_before_engine
(monkeypatch, tmp_path: Path)
tests/test_tier3_public_runtime.py:795
Functiontest_evaluate_summary_renders_budget_warning
()
tests/tier3/test_run_budget.py:622
Functiontest_evaluate_terminal_shows_coverage_even_in_compact_mode
(verbose)
tests/reporting/test_declarative_coverage.py:193
Functiontest_evaluate_without_the_tier3_extra_names_the_missing_extra
(monkeypatch: pytest.MonkeyPatch, command)
tests/test_cli.py:1009
Functiontest_evaluated_counts_and_rate_equality
()
tests/tier3/test_grader_policy.py:150
Functiontest_evaluation_date_is_normalized_to_utc_before_publication
(tmp_path: Path)
tests/golden/test_benchmark_card.py:404
Functiontest_evaluation_failure_fields_are_reserved_metadata
Confirm evaluation failure fields are reserved metadata in custom grader runner.
tests/tier3/test_judge_failure_artifacts.py:1364
Methodtest_evaluation_options_forward_the_identity
(self)
tests/test_benchmark_source_provenance.py:1234
Functiontest_evaluation_options_leave_agent_selection_to_the_provider
()
tests/test_evaluation_service.py:119
Functiontest_evaluation_options_preserves_legacy_positional_skip_baseline
()
tests/test_evaluation_service.py:112
Functiontest_evaluation_options_reach_the_engine
()
tests/tier3/test_baseline_timeouts.py:1801
Functiontest_evaluation_summary_omits_unremarkable_fresh_baseline
()
tests/test_tier3_result_display.py:608
Functiontest_evaluation_summary_reports_reused_or_fallback_baseline
(baseline, expected: str)
tests/test_tier3_result_display.py:599
Functiontest_evaluator_card_names_the_cases_its_lift_left_out
(tmp_path: Path, all_mixed: bool)
tests/test_judge_lift_integrity.py:381
Functiontest_evaluator_edits_leave_runtime_identity_unchanged
(tmp_path: Path, filename: str)
tests/tier3/test_decoy_identity.py:25
Methodtest_evaluator_only_assets_cannot_be_declared_as_task_input
( self, tmp_path: Path, relative_path: str, is_directory: bool, )
tests/test_harbor_input_staging.py:422
Functiontest_evaluator_only_changes_preserve_frozen_selection
( selection: SimpleNamespace, tmp_path: Path, filename: str, )
tests/tier3/test_decoys.py:384
Functiontest_evaluator_python_path_uses_only_selected_runtime_prefix
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_local_mode.py:5257
Functiontest_evaluator_read_accepts_windows_crt_descriptor_identity
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_skill_isolation.py:4859
Methodtest_evaluator_version_is_not_the_source_revision
(self)
tests/test_benchmark_source_provenance.py:267
Functiontest_events_and_groups_past_the_limits_are_reported
(events: dict)
tests/validators/test_plugin_hook_risk_bounds.py:261
Functiontest_every_bad_rule_is_reported
()
tests/tier3/test_f90_mcp_arguments.py:116
Functiontest_every_canonical_token_shape_is_a_secret_and_is_redacted
(token: str)
tests/validators/test_url_policy.py:313
Functiontest_every_case_row_uses_a_dataset_id_at_most_once
()
tests/tier3/test_token_usage.py:678
Functiontest_every_ci_uv_install_uses_the_one_reviewed_version
GitHub jobs, GitLab jobs, the Tier 1 image and the publisher all run one uv. Covers ``setup-uv`` steps (including an older action pin with no ``v
tests/test_ci_workflows.py:542
Functiontest_every_denied_boundary_mutation_produces_a_path_rule_and_line
(rule: str, mutation: str)
tests/test_oss_boundary.py:74
Functiontest_every_directory_download_accepts_dotdot_paths_that_stay_inside_the_run_directory
A ".." that resolves inside the run directory is fine, and the filters still apply.
tests/test_harbor_local_environment.py:742
Functiontest_every_directory_download_refuses_dotdot_paths_that_leave_the_run_directory
download_dir and both tar/find downloads refuse the same sources, whichever root the path starts from.
tests/test_harbor_local_environment.py:691
Functiontest_every_dns_answer_uses_allowlist_and_metadata_never_does
(er)
tests/validators/test_private_endpoint_resolution.py:82
Functiontest_every_embedding_input_is_bounded
(extension)
tests/deduplication/utils/test_chunk_bounds.py:19
Methodtest_every_error_stays_when_asked
(self)
tests/models/test_result.py:240
Functiontest_every_git_root_lookup_names_a_non_ascii_root
Each repository-root lookup decodes git's path bytes as file names, not with the locale code page.
tests/utils/test_helpers.py:261
Functiontest_every_github_token_prefix_has_a_secret_shape
(prefix: str)
tests/validators/test_url_policy.py:328
Functiontest_every_heredoc_declared_on_a_line_takes_its_body_in_order
Every heredoc declared on one line, across commands joined on that line included, reads its body after the line ends, in the order declared, as
tests/tier3/test_script_invocation_evidence.py:1628
Functiontest_every_judge_graded_string_leaf_is_checked_after_loading
(tmp_path, fields)
tests/tier3/test_dataset_baseline_fairness.py:1681
Functiontest_every_judge_prompt_example_passes_its_own_validator
(judge_module, monkeypatch)
tests/tier3/test_judge_contracts.py:107
Functiontest_every_managed_install_pins_the_release_harbor_installs
(tmp_path)
tests/tier3/test_local_runtime_install.py:22
Methodtest_every_matched_term_is_listed_once_and_deducted_once
(self, tmp_path: Path)
tests/validators/test_quality_score.py:2007
Methodtest_every_missing_reference_needs_a_row_on_its_line
(self, tmp_path: Path)
tests/validators/test_security.py:10012
Functiontest_every_name_a_substitution_lists_is_a_reading
()
tests/tier3/test_network_client_mentions.py:686
Functiontest_every_output_publishes_one_overall_and_lift
( tmp_path: Path, scenario: str, baseline_not_applicable: tuple[str, ...], with_overall: float
tests/test_overall_cross_format_consistency.py:183
Functiontest_every_oversized_file_long_line_and_unlisted_folder_is_counted
(tmp_path: Path)
tests/harness/test_overlay.py:827
Functiontest_every_path_the_classifier_calls_documentation_selects_its_tests
A path added to the classifier is covered here without a second edit.
tests/test_select_docs_tests.py:55
Functiontest_every_profile_has_plugin_section
()
tests/harness/test_plugin_compat.py:46
Functiontest_every_rule_has_a_fix_and_placeholders_are_filled
(tmp_path: Path)
tests/harness/test_overlay.py:710
Functiontest_every_scope_nests_in_either_order_and_each_compound_has_its_own_pipe
Groups and compound pipeline stages are one stack of scopes, innermost last: a ``( ... )`` inside a compound stage copies the stage's bindings and
tests/tier3/test_script_invocation_evidence.py:1467
Methodtest_every_scored_reader_verb_credits_the_member
(self, verb: str)
tests/tier3/test_plugin_signals.py:396
Functiontest_every_short_string_renders_literally
(context: str)
tests/utils/test_rich_markup.py:80
Methodtest_every_spelling_and_namespace_credits_the_declared_component
(self)
tests/tier3/test_plugin_signals.py:1138
Methodtest_every_spelling_of_a_declared_server_read_credits_the_member
(self, fn: str)
tests/tier3/test_plugin_signals.py:242
Methodtest_every_spelling_of_an_mcp_edit_tool_writes_the_artifact
(self, fn: str)
tests/tier3/test_plugin_signals.py:261
Functiontest_every_status_is_known
(tmp_path: Path)
tests/harness/test_overlay.py:223
Functiontest_every_value_of_the_allowed_tools_flag_is_checked
(tmp_path: Path, agent: dict, flagged: bool)
tests/validators/test_plugin_static_risk_review_fixes.py:859
Functiontest_every_workflow_does_not_persist_checkout_credentials
()
tests/test_ci_workflows.py:566
Functiontest_every_workflow_pins_every_action_to_a_commit
()
tests/test_ci_workflows.py:495
Methodtest_evidence_and_limits_are_bounded_and_sanitized
(self)
tests/validators/test_skillspector_reference_diagnostics.py:289
Functiontest_evidence_artifact_keeps_provenance_attempt_and_size_checks
(change, message)
tests/ci/test_publish_reports.py:756
Functiontest_evidence_block_is_absent_from_the_tier3_scoped_report
()
tests/reporting/test_verdict_evidence.py:306
Functiontest_evidence_destination_is_outside_hidden_tier_panels
()
tests/reporting/test_report_navigation.py:87
Functiontest_evidence_failure_is_recorded_and_publication_still_runs
(tmp_path: Path, monkeypatch)
tests/tier3/test_plugin_execution.py:277
Functiontest_evidence_markup_is_escaped_exactly_once
()
tests/reporting/test_tier3_failure_reasons.py:359
Functiontest_evidence_reason_is_absent_when_refs_are_present_or_finding_is_ok
()
tests/tier3/test_plain_language_findings.py:176
Functiontest_evidence_redaction_never_publishes_a_credential_split_by_the_scan_budget
(security)
tests/tier3/test_private_plugin_security.py:464
Functiontest_evidence_snapshot_binds_complete_files_corpus_for_explicit_entries
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:4671
Functiontest_evidence_snapshot_rejects_unselected_unsafe_file_it_must_bind
(tmp_path: Path)
tests/test_harbor_runtime_skill_isolation.py:4710
Functiontest_evidence_unavailable_labels_are_limited_to_evidence_ref_metrics
()
tests/tier3/test_plain_language_findings.py:118
Functiontest_evidence_volume_case_fallbacks_and_unknown_case_counts
()
tests/reporting/test_evidence_volume_rendering.py:200
Functiontest_evidence_volume_counts_cases_once_and_separates_failures_by_arm
()
tests/reporting/test_evidence_volume_rendering.py:317
Functiontest_evidence_volume_formats_singular_and_complete_coverage
()
tests/reporting/test_evidence_volume_rendering.py:343
← previousnext →15,601–15,700 of 24,741, ranked by callers