MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Methodtest_tier2_analysis_diagnostics_are_exported_without_unrelated_metadata
(self)
tests/reporting/test_reporters.py:751
Functiontest_tier2_anthropic_chat_probe_is_bounded_by_the_timeout
(provider_api, monkeypatch)
tests/test_doctor.py:349
Functiontest_tier2_bedrock_chat_probe_passes_the_timeout_to_litellm
(provider_api, monkeypatch)
tests/test_doctor.py:375
Functiontest_tier2_cli_reports_preserve_analysis_errors_with_findings
Exercise collection, clustering, SDK requests and all reports over loopback HTTP.
tests/reporting/test_mixed_execution_errors.py:44
Functiontest_tier2_clis_scan_contained_agent_directory_alias_once
(tmp_path: Path, monkeypatch)
tests/test_cli.py:203
Functiontest_tier2_combines_intra_and_catalog_comparison_once
(skill, tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:265
Functiontest_tier2_commands_redact_absolute_target_paths_from_all_reports
( tmp_path: Path, monkeypatch, command: str, runner_path: str, basename: str, )
tests/test_cli.py:573
Functiontest_tier2_commands_reject_unresolved_root_symlinks
( tmp_path: Path, monkeypatch, command: str, runner_path: str, )
tests/test_cli.py:257
Functiontest_tier2_invalid_catalog_fails_before_model_work
(skill, tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:296
Functiontest_tier2_live_failures_are_reported_without_provider_bodies
(provider_api, capsys, status)
tests/test_doctor.py:396
Functiontest_tier2_live_probe_sends_one_bounded_request_per_model
(provider_api, timeout)
tests/test_doctor.py:337
Functiontest_tier2_missing_catalog_metadata_does_not_make_unavailable_tile_grid
()
tests/reporting/test_report_clarity.py:144
Functiontest_tier2_missing_provider_fails_without_network
(provider_api, monkeypatch, capsys)
tests/test_doctor.py:308
Functiontest_tier2_optional_policy_does_not_waive_missing_tier3
()
tests/golden/test_benchmark_card.py:519
Functiontest_tier2_rejects_catalog_report_collision_before_reading_or_overwriting
(skill, tmp_path)
tests/cli/test_tier_workflows.py:441
Functiontest_tier2_rejects_linked_target_before_provider_resolution
(skill, tmp_path)
tests/cli/test_tier_workflows.py:433
Functiontest_tier2_resolves_providers_offline_by_default
(provider_api)
tests/test_doctor.py:298
Functiontest_tier2_retains_llm_and_work_limit_evidence
()
tests/reporting/test_retained_tiers.py:48
Functiontest_tier2_runs_on_macos_after_merge_only_and_keeps_its_required_name
()
tests/test_ci_workflows.py:291
Functiontest_tier2_still_requires_a_valid_selected_embedding_route
()
tests/ci/test_model_config.py:207
Functiontest_tier2_summary_checks_are_trustworthy_execution_evidence
()
tests/golden/test_benchmark_card.py:906
Functiontest_tier2_without_catalog_does_not_manufacture_a_similarity_pass
(skill, tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:328
Functiontest_tier2_without_the_extra_reports_it_missing
(monkeypatch)
tests/test_doctor.py:321
Functiontest_tier3_attempt_coverage_is_bound_to_dataset_size
()
tests/reporting/test_advisory_skip_verdict.py:3105
Functiontest_tier3_benchmark_callout_includes_required_failure_reason
(monkeypatch)
tests/test_tier3_attempt_verdict.py:464
Functiontest_tier3_benchmark_gives_its_own_reason_for_not_assessed
()
tests/reporting/test_tier3_completed_agent_docs.py:382
Methodtest_tier3_blocking_policy_keeps_accurate_gate_counts
(self)
tests/reporting/test_reporters.py:2137
Functiontest_tier3_bounded_copy_stops_before_iterating_or_loading_wide_map_tail
( monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_advisory_skip_verdict.py:1679
Functiontest_tier3_charts_are_redrawn_with_print_colors
()
tests/reporting/test_report_redesign.py:383
Functiontest_tier3_command_demands_all_cases_both_arms_and_explicit_mode
(consumer, tmp_path)
tests/ci/test_run_tiers.py:95
Functiontest_tier3_compare_renders_agent_and_summary_values_literally
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/reporting/test_rich_markup_escaping.py:489
Functiontest_tier3_dedup_observation_cannot_supply_tier2_publication_evidence
()
tests/golden/test_benchmark_card.py:922
Functiontest_tier3_direct_dependencies_have_complete_license_notices
()
tests/test_oss_packaging.py:481
Functiontest_tier3_docs_describe_criterion_derived_accuracy
()
tests/test_accuracy_arithmetic.py:53
Functiontest_tier3_docs_describe_the_current_nvidia_build_docker_handoff
()
tests/test_oss_packaging.py:931
Functiontest_tier3_docs_explain_cost_controls_and_local_mode_tradeoffs
()
tests/test_oss_packaging.py:903
Functiontest_tier3_docs_name_the_supported_harbor_backend_version
()
tests/test_oss_packaging.py:918
Functiontest_tier3_doctor_accepts_backend_option
()
tests/test_nemo_gym_backend.py:1882
Functiontest_tier3_doctor_renders_provider_and_prerequisite_errors_literally
(monkeypatch: pytest.MonkeyPatch)
tests/reporting/test_rich_markup_escaping.py:525
Functiontest_tier3_doctor_stays_live_evaluation_only
(doctor_calls)
tests/test_cli.py:846
Functiontest_tier3_doctor_strips_terminal_control_sequences
(monkeypatch: pytest.MonkeyPatch)
tests/reporting/test_rich_markup_escaping.py:545
Functiontest_tier3_embeds_verified_chart_runtime_without_network_or_source_map
()
tests/reporting/test_chart_dependency.py:81
Functiontest_tier3_evaluate_final_summary_renders_dataset_size
(sufficiency)
tests/reporting/test_dataset_sufficiency_rendering.py:47
Functiontest_tier3_evaluate_help_lists_the_provenance_options
()
tests/test_cli.py:1111
Functiontest_tier3_evaluate_prints_one_recovery_line_per_agent_with_recovered_timeouts
()
tests/test_harbor_timeout_recovery.py:559
Functiontest_tier3_evaluate_prints_the_collected_recovery_line
(tmp_path)
tests/test_harbor_timeout_recovery.py:543
Functiontest_tier3_evaluate_terminal_names_only_the_reports_the_run_wrote
(tmp_path: Path, written: str)
tests/reporting/test_tier3_token_usage_reports.py:903
Functiontest_tier3_evaluate_terminal_prints_the_agent_sums_the_report_step_recorded
(tmp_path: Path)
tests/reporting/test_tier3_token_usage_reports.py:839
Functiontest_tier3_evaluate_terminal_prints_the_token_usage_of_a_run_whose_arms_all_failed
(tmp_path: Path)
tests/reporting/test_tier3_token_usage_reports.py:865
Functiontest_tier3_evaluate_visible_copy_owns_its_params
()
tests/test_commands.py:3663
Methodtest_tier3_failure_remains_advisory
(self)
tests/reporting/test_reporters.py:2216
Functiontest_tier3_failure_tile_and_hero_keep_command_outcome_separate_from_quality
()
tests/reporting/test_final_render_context.py:325
Functiontest_tier3_grader_starter_renders_created_path_literally
( payload: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/reporting/test_rich_markup_escaping.py:460
Functiontest_tier3_help_keeps_option_descriptions_visible
(width)
tests/cli/test_tier_workflows.py:421
Functiontest_tier3_help_lists_feedback
()
tests/evaluation/test_tier3_feedback_chat.py:1401
Functiontest_tier3_html_renders_plain_accuracy_reason_and_unavailable_evidence
()
tests/reporting/test_unified_tier3_report.py:1736
Functiontest_tier3_judge_and_inference_client_share_the_helpers
()
tests/inference/test_json_span.py:14
Functiontest_tier3_missing_execution_does_not_invent_scores_or_zero_coverage
()
tests/ci/test_summary_metrics.py:206
Functiontest_tier3_neutral_evidence_accepts_recorded_neutral_or_producer_incomplete_downgrade
(blocking, complete, status)
tests/ci/test_feedback.py:1141
Functiontest_tier3_neutral_evidence_is_never_promoted
(complete, status)
tests/ci/test_feedback.py:1148
Functiontest_tier3_optional_policy_does_not_waive_missing_tier2
()
tests/golden/test_benchmark_card.py:532
Methodtest_tier3_output_in_the_skill_counts_as_not_inspected
A Tier 3 run in the default ``<skill>/evals/results`` turns a passing line into ``INCOMPLETE``. The run's names (``results``, ``report.html``
tests/validators/test_security.py:10467
Functiontest_tier3_partial_execution_retains_coverage_without_claiming_clean_counts_or_scores
()
tests/ci/test_summary_metrics.py:217
Functiontest_tier3_payload_name_must_exactly_match_its_publication_target
()
tests/reporting/test_advisory_skip_verdict.py:2240
Functiontest_tier3_platform_jobs_are_skipped_for_unrelated_source_and_tests
(paths: list[bytes])
tests/test_ci_change_classifier.py:142
Functiontest_tier3_platform_jobs_run_for_related_and_unknown_paths
(paths: list[bytes])
tests/test_ci_change_classifier.py:118
Functiontest_tier3_platform_jobs_run_only_for_changes_that_can_affect_them
()
tests/test_ci_workflows.py:262
Functiontest_tier3_reference_uses_canonical_dimension_mapping
()
tests/test_benchmark_rollout_docs.py:63
Functiontest_tier3_rejects_malformed_native_containers
(key, value)
tests/reporting/test_retained_tiers.py:134
Functiontest_tier3_result_cannot_double_as_required_tier1_or_tier2_evidence
( tmp_path: Path, dual_tier: int, )
tests/reporting/test_advisory_skip_verdict.py:2260
Functiontest_tier3_result_cannot_reuse_a_tier1_or_tier2_producer_marker
(tier: int, check_id: str)
tests/test_publication_evidence.py:292
Functiontest_tier3_roundtrip_keeps_canonical_agent_payload
()
tests/reporting/test_retained_tiers.py:72
Functiontest_tier3_run_report_expects_recorded_publication_name
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_commands.py:836
Functiontest_tier3_serialization_budget_fails_publication_closed
( monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_advisory_skip_verdict.py:1608
Functiontest_tier3_serialization_cannot_omit_proof_while_reporting_publication_pass
( monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_advisory_skip_verdict.py:1817
Functiontest_tier3_serialization_rejects_huge_text_before_normalizing_it
( monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_advisory_skip_verdict.py:1643
Functiontest_tier3_text_check_still_rejects_controls_and_identity_line_breaks
(mutation: str)
tests/reporting/test_advisory_skip_verdict.py:2433
Functiontest_tier3_uses_validated_result_metadata_for_scores_and_coverage
()
tests/ci/test_summary_metrics.py:181
Functiontest_tier3_validate_cli_fails_on_fairness_warning_only_with_strict
(tmp_path: Path, strict: bool, exit_code: int)
tests/tier3/test_dataset_baseline_fairness.py:315
Functiontest_tier3_validate_cli_rejects_unsafe_case_id
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:162
Functiontest_tier3_validate_help
()
tests/test_cli.py:602
Functiontest_tier3_validate_panel_renders_skill_name_literally
( name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/reporting/test_rich_markup_escaping.py:426
Functiontest_tier3_validate_panel_strips_terminal_control_sequences
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/reporting/test_rich_markup_escaping.py:441
Functiontest_tier3_validate_reports_broken_config_symlink
(tmp_path)
tests/tier3/test_evals_config.py:382
Functiontest_tier3_validate_strict_fails_on_a_skill_script_assertion
(tmp_path: Path, assertions)
tests/tier3/test_dataset_baseline_fairness.py:670
Functiontest_tier3_validate_strict_passes_an_outcome_dataset
(tmp_path: Path)
tests/tier3/test_dataset_baseline_fairness.py:329
Functiontest_tier_alias_help
()
tests/test_cli.py:115
Functiontest_tier_budget_is_shared_across_skills_and_keeps_command_logs_first
(tmp_path, monkeypatch)
tests/ci/test_ci_worker_diagnostics.py:272
Functiontest_tier_caps_stay_within_what_the_feedback_reader_accepts
()
tests/ci/test_ci_worker_diagnostics.py:304
Functiontest_tier_counts_exclude_na_and_do_not_leak_other_tiers
()
tests/reporting/test_final_render_context.py:35
Functiontest_tier_diagnostics_budget_is_shared_by_every_skill
(consumer, tmp_path, monkeypatch)
tests/ci/test_run_tiers.py:956
Functiontest_tier_help_shows_default_workflow_options
(tier, flag)
tests/cli/test_tier_workflows.py:414
Functiontest_tier_name_collision_requires_explicit_path
(tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:389
Functiontest_tier_navigation_has_one_headline_and_tier1_owns_severity_detail
()
tests/reporting/test_report_clarity.py:152
Functiontest_tier_source_attempt_is_an_earlier_or_current_attempt_of_the_run
()
tests/ci/test_tier_manifest.py:245
Functiontest_ties_and_reordered_catalog_are_repeatable
(selection: SimpleNamespace)
tests/tier3/test_decoys.py:103
Functiontest_timeout_diagnostic_cannot_synthesize_sensitive_value
( tmp_path: Path, with_callback: bool, )
tests/test_harbor_local_mode.py:2929
Functiontest_timeout_diagnostic_uses_the_live_stderr_redactor_across_its_boundary
( tmp_path: Path, with_callback: bool, secret: str, )
tests/test_harbor_local_mode.py:2963
Functiontest_timeout_is_unscored
(monkeypatch, tmp_path)
tests/tier3/test_grader_validation.py:834
Functiontest_timeout_keeps_the_time_budget_diagnostic_for_every_skill
(consumer, tmp_path, monkeypatch)
tests/ci/test_run_tiers.py:506
← previousnext →23,301–23,400 of 24,741, ranked by callers