MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_live_case_marks_reads_harbor_jobs
(tmp_path)
tests/reporting/test_tier3_validate_cases.py:201
Functiontest_live_case_marks_reads_nemo_gym_rollouts
(tmp_path)
tests/reporting/test_tier3_validate_cases.py:276
Functiontest_live_correct_unavailable_case_meets_observed_category_only
(judge_server, qualification_policy, tmp_path)
tests/tier3/test_grader_validation.py:1161
Functiontest_live_custom_catalog_401_is_inconclusive_without_exposing_secret
(provider_name: str)
tests/test_harbor_runtime_preflight.py:4038
Functiontest_live_engine_links_win_per_job_and_keep_distinct_recorded_jobs
(tmp_path, status)
tests/reporting/test_recorded_harbor_uploads.py:179
Functiontest_live_engine_result_and_run_config_name_the_backend
(monkeypatch, tmp_path)
tests/reporting/test_execution_backend_reporting.py:65
Functiontest_live_eval_exposes_only_harbor_native_environments
()
tests/test_tier3_public_runtime.py:59
Functiontest_live_eval_help_uses_skill_evaluator_runtime_and_grading_names
()
tests/test_cli.py:863
Functiontest_live_eval_progress_interval_configures_reporter
( monkeypatch: pytest.MonkeyPatch, command: tuple[str, ...], interval_args: tuple[str, ...], e
tests/test_cli.py:906
Functiontest_live_eval_progress_is_presentation_only
(monkeypatch: pytest.MonkeyPatch)
tests/test_cli.py:882
Functiontest_live_eval_rejects_invalid_progress_interval
(monkeypatch: pytest.MonkeyPatch, value: str)
tests/test_cli.py:941
Functiontest_live_evaluation_docs_say_a_stream_can_still_send_refused_output_later
()
tests/test_tier3_progress.py:2379
Methodtest_live_issue_is_kept
(self)
tests/validators/test_artifact_dir_exclusion.py:526
Functiontest_live_mark_merges_verifier_security_sidecar
(tmp_path: Path)
tests/test_tier3_case_summary.py:785
Functiontest_live_marks_key_a_trial_by_its_whole_task_directory
(tmp_path: Path)
tests/test_tier3_case_summary.py:1262
Functiontest_live_marks_score_reward_logs_and_skip_normalized_gym_jobs
(tmp_path: Path)
tests/test_nemo_gym_backend.py:2243
Functiontest_live_rows_mark_running_and_waiting_cases
()
tests/reporting/test_tier3_validate_cases.py:60
Functiontest_live_view_draws_the_header_and_finished_tiers_once
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:603
Functiontest_live_view_prints_log_records_through_its_own_console
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:350
Functiontest_llm_and_tool_seconds_flow_from_trajectories_to_json_and_html
(tmp_path)
tests/reporting/test_resource_comparison.py:164
Methodtest_llm_auth_failure_hidden_behind_clean_report_is_incomplete
SkillSpector 2.3.7 can exit 0 after logging a rejected-key failure.
tests/validators/test_security.py:1629
Functiontest_llm_client_exhausts_retries_on_rate_limit
Verify LLMClient.completions exhausts retries and raises when 429 persists.
tests/test_llm_client_retry.py:99
Functiontest_llm_client_fails_fast_on_auth_error
Verify LLMClient.completions raises immediately without retry on AuthenticationError.
tests/test_llm_client_retry.py:68
Functiontest_llm_client_respects_custom_retry_params
Verify LLMClient stores and exposes custom retry configuration parameters.
tests/test_llm_client_retry.py:130
Functiontest_llm_client_retries_on_rate_limit_and_succeeds
Verify LLMClient.completions retries on RateLimitError and succeeds on subsequent attempt.
tests/test_llm_client_retry.py:34
Methodtest_llm_cluster_limit_is_incomplete_before_llm_analysis
( self, mock_embed, mock_build_clusters, mock_llm, tmp_path: Path, monkeypatch )
tests/deduplication/intra_skill/test_intra_skill_validator.py:489
Methodtest_llm_enrichment_requires_explicit_positive_execution_metadata
( self, mock_tools, llm_metadata: dict, sample_skill_dir: Path, )
tests/validators/test_security.py:1752
Methodtest_llm_enrichment_requires_semantic_analyzer_evidence
( self, mock_tools, sample_skill_dir: Path, missing_analyzer: str, )
tests/validators/test_security.py:1792
Methodtest_llm_error_is_incomplete_without_skill_finding
( self, mock_embed, _mock_llm, mock_analyze, tmp_path: Path, caplog )
tests/deduplication/intra_skill/test_intra_skill_validator.py:1097
Methodtest_llm_fallback_when_no_key
(self, tmp_path)
tests/validators/test_rubric_eval.py:750
Functiontest_llm_fix_asks_for_the_key_not_a_scanner_install
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:314
Functiontest_llm_fix_follows_a_skillspector_adapter_chosen_directly
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:455
Functiontest_llm_fix_for_bedrock_set_through_skillspector_gives_no_public_provider_advice
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:484
Functiontest_llm_fix_points_at_doctor_when_the_provider_call_fails
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:365
Functiontest_llm_fix_still_covers_the_public_provider_for_llm_verify_with_a_direct_adapter
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:511
Functiontest_llm_fix_with_openai_and_no_key_names_both_accepted_variables
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:441
Functiontest_llm_fix_without_a_provider_gives_every_export
(monkeypatch)
tests/reporting/test_tier3_validate_cases.py:335
Functiontest_llm_generation_falls_back_for_non_string_expected_script
(tmp_path, monkeypatch, capsys)
tests/tier3/test_generate_dataset_results.py:767
Functiontest_llm_generation_prompt_examples_are_fair_to_the_baseline
(llm_prompts, full)
tests/tier3/test_dataset_baseline_fairness.py:526
Functiontest_llm_generation_records_mode_and_trace_refinement_flag
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:594
Functiontest_llm_generation_writes_null_for_placeholder_expected_script
(tmp_path, monkeypatch)
tests/tier3/test_generate_dataset_results.py:729
Functiontest_llm_generator_rejects_malformed_case_and_uses_safe_one_case_template
( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], )
tests/test_evaluation_service.py:728
Methodtest_llm_initialization_error_preserves_collection_evidence
(self, mock_embed, mock_llm, tmp_path: Path)
tests/deduplication/intra_skill/test_intra_skill_validator.py:1353
Methodtest_llm_insights_name_the_model_that_wrote_them
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:1651
Functiontest_llm_insights_use_eligible_trials_without_hiding_diagnostics
(monkeypatch, activated: bool)
tests/reporting/test_skill_activation_reporting.py:162
Functiontest_llm_judge_prompts_are_product_neutral
()
tests/test_oss_source_hygiene.py:24
Functiontest_llm_maps_only_explicit_provider_and_credentials
(tmp_path)
tests/ci/test_run_tier1.py:474
Functiontest_llm_provider_error_lists_supported_choices
()
tests/test_provider_config.py:459
Functiontest_llm_refine_prompt_does_not_ask_for_skill_use_behaviors
(llm_prompts)
tests/tier3/test_dataset_baseline_fairness.py:540
Functiontest_llm_refinement_falls_back_and_flags_missing_case_ids
(monkeypatch)
tests/tier3/test_dataset_grounding.py:453
Functiontest_llm_refinement_restores_and_flags_dropped_authored_behavior
(monkeypatch)
tests/tier3/test_dataset_grounding.py:337
Methodtest_llm_refusal_beside_a_duplicate_keeps_the_duplicate_blocking
( self, mock_embed, _mock_llm, mock_analyze, tmp_path: Path )
tests/deduplication/intra_skill/test_intra_skill_validator.py:1304
Methodtest_llm_refusal_beside_a_failed_cluster_keeps_the_scan_incomplete
( self, mock_embed, _mock_llm, mock_analyze, tmp_path: Path )
tests/deduplication/intra_skill/test_intra_skill_validator.py:1276
Methodtest_llm_refusal_is_an_advisory_finding
A safety refusal says nothing about duplication, so it must not block the skill.
tests/deduplication/intra_skill/test_intra_skill_validator.py:1239
Methodtest_llm_review_limits_are_incomplete_before_llm_analysis
( self, mock_embed, mock_build_clusters, mock_llm, limit_name: str,
tests/deduplication/intra_skill/test_intra_skill_validator.py:536
Functiontest_llm_scan_that_never_started_gets_the_scanner_fix_not_a_provider_fix
``--llm`` records skillspector-llm before any scan runs; the real validator paths must not read as LLM-only.
tests/reporting/test_tier3_validate_cases.py:400
Methodtest_llm_stage_counts_each_reference_once
Both stages report the AE1 findings of an accepted image gap; each reference line counts once.
tests/validators/test_skillspector_reference_diagnostics.py:223
Methodtest_llm_stage_keeps_the_static_applicability_block
(self, mock_tools, sample_skill_dir: Path)
tests/validators/test_security.py:1438
Methodtest_llm_stage_records_requested_skillspector_model_and_effort
( self, monkeypatch, sample_skill_dir: Path, )
tests/validators/test_security.py:5697
Methodtest_llm_stage_records_skillspector_provenance_and_bridged_provider
( self, monkeypatch, sample_skill_dir: Path, )
tests/validators/test_security.py:5829
Functiontest_llm_stage_without_executed_analyzers_is_not_applicable
()
tests/test_skillspector_provenance.py:87
Methodtest_llm_verdict_explanation_reads_rationale_or_legacy_reasoning
The prompt asks for "rationale"; a reply using the older "reasoning" key still counts.
tests/validators/test_security.py:6040
Methodtest_llm_verdict_labels_are_sanitized_or_left_out
( self, tmp_path: Path, provider: str, model_used: str | None, expected: dict[str, str] )
tests/validators/test_security.py:6128
Methodtest_llm_verdicts_name_the_model_that_answered
A verdict from a server-side fallback model is labeled with that model and its provider.
tests/validators/test_security.py:6076
Methodtest_llm_verification_cannot_downgrade_the_hook_critical
The verifier never sees the Critical, so it gets no verdict and needs none.
tests/validators/test_skillspector_hook_surface.py:543
Methodtest_llm_verification_skips_a_scan_whose_only_finding_is_the_critical
(self, tmp_path: Path)
tests/validators/test_skillspector_hook_surface.py:567
Methodtest_llm_verification_without_verdicts_reports_the_cause
(self, tmp_path: Path)
tests/validators/test_security.py:6060
Methodtest_llm_verify_docs_state_the_answering_model_rule_of_rubric_evaluation
Both labels come from the same client, so the two docs paragraphs state one rule.
tests/validators/test_security.py:6239
Methodtest_llm_verify_downgrade_drops_the_stale_baseline_blocker_error
The stale-fingerprint error explains a remaining blocker, so a downgrade of the last one passes the scan.
tests/validators/test_security.py:2102
Methodtest_llm_verify_downgrade_keeps_independent_errors_only_for_an_incomplete_scan
A downgrade rebuilds the result: a gate failure stays, other errors only when incomplete, a warning once.
tests/validators/test_security.py:2141
Methodtest_llm_verify_downgrade_keeps_the_messages_no_finding_repeats
An ``--llm-verify`` downgrade rebuilds the result from its findings, which back no reference message. The accepted missing-reference warning
tests/validators/test_security.py:11758
Methodtest_llm_verify_names_the_fallback_model_for_verdicts_kept_from_a_truncated_reply
A fallback model's reply cut off at the token limit still labels the verdicts kept from it.
tests/validators/test_security.py:6251
Methodtest_llm_verify_through_the_client_names_the_answering_model
The real verifier reports the model a Claude 5.5 reply names, and the configured model on OpenAI.
tests/validators/test_security.py:6164
Methodtest_load_accepts_bracketed_catalog_text
(self, tmp_path: Path)
tests/embedding/test_catalog.py:303
Methodtest_load_enforces_entry_and_file_size_limits
(self, tmp_path: Path, monkeypatch)
tests/embedding/test_catalog.py:314
Functiontest_load_evals_config_accepts_a_list_at_the_entry_limit
(tmp_path)
tests/tier3/test_evals_config.py:308
Functiontest_load_evals_config_keeps_yaml_merge_overrides
(tmp_path)
tests/tier3/test_evals_config.py:205
Functiontest_load_evals_config_missing_is_empty
(tmp_path)
tests/tier3/test_evals_config.py:144
Functiontest_load_evals_config_names_the_position_of_a_duplicate_key
(tmp_path)
tests/tier3/test_evals_config.py:193
Functiontest_load_evals_config_reads_config_through_links_in_the_skill_path
(tmp_path, link_target, skill_path)
tests/tier3/test_evals_config.py:421
Functiontest_load_evals_config_rejects_broken_config_symlink
(tmp_path)
tests/tier3/test_evals_config.py:373
Functiontest_load_evals_config_rejects_config_in_symlinked_evals_directory
(tmp_path)
tests/tier3/test_evals_config.py:397
Functiontest_load_evals_config_rejects_config_over_the_size_limit
(tmp_path)
tests/tier3/test_evals_config.py:455
Functiontest_load_evals_config_rejects_cyclic_yaml_merge
(tmp_path)
tests/tier3/test_evals_config.py:217
Functiontest_load_evals_config_rejects_documents_over_the_yaml_complexity_limits
(tmp_path, document, limit)
tests/tier3/test_evals_config.py:299
Functiontest_load_evals_config_rejects_duplicate_keys
(tmp_path, document)
tests/tier3/test_evals_config.py:182
Functiontest_load_evals_config_rejects_environment_kwargs_because_they_are_operator_only
(tmp_path)
tests/tier3/test_evals_config.py:463
Functiontest_load_evals_config_rejects_fifo_without_blocking
(tmp_path)
tests/tier3/test_evals_config.py:446
Functiontest_load_evals_config_rejects_finite_multiplier_that_overflows_default_timeouts
(tmp_path)
tests/tier3/test_evals_config.py:120
Functiontest_load_evals_config_rejects_hard_linked_config
(tmp_path)
tests/tier3/test_evals_config.py:434
Functiontest_load_evals_config_rejects_non_string_keys
(tmp_path, document, field)
tests/tier3/test_evals_config.py:335
Functiontest_load_evals_config_rejects_nonfinite_timeout_multiplier
(tmp_path, value)
tests/tier3/test_evals_config.py:96
Functiontest_load_evals_config_rejects_overflowing_pass_threshold
(tmp_path)
tests/tier3/test_evals_config.py:132
Functiontest_load_evals_config_rejects_overflowing_timeout_multiplier
(tmp_path)
tests/tier3/test_evals_config.py:108
Functiontest_load_evals_config_rejects_symlinked_config
(tmp_path)
tests/tier3/test_evals_config.py:362
Functiontest_load_evals_config_rejects_unknown_keys
(tmp_path)
tests/tier3/test_evals_config.py:154
Functiontest_load_evals_config_reports_unconstructible_value_without_echoing_it
(tmp_path, value)
tests/tier3/test_evals_config.py:270
Functiontest_load_evals_config_valid_harbor_policy
(tmp_path)
tests/tier3/test_evals_config.py:41
Functiontest_load_evals_config_validates_ranges
(tmp_path)
tests/tier3/test_evals_config.py:623
Functiontest_load_evals_config_validates_resource_shapes
(tmp_path)
tests/tier3/test_evals_config.py:606
← previousnext →18,001–18,100 of 24,741, ranked by callers