MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_required_check_failure_reason_pluralizes_and_omits_zero_counts
()
tests/test_tier3_attempt_verdict.py:592
Functiontest_required_check_verdict_precedes_counts_and_positive_lift_is_reference
()
tests/test_tier3_result_display.py:111
Functiontest_required_choice_with_only_server_web_search_is_a_structured_400
( bridge_services: _BridgeServices, )
tests/test_nvidia_build_bridge.py:1473
Functiontest_required_failure_cannot_be_replaced_by_completed_quality_verdict
()
tests/reporting/test_final_render_context.py:211
Functiontest_required_failure_with_unscored_overall_is_a_failing_trial
()
tests/test_tier3_attempt_verdict.py:283
Methodtest_required_guidance_fits_when_the_budget_covers_every_document
(self, tmp_path)
tests/validators/test_rubric_eval.py:376
Methodtest_required_guidance_gets_budget_before_optional_previews
(self, tmp_path)
tests/validators/test_rubric_eval.py:321
Methodtest_required_guidance_is_not_bounded_by_the_preview_size
(self, tmp_path)
tests/validators/test_rubric_eval.py:388
Functiontest_required_judge_accepts_the_deprecated_accuracy_keyword
(monkeypatch)
tests/tier3/test_judge_contracts.py:690
Functiontest_required_judge_deadline_interrupts_a_stalled_response_body
Interrupt stalled provider response body when required judge deadline expires.
tests/tier3/test_judge_failure_artifacts.py:350
Functiontest_required_judge_rechecks_its_deadline_after_a_swallowed_timeout
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, elapsed: float, not_applicable: bool )
tests/tier3/test_judge_failure_artifacts.py:920
Functiontest_required_judge_rechecks_the_inherited_deadline
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/tier3/test_judge_failure_artifacts.py:956
Functiontest_required_judge_restores_alarm_handler_after_teardown_interrupt
Restore previous SIGALRM handler when judge deadline teardown is interrupted.
tests/tier3/test_judge_failure_artifacts.py:384
Functiontest_required_provider_failure_is_a_configuration_error
(tier, options, skill, tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:160
Methodtest_required_schema_and_semantic_constraints
(self)
tests/tier3/test_plugin_signals.py:674
Functiontest_required_tier1_reuses_hardened_workflow_and_parallel_paid_tiers
()
tests/ci/test_all_tier_workflows.py:24
Functiontest_required_tier_nonpass_blocks_overall
(tier, status)
tests/ci/test_tier_manifest.py:62
Functiontest_required_universal_analyzer_must_inspect_every_component
(mutation)
tests/validators/test_security_reference_caveats.py:132
Methodtest_required_utf8_content_is_never_truncated
(self, tmp_path, earlier_guides, total, remaining)
tests/validators/test_rubric_eval.py:362
Functiontest_requirement_filename_sets_dependency_role
(tmp_path, pip_audit, filename, role)
tests/validators/test_dependency_advisory_findings.py:165
Functiontest_requirements_advisory_is_a_structured_finding_at_its_line
(tmp_path, pip_audit)
tests/validators/test_dependency_advisory_findings.py:73
Functiontest_requirements_files_are_audited_in_sorted_order
(tmp_path, pip_audit, monkeypatch)
tests/validators/test_dependency_advisory_findings.py:195
Methodtest_requirements_txt_comment_is_not_flagged
pip requirements.txt comments stay skipped.
tests/validators/test_security.py:970
Methodtest_requires_at_least_one_dependency
(self)
tests/models/test_plugin.py:74
Functiontest_requires_complete_evaluation_identity
(payload, field)
tests/tier3/test_external_metrics.py:128
Functiontest_requires_identical_case_coverage_not_equal_total_counts
(tmp_path)
tests/tier3/test_interactions.py:261
Methodtest_requires_openai_provider_intent
(self, provider: str)
tests/inference/test_llm_client.py:280
Functiontest_requires_summary_for_every_configured_agent
(tmp_path: Path)
tests/test_results_location_legacy_compatibility.py:399
Functiontest_requires_timezone_provenance
(payload, timestamp)
tests/tier3/test_external_metrics.py:135
Functiontest_requires_two_individuals
()
tests/tier3/test_interactions.py:145
Functiontest_requires_valid_observation_identity
(payload, field, value)
tests/tier3/test_external_metrics.py:145
Functiontest_rerender_does_not_replace_malformed_persisted_identity_with_engine_claim
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:3084
Functiontest_rerender_falls_back_from_invalid_snapshot_schema
(tmp_path: Path, schema_version: object)
tests/reporting/test_unified_tier3_report.py:2899
Functiontest_rerender_preserves_legacy_snapshot_dataset_with_corrected_summary
( tmp_path: Path, manifest_source: str )
tests/reporting/test_unified_tier3_report.py:2857
Functiontest_rerender_preserves_persisted_skill_name_when_live_path_is_renamed
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:2964
Functiontest_rerender_rejects_canonical_target_when_run_marks_identity_conflict
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:2994
Functiontest_rerender_reports_runtime_context_conflict_without_claiming_source_mutation
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:3039
Functiontest_rerender_uses_persisted_publication_identity_after_live_source_changes
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:2929
Functiontest_rerender_uses_run_owned_dataset_and_evaluator_version
(tmp_path: Path)
tests/reporting/test_unified_tier3_report.py:2827
Functiontest_rerendered_report_keeps_a_failed_gym_launch
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/test_nemo_gym_backend.py:1191
Functiontest_rerun_failed_jobs_reuses_earlier_attempt_tier_evidence
(consumer, tmp_path)
tests/ci/test_run_tiers.py:611
Functiontest_rerun_hint_preserves_run_shaping_flags
(monkeypatch)
tests/test_commands.py:3485
Functiontest_rerun_reuses_owned_branch_and_pr
(trusted, monkeypatch)
tests/ci/test_benchmark_publish.py:289
Functiontest_rerun_started_during_rendering_suppresses_comment
(tmp_path, monkeypatch)
tests/ci/test_prepare_comments.py:180
Functiontest_resample_means_draw_the_same_indices_whatever_the_values
The seam the additivity rests on: ``rng.choices`` picks indices from the seed and n alone.
tests/tier3/test_private_ladder_stats.py:490
Methodtest_rescreening_a_cataloged_skill_skips_only_its_own_entry
(self, _mock_client_cls, tmp_path: Path)
tests/validators/test_similarity.py:586
Functiontest_reservation_projection_includes_in_flight_and_whole_unit
(monkeypatch, dimension)
tests/tier3/test_budget_reservation.py:37
Methodtest_reserved_domain_emails_are_not_pii
(self, tmp_path: Path, domain: str, domain_format: str)
tests/validators/test_security.py:1323
Methodtest_reserved_name_segment_is_still_reported
(self, tmp_path: Path)
tests/validators/test_quality_score.py:962
Methodtest_reserved_names_match_complete_name_segments
(self, tmp_path: Path)
tests/validators/test_quality_score.py:957
Functiontest_reserved_native_agent_labels_stay_sanitized_bounded_and_distinct
()
tests/reporting/test_native_evidence.py:385
Functiontest_reserved_or_duplicate_private_variants_are_refused
(tmp_path)
tests/tier3/test_private_comparison_engine.py:89
Functiontest_reserved_terms_are_stored_without_surrounding_whitespace
(tmp_path: Path)
tests/test_quality_config.py:733
Methodtest_reserved_word_in_name_rejected
Test validation fails when name contains reserved words ('anthropic', 'claude').
tests/validators/test_schema.py:437
Methodtest_reserved_word_substring_in_name_is_allowed
Reserved providers only match complete hyphen-delimited name segments.
tests/validators/test_schema.py:466
Methodtest_reset_finding_counts_zeros_tracked_fields
(self)
tests/models/test_result.py:84
Functiontest_resolve_acceptable_skills_supports_legacy_alias_and_dedupes
()
tests/tier3/test_checks_path_extraction.py:148
Functiontest_resolve_agent_runtime_plan_preserves_refreshed_adc_token
Verify _resolve_agent_runtime_plan preserves refreshed ADC token and does not overwrite with expired token.
tests/test_harbor_runner_environment.py:2355
Functiontest_resolve_agent_runtime_plan_retains_retry_settings_in_subprocess_env
Verify _resolve_agent_runtime_plan forwards host retry settings to Harbor subprocess_env.
tests/test_harbor_runner_environment.py:352
Functiontest_resolve_environment_kwargs_precedence
Verify CLI overrides environment variables, which override config.yml (infrastructure kwargs ignored from config).
tests/test_harbor_gke_mode.py:67
Functiontest_resolve_environment_kwargs_strict_missing
Strict explicit configuration: missing kwargs are flagged.
tests/test_harbor_gke_mode.py:100
Functiontest_resolve_judge_wall_time_budget_from_env
Verify _resolve_judge_wall_time_budget parses positive finite seconds and falls back to 180.0 on invalid inputs.
tests/test_retry.py:689
Functiontest_resolve_llm_provider_vertex_openapi_adc
Resolve Vertex AI OpenAPI endpoint using Google Application Default Credentials without API key.
tests/test_provider_config.py:590
Functiontest_resolve_llm_provider_vertex_openapi_adc_missing_token
Raise ProviderConfigurationError when ADC token acquisition returns None.
tests/test_provider_config.py:637
Functiontest_resolve_llm_provider_vertex_openapi_adc_openai_alias
Resolve openai provider pointing to Vertex OpenAPI endpoint using ADC.
tests/test_provider_config.py:617
Methodtest_resolve_plugin_path_from_directory
Test resolve_plugin_path returns a directory unchanged.
tests/cli/test_content_type_detection.py:310
Methodtest_resolve_plugin_path_from_file
Test resolve_plugin_path collapses an agent_plugin.yaml file to its dir.
tests/cli/test_content_type_detection.py:304
Functiontest_resolve_retry_config_custom_env
Verify resolve_retry_config accepts standard SKILL_EVAL_LLM_* environment variables.
tests/test_retry.py:125
Functiontest_resolve_retry_config_defaults
Verify resolve_retry_config returns sensible defaults when environment is empty.
tests/test_retry.py:117
Functiontest_resolve_retry_config_defensive_fallbacks
Verify resolve_retry_config safely falls back to defaults when values are malformed.
tests/test_retry.py:138
Functiontest_resolve_retry_config_negative_values_fallback
Verify resolve_retry_config falls back to defaults when values are negative.
tests/test_retry.py:151
Methodtest_resolve_rules_path
Test resolve_rules_path returns path as-is.
tests/cli/test_content_type_detection.py:285
Functiontest_resolve_runtime_env_allows_approved_mcp_secret_and_blocks_operator_or_unapproved
Allow SKILLEVALUATOR_ALLOWED_MCP_SECRETS in harbor.runtime_env while blocking unapproved or operator secrets.
tests/test_harbor_gke_mode.py:1649
Functiontest_resolve_single_kubeconfig_absolutizes_relative_cert_and_token_paths
Resolve relative certificate, key, tokenFile, and exec command paths against originating kubeconfig dirs.
tests/test_harbor_gke_mode.py:935
Functiontest_resolve_single_kubeconfig_all_missing
Return None when all paths in KUBECONFIG are non-existent.
tests/test_harbor_gke_mode.py:877
Functiontest_resolve_single_kubeconfig_merge_failure_returns_none
Verify merge failures return None rather than falling back to a partial config.
tests/test_harbor_gke_mode.py:241
Functiontest_resolve_single_kubeconfig_merges_split_kubeconfig
Merge multiple KUBECONFIG files preserving clusters, users, and contexts.
tests/test_harbor_gke_mode.py:192
Functiontest_resolve_single_kubeconfig_multi_path
Resolve and merge valid file paths from a multi-path KUBECONFIG string.
tests/test_harbor_gke_mode.py:851
Functiontest_resolve_skill_path_accepts_a_decomposed_spelling_of_the_recorded_name
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:193
Functiontest_resolve_skill_path_checks_explicit_skill_against_run
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:171
Methodtest_resolve_skill_path_from_directory
Test resolve_skill_path with directory.
tests/cli/test_content_type_detection.py:273
Methodtest_resolve_skill_path_from_file
Test resolve_skill_path with file.
tests/cli/test_content_type_detection.py:278
Functiontest_resolve_skill_path_from_publication_target
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:149
Functiontest_resolve_skill_path_ignores_a_target_from_another_run
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:154
Functiontest_resolve_skill_path_ignores_case_of_skill_directory
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:186
Functiontest_resolve_skill_path_requires_hint_when_missing
(tmp_path: Path)
tests/evaluation/test_tier3_feedback_chat.py:164
Functiontest_resolve_vertex_model_id_aliases
Verify common Claude model names are mapped to Vertex versioned model IDs.
tests/test_harbor_runtime_preflight.py:3669
Functiontest_resolve_vertex_model_id_lowercases_unmapped_models
Ensure unmapped Claude model names without @ fall back to lowercase for Vertex endpoints.
tests/test_harbor_gke_mode.py:721
Functiontest_resolve_vertex_project_id_environ_dict
Verify _resolve_vertex_project_id respects an explicit environ dictionary.
tests/test_harbor_runtime_preflight.py:3934
Functiontest_resolve_vertex_region_whitespace_resilience
Whitespace-only region evaluates to fallback, while padded region is stripped.
tests/test_harbor_runtime_preflight.py:3925
Methodtest_resolve_workflows_path_from_directory
Test resolve_workflows_path with directory.
tests/cli/test_content_type_detection.py:292
Methodtest_resolve_workflows_path_from_file
Test resolve_workflows_path with workflow-rules.mdc file.
tests/cli/test_content_type_detection.py:297
Functiontest_resolved_results_are_classified_once
(monkeypatch)
tests/reporting/test_shared_run_status.py:133
Functiontest_resolves_existing_tag_and_checks_manifest_content
(monkeypatch)
tests/ci/test_release_image.py:44
Methodtest_resolves_nested_directory_and_file
(self, tmp_path: Path)
tests/utils/test_helpers.py:223
Methodtest_resource_based
(self, tmp_path)
tests/validators/test_quality_score.py:238
Methodtest_resource_budget_exhaustion_keeps_the_finding
( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, budget_name: str )
tests/validators/test_skillspector_scope.py:1095
Functiontest_resource_diagnostics_do_not_imply_activation
()
tests/reporting/test_activation_validation.py:349
Functiontest_resources_flow_from_harbor_artifacts_to_json_and_html
(tmp_path, missing_tokens)
tests/reporting/test_resource_comparison.py:53
Functiontest_response_must_identify_same_project_default_branch_pipeline
(request_env, monkeypatch, payload, capsys)
tests/ci/test_gitlab_feedback_request.py:194
← previousnext →21,301–21,400 of 24,741, ranked by callers