MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_invalid_agent_models_are_rejected_without_echo
(value)
tests/ci/test_model_config.py:150
Functiontest_invalid_agent_models_fail_before_files
(repository, routing_release, value)
tests/test_ci_onboarding_model_routing.py:132
Functiontest_invalid_agent_seconds_does_not_fall_back_to_legacy_duration
()
tests/reporting/test_dashboard_data.py:391
Functiontest_invalid_backend_completion_is_a_safe_gateway_error
(bridge_services: _BridgeServices)
tests/test_nvidia_build_bridge.py:1589
Functiontest_invalid_baseline_opt_in_fails_before_credential_launch
(launch, baselines, capsys)
tests/ci/test_run_tier1_image.py:295
Functiontest_invalid_bounds_are_rejected
(maximum, radius)
tests/test_character_facts.py:58
Functiontest_invalid_canary_specs_are_rejected
(security, raw)
tests/tier3/test_private_plugin_security.py:226
Functiontest_invalid_canonical_transport_is_not_treated_as_legacy_http
(tmp_path: Path)
tests/validators/test_openclaw_mcp.py:72
Functiontest_invalid_case_fields_are_reported_and_dropped
(field: str, value: Any, fragment: str)
tests/tier3/test_plugin_signals.py:1312
Functiontest_invalid_case_id_is_rejected_before_any_output_mutation
(tmp_path: Path, case_id: object)
tests/test_harbor_case_id_safety.py:73
Functiontest_invalid_case_selection_cannot_certify_publication
( tmp_path: Path, payload_selection: object, summary_selection: object, )
tests/reporting/test_advisory_skip_verdict.py:2879
Methodtest_invalid_catalog_is_not_reclassified_as_provider_failure
( self, _mock_client_cls, mock_registry_cls, tmp_path: Path )
tests/validators/test_similarity.py:245
Functiontest_invalid_claimed_custom_metric_values_are_omitted_without_score_drift
( tmp_path: Path, surface: str, )
tests/test_issue55_collector_truth.py:3261
Functiontest_invalid_claude_gateway_root_is_rejected_before_execution
(gateway)
tests/test_gateway_defaults.py:180
Functiontest_invalid_configuration_does_not_generate
(skill, configured, monkeypatch)
tests/tier3/test_workflow.py:248
Functiontest_invalid_contract
(fmt, version)
tests/test_trace_import.py:79
Functiontest_invalid_counters_are_rejected_and_named_without_their_values
(field: str, value: object)
tests/tier3/test_token_usage.py:104
Functiontest_invalid_custom_metric_cardinality_is_stripped_before_publication
( tmp_path: Path, surface: str, )
tests/test_issue55_collector_truth.py:3297
Methodtest_invalid_data
Test validation with invalid data.
tests/validators/test_frontmatter_parser.py:126
Functiontest_invalid_decoy_manifest_fails_before_provider_resolution
(tmp_path: Path, monkeypatch)
tests/tier3/test_decoy_workspace.py:136
Functiontest_invalid_entry_limits_fail_at_python_boundaries
(tmp_path: Path, value: object)
tests/embedding/test_similarity_limits.py:50
Functiontest_invalid_exec_environment_fails_without_serializing_value
( environment: dict[str, str], message: str, )
tests/test_harbor_secure_docker_environment.py:8608
Functiontest_invalid_existing_dataset_errors_without_overwrite
(skill, configured, monkeypatch, body)
tests/tier3/test_workflow.py:198
Functiontest_invalid_fields_are_ignored_by_graders
()
tests/tier3/test_plugin_signals.py:1443
Methodtest_invalid_filename_with_spaces
Test filename with spaces fails/warns.
tests/validators/test_rules_schema.py:451
Functiontest_invalid_fixture_content_dropped
()
tests/test_trace_import.py:305
Functiontest_invalid_fixture_reference_is_refused_before_task_generation
(tmp_path: Path, ref: str, explicit: bool)
tests/tier3/test_plugin_prepare_review.py:133
Functiontest_invalid_folder_choice_can_be_corrected_in_same_interview
(repository, release_file, tmp_path)
tests/test_ci_onboarding.py:112
Functiontest_invalid_generated_dataset_is_not_evaluated
(skill, configured, monkeypatch)
tests/tier3/test_workflow.py:337
Functiontest_invalid_git_baseline_does_not_change_imports
()
tests/test_script_invocation_differential.py:51
Functiontest_invalid_gym_timeout_is_rejected_before_launch
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, value: str )
tests/test_nemo_gym_backend.py:1150
Functiontest_invalid_harbor_counters_stay_unavailable
(tmp_path, value)
tests/tier3/test_private_usage_recovery_review2.py:117
Functiontest_invalid_higher_precedence_tier2_policy_uses_lower_valid_boolean
()
tests/golden/test_benchmark_card.py:757
Methodtest_invalid_history_is_rejected_without_creating_client
(self, messages, match: str)
tests/inference/test_llm_client.py:1382
Methodtest_invalid_id_is_advisory_by_default_and_blocks_in_registry
(self, tmp_path: Path, id_line: str, reason: str)
tests/validators/test_schema.py:1304
Functiontest_invalid_import_writes_nothing
(payload, tmp_path)
tests/tier3/test_external_metrics.py:285
Functiontest_invalid_independent_claude_base_url_fails_before_harbor_without_echoing_secrets
( monkeypatch: pytest.MonkeyPatch, provider_name: str, env_mode: str, configured_base_url: str
tests/test_harbor_runner_environment.py:795
Functiontest_invalid_json_is_reported_and_later_skills_still_run
(tmp_path)
tests/ci/test_run_tier1.py:565
Methodtest_invalid_json_raises
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:985
Methodtest_invalid_json_returns_none
Invalid JSON returns None.
tests/utils/test_tool_runner.py:77
Functiontest_invalid_late_entry_leaves_existing_generated_output_unchanged
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:420
Functiontest_invalid_later_config_file_server_does_not_replace_earlier
Regression: Claude Code drops a config-file server that fails its schema before merging, so the earlier valid server runs; the collection marked t
tests/validators/test_plugin_mcp.py:35
Functiontest_invalid_later_inline_server_still_replaces_earlier
An inline map is merged before it is checked, so its server replaces the file's either way.
tests/validators/test_plugin_mcp.py:54
Functiontest_invalid_latest_without_completed_run_returns_safe_absent_path
( tmp_path: Path, latest_kind: str, )
tests/test_results_location.py:511
Functiontest_invalid_legacy_attempt_threshold_does_not_crash_canonical_report
(invalid: float | int)
tests/reporting/test_unified_tier3_report.py:2565
Functiontest_invalid_llm_configuration_is_reported_without_running
(tmp_path, extra)
tests/ci/test_run_tier1.py:509
Functiontest_invalid_llm_request_never_launches_or_exposes_keys
(launch, mutation, capsys)
tests/ci/test_run_tier1_image.py:120
Functiontest_invalid_location_header_type_is_incomplete
(er)
tests/validators/test_private_endpoint_resolution.py:223
Functiontest_invalid_logical_scores_are_unavailable
(bad)
tests/tier3/test_private_comparison_contracts.py:164
Functiontest_invalid_lsp_declarations_do_not_override_a_configured_server
(tmp_path, first, later, effective)
tests/test_plugin_discovery.py:376
Functiontest_invalid_manifest_is_excluded_and_the_rest_are_compared
( tmp_path: Path, monkeypatch, manifest: str, reason: str )
tests/embedding/test_similarity_limits.py:330
Functiontest_invalid_marker_blocks_in_skill_atomic_replacement
(tmp_path: Path, native: bool)
tests/test_harbor_output_provenance.py:493
Functiontest_invalid_max_cases_is_rejected_before_staging
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, value: object, )
tests/test_tier3_max_cases.py:220
Methodtest_invalid_metadata_domain_reports_field_finding
An invalid domain, including an over-long single string, is a HIGH metadata.domain finding.
tests/validators/test_schema.py:407
Methodtest_invalid_mode_raises
(self)
tests/test_harbor_local_sandbox.py:315
Methodtest_invalid_mode_rejected
(self)
tests/test_harbor_local_sandbox.py:114
Functiontest_invalid_model_rejected
(model)
tests/test_hermes_agent.py:125
Functiontest_invalid_native_contract_is_a_blocking_schema_finding
(tmp_path: Path, fields: dict, field: str)
tests/test_openclaw_plugin_discovery.py:106
Functiontest_invalid_native_manifest_is_refused_before_staging
(tmp_path: Path, fields: dict)
tests/tier3/test_openclaw_prepare.py:147
Functiontest_invalid_native_report_cannot_publish
(trusted, tmp_path)
tests/ci/test_benchmark_publish.py:260
Functiontest_invalid_native_tool_timing_is_not_a_measurement
(extra: dict[str, Any])
tests/tier3/test_trajectory_timing.py:194
Functiontest_invalid_options_error_before_generating
(skill, configured, monkeypatch, options)
tests/tier3/test_workflow.py:222
Functiontest_invalid_or_ambiguous_input_is_rejected_before_writing
( tmp_path: Path, data: object, skill_name: str | None, error: str )
tests/tier3/test_dataset_migration.py:141
Functiontest_invalid_or_duplicate_repository_tree_paths_rejected
(trusted, monkeypatch, path)
tests/ci/test_benchmark_publish.py:499
Functiontest_invalid_or_dynamic_metadata_is_incomplete
(tmp_path, monkeypatch, text)
tests/validators/test_target_dependencies.py:84
Functiontest_invalid_or_missing_api_identity_fails_closed
(gate, monkeypatch, key, value, capsys)
tests/ci/test_gitlab_default_head_admission.py:133
Functiontest_invalid_or_unsafe_records_are_omitted
(mutate: Callable[[dict], object])
tests/test_skillspector_provenance.py:223
Methodtest_invalid_override_never_falls_back
(self, monkeypatch)
tests/utils/test_scanner_install.py:140
Functiontest_invalid_paired_case_counts_are_unavailable
(paired_cases)
tests/reporting/test_dataset_sufficiency_rendering.py:180
Functiontest_invalid_physical_reward_number_fails_closed_without_crashing
( tmp_path: Path, invalid_reward: float | int, )
tests/test_harbor_collector_runtime_failures.py:1622
Functiontest_invalid_policy_fails_closed_before_any_request
(verifier, monkeypatch, name, value)
tests/tier3/test_judge_policies.py:774
Functiontest_invalid_policy_keeps_the_ragas_goal_scorer_off
(verifier, monkeypatch, name, value)
tests/tier3/test_judge_policies.py:791
Functiontest_invalid_price_file_fails_without_provenance
(tmp_path, contents)
tests/tier3/test_price_provenance.py:89
Functiontest_invalid_price_scalars_have_configuration_errors
(tmp_path: Path, field: str, value: object)
tests/tier3/test_price_file_validation.py:42
Functiontest_invalid_private_scan_canary_is_incomplete
(security)
tests/tier3/test_private_plugin_security.py:322
Functiontest_invalid_publisher_revision_is_rejected_before_writing
(tmp_path, publisher_ref)
tests/ci/test_render_templates.py:96
Functiontest_invalid_quality_threshold_fails_without_running_scanner
(tmp_path, score)
tests/ci/test_run_tier1.py:1235
Functiontest_invalid_recorded_image_provenance_is_rejected
(tmp_path, execution)
tests/ci/test_publish_reports.py:1031
Functiontest_invalid_release_pin_stops_before_writing
(repository, release_file)
tests/test_ci_onboarding.py:259
Functiontest_invalid_replacement_preserves_previous_verifier
(tmp_path: Path, source_kind: str)
tests/test_harbor_local_upload.py:97
Methodtest_invalid_reply_is_retried_once_with_the_problem_then_scored
(self, tmp_path, first_reply, problem)
tests/validators/test_rubric_eval.py:1374
Functiontest_invalid_reply_is_retried_once_with_the_specific_problem
( judge_module, monkeypatch, metric: str, invalid: str, problem: str )
tests/tier3/test_judge_contracts.py:652
Functiontest_invalid_report_urls_fail_before_scanning_and_keep_diagnostics
(tmp_path, variable, url)
tests/ci/test_run_tier1.py:1136
Functiontest_invalid_resource_override_does_not_generate
(skill, configured, monkeypatch)
tests/tier3/test_workflow.py:387
Methodtest_invalid_response_index_does_not_echo_provider_content
(self)
tests/embedding/test_client.py:206
Functiontest_invalid_retained_runtime_is_rejected_before_html_rendering
(entrypoint)
tests/reporting/test_plugin_reports_runtime_composition.py:75
Functiontest_invalid_review
(workspace, decision)
tests/test_trace_import.py:232
Functiontest_invalid_root_reward_number_fails_closed_and_persists_strict_json
( tmp_path: Path, invalid_reward: float | int, )
tests/test_harbor_collector_runtime_failures.py:1419
Functiontest_invalid_runtime_is_unavailable
(runtime: object)
tests/evaluation/test_tier3_feedback_chat.py:220
Functiontest_invalid_scalar_limits_fail_at_python_boundaries
(value: object)
tests/embedding/test_similarity_limits.py:60
Functiontest_invalid_scan_worker_count_fails_before_scanning
(tmp_path, value)
tests/ci/test_run_tier1.py:1465
Functiontest_invalid_sections_are_rejected_before_rendering
(mutation)
tests/reporting/test_plugin_evidence_report.py:87
Functiontest_invalid_selection_is_error_without_full_scan_fallback
(tmp_path, selection)
tests/ci/test_run_tier1.py:1124
Methodtest_invalid_skill_name
Test validation fails for invalid skill name (not kebab-case).
tests/validators/test_schema.py:65
Methodtest_invalid_skillspector_json_fails_closed
A completed scanner process without parseable output is not a clean scan.
tests/validators/test_security.py:1420
Functiontest_invalid_source_keeps_failure_exit
(skill, content)
tests/tier3/test_dataset_coverage.py:109
Functiontest_invalid_step_reward_number_cannot_bypass_missing_root_fail_closed
( tmp_path: Path, invalid_reward: float | int, )
tests/test_harbor_collector_runtime_failures.py:1708
Functiontest_invalid_suite
(suite_dir, mutation, message)
tests/tier3/test_grader_validation.py:71
Functiontest_invalid_tier3_fingerprint_uses_bounded_fallback_without_full_json_dump
( monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_advisory_skip_verdict.py:1725
Functiontest_invalid_tier_selection_fails_before_writes
(repository, all_release, tiers)
tests/test_ci_onboarding_all_tiers.py:253
← previousnext →17,501–17,600 of 24,741, ranked by callers