Code
Hub
Workspaces
Following
Trending
Connect
MCP
copy
Create free account
hub
/
github.com/NVIDIA/SkillEvaluator
/ functions
Functions
24,741 in github.com/NVIDIA/SkillEvaluator
⨍
Functions
24,741
◇
Types & classes
1,292
↳
Endpoints
703
Function
test_agent_failure_quoting_docker_address_pool_error_keeps_its_reason
(tmp_path: Path)
tests/test_harbor_collector_runtime_failures.py:2870
Function
test_agent_home_and_codex_workdir_are_resolved_per_call
(security)
tests/tier3/test_private_plugin_security.py:101
Function
test_agent_identity_survives_an_exhausted_shared_native_text_budget
()
tests/reporting/test_native_evidence.py:337
Function
test_agent_inheriting_all_tools_with_write_capable_mcp_is_medium
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:184
Function
test_agent_inheriting_all_tools_without_mcp_is_advisory
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:244
Function
test_agent_lift_status_is_independent_of_run_status
(tmp_path)
tests/test_tier3_history.py:252
Function
test_agent_model_is_trimmed_during_config_loading
(tmp_path: Path)
tests/test_evals_config_branding.py:53
Function
test_agent_models_preserve_native_catalog_ids
()
tests/ci/test_model_config.py:130
Function
test_agent_only_validation_accepts_actual_harbor_022_single_step_result
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:302
Function
test_agent_only_validation_accepts_harbor_0132_unscored_multistep_agent_success
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:368
Function
test_agent_only_validation_accepts_visible_agent_artifacts_in_docker_mode
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:4077
Function
test_agent_only_validation_accepts_visible_multistep_agent_artifacts_in_docker_mode
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:4088
Function
test_agent_only_validation_does_not_follow_agent_authored_symlinks
A link into the host must not be accepted as proof the mount worked.
tests/test_harbor_runtime_preflight.py:4217
Function
test_agent_only_validation_does_not_follow_nested_step_ancestor_symlink
Each path component leading to a nested step's agent tree is untrusted.
tests/test_harbor_runtime_preflight.py:4251
Function
test_agent_only_validation_does_not_follow_symlinked_agent_directory
The walk must reject a link at its root, not just links found below it.
tests/test_harbor_runtime_preflight.py:4234
Function
test_agent_only_validation_does_not_follow_symlinked_agent_subdirectories
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:4287
Function
test_agent_only_validation_finds_nested_step_agent_artifacts
Harbor declares StepConfig.name as a bare str, so a step name may nest.
tests/test_harbor_runtime_preflight.py:4178
Function
test_agent_only_validation_ignores_empty_files_when_checking_visibility
A zero-byte placeholder is not evidence that container writes reached the host.
tests/test_harbor_runtime_preflight.py:4140
Function
test_agent_only_validation_ignores_step_names_that_escape_the_trial
A traversing step name must not be joined onto the trial directory.
tests/test_harbor_runtime_preflight.py:4202
Function
test_agent_only_validation_rejects_empty_malformed_or_failed_multistep_results
( tmp_path: Path, step_results: object, expected: str, )
tests/test_harbor_runtime_preflight.py:475
Function
test_agent_only_validation_rejects_empty_nested_step_agent_directory
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:4190
Function
test_agent_only_validation_rejects_incomplete_harbor_0132_state
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:645
Function
test_agent_only_validation_rejects_invisible_agent_artifacts_in_docker_mode
An empty host-side agent tree means the daemon never shared the results directory.
tests/test_harbor_runtime_preflight.py:4117
Function
test_agent_only_validation_rejects_missing_malformed_or_failed_trial_result
( tmp_path: Path, trial_payload: str | None, expected: str, )
tests/test_harbor_runtime_preflight.py:683
Function
test_agent_only_validation_rejects_missing_or_malformed_job_result
( tmp_path: Path, result_text: str | None, )
tests/test_harbor_runtime_preflight.py:658
Function
test_agent_only_validation_rejects_mixed_single_and_multistep_agent_results
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:419
Function
test_agent_only_validation_rejects_non_successful_harbor_0132_states
( tmp_path: Path, counter: str, expected: str, )
tests/test_harbor_runtime_preflight.py:518
Function
test_agent_only_validation_skips_mount_check_outside_docker_mode
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:4131
Function
test_agent_only_validation_surfaces_first_errored_trial_exception
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:534
Function
test_agent_only_validation_surfaces_multistep_trial_exception
(tmp_path: Path)
tests/test_harbor_runtime_preflight.py:566
Function
test_agent_pair_assigns_the_full_concurrency_budget_when_baseline_is_skipped
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runner_status.py:1110
Function
test_agent_pair_launches_nothing_after_cancellation
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, stop_on_pass: bool, )
tests/test_harbor_runner_status.py:1216
Function
test_agent_pair_launches_only_the_baseline_job
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stop_on_pass: bool )
tests/tier3/test_skip_with_skill.py:386
Function
test_agent_pair_treats_concurrency_as_a_global_condition_cap
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, configured_concurrency: int, )
tests/test_harbor_runner_status.py:1063
Function
test_agent_payload_includes_evidence_volume_with_per_arm_counts
()
tests/reporting/test_evidence_volume_rendering.py:357
Function
test_agent_phase_excludes_build_and_verifier
()
tests/tier3/test_resource_usage.py:69
Function
test_agent_plugins_bad_version_remains_advisory_while_schema_version_blocks
()
tests/validators/test_private_format_boundaries.py:144
Function
test_agent_plugins_cursor_namespace_uses_cursor_semantics
(tmp_path: Path)
tests/validators/test_plugin_static_risk_review_fixes.py:182
Function
test_agent_plugins_mcp_schema_and_missing_transport_remain_visible
(tmp_path)
tests/test_plugin_discovery.py:317
Function
test_agent_plugins_schema_version_requires_the_exact_url
(kind)
tests/test_plugin_format_profiles.py:58
Function
test_agent_powershell_grant_denied_by_disallowed_tools_is_quiet
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:174
Function
test_agent_powershell_grant_is_worded_by_powershell
Regression: the remediation for a PowerShell grant said to deny Bash, and denying Bash hid the grant.
tests/validators/test_plugin_component_risk.py:163
Function
test_agent_provider_config_claude_vertex_decoupling
Verify Claude on Vertex agent route does not inherit evaluator base_url or region.
tests/test_provider_agent_routing.py:427
Function
test_agent_quality_failure_takes_precedence_over_missing_dimension
(security: float)
tests/evaluation/test_tier3_benchmark_contract.py:381
Function
test_agent_quality_failure_takes_precedence_over_non_finite_dimension
(correctness: object)
tests/evaluation/test_tier3_benchmark_contract.py:425
Function
test_agent_runtime_failures_get_no_address_pool_note
(exception_type: str)
tests/test_harbor_collector_runtime_failures.py:2861
Function
test_agent_runtime_preflight_cli_value_overrides_config
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_config_key_parity.py:369
Function
test_agent_runtime_preflight_config_key_disables_preflight
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_config_key_parity.py:294
Function
test_agent_runtime_preflight_config_key_enables_preflight
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_config_key_parity.py:324
Function
test_agent_runtime_preflight_defaults_to_disabled
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_config_key_parity.py:312
Function
test_agent_scoped_bash_and_accept_edits
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:149
Function
test_agent_segments_are_independent
(tmp_path)
tests/test_tier3_history.py:216
Function
test_agent_segments_ignore_other_agents_changes
(tmp_path)
tests/test_tier3_history.py:269
Function
test_agent_setup_uses_the_shared_discovery_budget
(tmp_path, monkeypatch)
tests/ci/test_ci_worker_diagnostics.py:167
Function
test_agent_skill_dataset_passes_the_strict_validate_its_docs_recommend
()
tests/tier3/test_dataset_baseline_fairness.py:1597
Function
test_agent_sums_within_range_can_still_pass_it_over_all_agents
()
tests/tier3/test_token_usage.py:604
Function
test_agent_timeout_invalidates_reward_and_is_reported_as_trial_failure
A timeout can leave verifier rewards behind, but it is not a valid scored trial.
tests/test_harbor_collector_runtime_failures.py:676
Function
test_agent_v1_needs_schema_for_hermes
(tmp_path: Path)
tests/harness/test_plugin_compat.py:77
Function
test_agent_verifier_and_ambiguous_failures_are_never_retried
(tmp_path: Path, result: dict)
tests/tier3/test_plugin_execution.py:78
Function
test_agent_wildcard_tools_is_medium
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:179
Function
test_agent_with_unrestricted_bash_and_bypass_is_high
(tmp_path: Path)
tests/validators/test_plugin_component_risk.py:129
Function
test_agentskills_evals_json_is_accepted_without_deprecation_warning
(tmp_path)
tests/tier3/test_evals_config.py:699
Function
test_agentskills_evals_json_rejects_non_object_items
(tmp_path)
tests/tier3/test_evals_config.py:752
Function
test_agentskills_evals_json_reports_authored_required_field_names
(tmp_path)
tests/tier3/test_evals_config.py:773
Function
test_agentskills_evals_json_requires_skill_name
(tmp_path)
tests/tier3/test_evals_config.py:733
Function
test_aggregate_attempt_counts_must_match_agent_evidence
()
tests/reporting/test_advisory_skip_verdict.py:3245
Function
test_aggregate_audit_cap_retains_critical_source_and_marks_truncation
(audit, tmp_path, ecosystem, critical_first)
tests/validators/test_private_dependency_audit.py:408
Function
test_aggregate_behavior_error_overrides_not_checked_step
(unchecked_behavior, failure)
tests/tier3/test_grader_validation.py:433
Function
test_aggregate_execution_preserves_hidden_child_error_occurrence_counts
()
tests/test_harbor_collector_runtime_failures.py:1093
Function
test_aggregate_keeps_each_tier_selection_so_unselected_skills_are_not_incomplete
(consumer, tmp_path)
tests/ci/test_run_tiers.py:709
Function
test_aggregate_only_tier1_blocks_despite_legacy_paid_requirements
(consumer, tmp_path, status)
tests/ci/test_run_tiers.py:360
Function
test_aggregate_rejects_a_completed_claim_missing_selected_skills
(consumer, tmp_path, status)
tests/ci/test_run_tiers.py:528
Function
test_aggregate_rejects_foreign_or_future_tier_attempts
(consumer, tmp_path, damage)
tests/ci/test_run_tiers.py:654
Function
test_aggregate_retains_every_tier_after_failure
()
tests/ci/test_all_tier_workflows.py:44
Function
test_aggregate_reuses_tier_evidence_from_jobs_not_rerun
()
tests/ci/test_all_tier_workflows.py:59
Function
test_aggregate_tier1_nonpass_still_blocks
(consumer, tmp_path, status)
tests/ci/test_run_tiers.py:379
Function
test_aggregate_uses_keyless_evidence_when_provider_job_is_not_eligible
(config, admission_case, tmp_path)
tests/ci/test_gitlab_enriched_admission.py:122
Method
test_agreeing_channels_do_not_conflict
(self)
tests/test_benchmark_source_provenance.py:817
Method
test_alias_amplified_field_fails_closed
(self, tmp_path: Path, field: str)
tests/embedding/test_extractor.py:733
Function
test_alias_amplified_frontmatter_fails_the_scan_before_any_embedding_request
( tmp_path: Path, monkeypatch, full_body: bool )
tests/embedding/test_similarity_limits.py:297
Method
test_alias_graph_id_is_rejected_with_a_bounded_finding
(self, tmp_path: Path)
tests/validators/test_schema.py:1518
Function
test_alias_order_is_run_then_attempt
()
tests/ci/test_publish_reports.py:316
Function
test_alias_origin_parses_after_insteadof_rewrite
(repository, all_release)
tests/test_ci_onboarding_all_tiers.py:849
Function
test_aliases_that_collide_within_one_request_are_refused
()
tests/test_transport_rules.py:100
Function
test_all_agent_and_baseline_arms_share_one_private_evaluator_snapshot
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:4144
Function
test_all_built_in_command_wrappers_stamp_canonical_check_identities
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_publication_evidence.py:83
Function
test_all_cases_cross_component_makes_the_cross_group_the_whole_ladder
()
tests/tier3/test_private_ladder_stats.py:304
Function
test_all_contained_formats_discover_and_analyze_without_execution
(tmp_path, monkeypatch, profile)
tests/test_plugin_discovery.py:49
Function
test_all_dataset_formats_reject_unsafe_ids_before_creating_output
( tmp_path: Path, filename: str, content: str, )
tests/test_harbor_case_id_safety.py:202
Function
test_all_dns_runs_before_one_credential_free_head_per_public_target
(er)
tests/validators/test_private_endpoint_resolution.py:31
Function
test_all_dynamic_task_toml_strings_and_keys_are_quoted_for_real_harbor
(tmp_path: Path)
tests/test_harbor_case_id_safety.py:347
Function
test_all_formats_keep_identity_source_files_and_static_risk
(tmp_path, monkeypatch, manifest)
tests/test_plugin_inventory.py:47
Function
test_all_harbor_jobs_are_navigable_without_attribution_guessing
(reporter)
tests/reporting/test_reporters.py:3048
Function
test_all_import_only_scripts_fall_back_to_the_first
(tmp_path: Path)
tests/tier3/test_dataset_primary_script.py:139
Function
test_all_invalid_latest_entries_return_collision_checked_hidden_sentinel
(tmp_path: Path)
tests/test_results_location.py:539
Function
test_all_judges_preserve_original_evidence_and_append_facts
(monkeypatch, implementation, judge_name)
tests/test_character_facts.py:148
Function
test_all_mixed_judges_leave_metric_dimension_and_overall_unavailable
(tmp_path: Path, native: bool)
tests/test_judge_lift_integrity.py:212
Function
test_all_na_tier_remains_na_without_claiming_execution
()
tests/reporting/test_final_render_context.py:253
Function
test_all_negative_dataset_scores_discoverability_but_cannot_pass_without_efficiency
(tmp_path)
tests/reporting/test_applicability_saved_runs.py:597
Method
test_all_pattern_categories_loaded
Verify all PII pattern categories are loaded.
tests/validators/test_security.py:6817
← previous
next →
12,501–12,600 of 24,741, ranked by callers