MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Methodtest_dict_creates_model
Test that dict input creates model instance.
tests/models/test_field_validators.py:48
Functiontest_different_condition_metric_sets_preserve_scores_without_overall_lift
( tmp_path: Path, reverse_conditions: bool )
tests/test_issue55_collector_truth.py:482
Functiontest_different_condition_score_contracts_do_not_claim_a_causal_skill_effect
(tmp_path: Path)
tests/test_tier3_case_summary.py:1998
Functiontest_different_declared_policies_are_not_called_one_overall_lift
()
tests/tier3/test_private_comparison_contracts.py:320
Functiontest_differently_cased_authored_directory_is_not_a_runtime_exclusion
(tmp_path)
tests/tier3/test_plugin_runtime_identity.py:286
Methodtest_differently_cased_target_path
(self, tmp_path: Path, write_skill)
tests/embedding/test_catalog.py:871
Functiontest_digest_authorization_evidence_in_findings
(copy: str, command: str, expected: str)
tests/tier3/test_security_finding_evidence.py:440
Functiontest_digest_authorization_keeps_its_scheme_and_the_rest_of_the_command
(text: str, expected: str)
tests/tier3/test_security_finding_evidence.py:432
Functiontest_digest_binds_authored_eval_inputs_but_not_tier3_run_outputs
(tmp_path: Path)
tests/test_publication_identity.py:326
Functiontest_digest_binds_empty_directories
(tmp_path: Path)
tests/test_publication_identity.py:192
Functiontest_digest_binds_executable_bits
(tmp_path: Path)
tests/test_publication_identity.py:205
Functiontest_digest_binds_normalized_relative_paths_and_file_bytes
(tmp_path: Path)
tests/test_publication_identity.py:425
Functiontest_digest_binds_the_root_directory_node
(tmp_path: Path)
tests/test_publication_identity.py:221
Functiontest_digest_binds_the_target_root_kind
(tmp_path: Path)
tests/test_publication_identity.py:456
Functiontest_digest_does_not_declare_packaged_headings_when_the_key_is_omitted
(tmp_path: Path)
tests/test_quality_config.py:457
Functiontest_digest_does_not_hide_symlinks_behind_generated_exclusions
(tmp_path: Path)
tests/test_publication_identity.py:482
Functiontest_digest_excludes_filesystem_case_alias_of_git_metadata
(tmp_path: Path, kind: str)
tests/test_publication_identity.py:303
Functiontest_digest_excludes_nested_git_pointer_files
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, descriptor_backend: bool, )
tests/test_publication_identity.py:282
Functiontest_digest_excludes_only_root_generated_benchmark_card
(tmp_path: Path)
tests/test_publication_identity.py:374
Functiontest_digest_excludes_root_git_metadata_for_directory_and_file_checkouts
(tmp_path: Path)
tests/test_publication_identity.py:250
Functiontest_digest_handles_case_alias_of_canonical_eval_results_by_filesystem_semantics
(tmp_path: Path)
tests/test_publication_identity.py:348
Functiontest_digest_is_deterministic_and_ignores_only_generated_artifacts
(tmp_path: Path)
tests/test_publication_identity.py:41
Functiontest_digest_is_not_masked_by_a_credential_elsewhere_in_the_skill
The digest is added after redaction, so a short password that occurs in its hex stays visible there.
tests/validators/test_pii_waivers.py:484
Methodtest_digest_pinned_container_is_published_verbatim
Escaping ``@`` to ``@`` would publish an unresolvable image reference.
tests/test_benchmark_source_provenance.py:292
Methodtest_digest_pinned_container_reference_is_accepted
``@sha256:`` is the only immutable way to pin the evaluator build.
tests/test_benchmark_source_provenance.py:154
Methodtest_digest_pinned_container_satisfies_a_pass
(self, tmp_path: Path)
tests/test_benchmark_source_provenance.py:1081
Functiontest_digest_rejects_a_manifest_without_a_canonical_file_digest
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_publication_identity.py:645
Functiontest_digest_rejects_hard_linked_files
(tmp_path: Path)
tests/test_publication_identity.py:496
Functiontest_digest_rejects_normalized_path_collisions
(tmp_path: Path)
tests/test_publication_identity.py:502
Functiontest_digest_rejects_root_and_nested_symlinks
(tmp_path: Path)
tests/test_publication_identity.py:471
Functiontest_digest_rejects_source_mutation_while_hashing
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/test_publication_identity.py:519
Functiontest_digest_rejects_special_files
(tmp_path: Path)
tests/test_publication_identity.py:490
Functiontest_digest_supports_a_single_regular_file_target
(tmp_path: Path)
tests/test_publication_identity.py:441
Functiontest_dimension_definitions_open_natively_beneath_each_dimension
(browser, reports, javascript)
tests/reporting/test_report_browser.py:825
Functiontest_dimension_filters_and_condition_retain_baseline_for_comparison
(tmp_path: Path)
tests/reporting/test_dashboard_app.py:175
Functiontest_dimension_table_matches_dimension_mapping
()
tests/test_metrics_reference_docs.py:201
Functiontest_dimension_verdict_bands_match_constants
()
tests/test_metrics_reference_docs.py:218
Functiontest_dimension_verdict_boundaries_pass_publication_gate
( tmp_path: Path, dimension_score: float, expected_verdict: str, )
tests/test_public_benchmark_gate.py:1024
Functiontest_direct_audit_inputs_cannot_inject_requirements_or_image_flags
(audit, tmp_path)
tests/validators/test_private_dependency_audit.py:352
Functiontest_direct_baseline_staging_rejects_pre_agent_setup_before_output_mutation
( tmp_path: Path, task_source: str, )
tests/test_harbor_runtime_skill_isolation.py:2787
Functiontest_direct_canary_check_rejects_invalid_specs_without_raising
(security)
tests/tier3/test_private_plugin_security.py:383
Functiontest_direct_case_diagnosis_still_loads_snapshot
(tmp_path: Path)
tests/test_tier3_case_summary.py:1842
Functiontest_direct_custom_artifact_is_rejected_before_adding_incompatible_publisher
(repository, release_file)
tests/test_ci_onboarding.py:593
Functiontest_direct_evaluator_keeps_its_existing_pin_and_inputs
()
tests/test_ci_onboarding_existing.py:81
Methodtest_direct_frontmatter_license_matches_license_file
Matching SKILL.md and LICENSE declarations should pass.
tests/validators/test_license.py:349
Functiontest_direct_help_explains_generation_without_changing_expert_help
()
tests/tier3/test_workflow.py:274
Functiontest_direct_max_chars_argument_reserves_tool_history_headroom
(max_chars: int)
tests/tier3/test_behavior_evidence.py:1049
Methodtest_direct_node_modules_bin_shim_does_not_expose_siblings
( self, fake_home_tree: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch,
tests/test_harbor_local_sandbox.py:1295
Methodtest_direct_rule_scan_requires_two_items
(self, _mock_client_cls, mock_registry_cls, tmp_path: Path)
tests/validators/test_similarity.py:91
Functiontest_direct_run_probes_once_before_provider_resolution_and_staging
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/test_nemo_gym_backend.py:2777
Functiontest_direct_run_rejects_finite_multiplier_that_overflows_default_timeouts
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runner_environment.py:2010
Functiontest_direct_run_rejects_incompatible_gym_before_staging
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/test_nemo_gym_backend.py:775
Functiontest_direct_run_rejects_nonfinite_timeout_multiplier_before_creating_results
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, timeout_multiplier: float, )
tests/test_harbor_runner_environment.py:1963
Functiontest_direct_run_rejects_overflowing_pass_threshold_before_creating_results
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runner_environment.py:2033
Functiontest_direct_run_rejects_overflowing_timeout_multiplier_before_creating_results
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runner_environment.py:1987
Functiontest_direct_run_returns_configuration_error_for_cyclic_environment_kwargs
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runner_environment.py:1943
Methodtest_direct_runtime_file_does_not_expose_siblings
( self, fake_home_tree: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch,
tests/test_harbor_local_sandbox.py:1272
Functiontest_direct_tier1_default_reports_stay_outside_publication_target
(skill, tmp_path, monkeypatch)
tests/cli/test_tier_workflows.py:94
Functiontest_direct_tier1_options_before_path_and_failed_findings
(skill, tmp_path)
tests/cli/test_tier_workflows.py:113
Functiontest_direct_tier1_runs_only_requested_static_checks
(skill, tmp_path)
tests/cli/test_tier_workflows.py:81
Functiontest_direct_tier3_preserves_conditions_and_bars_at_narrow_width
(width)
tests/reporting/test_terminal_polish.py:140
Functiontest_direct_tier_json_names_the_clean_skill_and_records_the_target_as_given
A clean result names no file and has no quality score, so the JSON names the skill from the target path, which records the path as typed. ``.`
tests/cli/test_tier_workflows.py:353
Functiontest_direct_workflow_generates_missing_dataset_then_evaluates
(skill, configured, monkeypatch)
tests/tier3/test_workflow.py:120
Functiontest_direct_workflow_rejects_skipping_both_arms_before_dataset_generation
( skill: Path, captured_options: list[Any], monkeypatch: pytest.MonkeyPatch )
tests/tier3/test_skip_with_skill.py:142
Functiontest_direct_workflow_reuses_dataset_and_forwards_evaluation_options
(skill, configured, monkeypatch, tmp_path)
tests/tier3/test_workflow.py:69
Functiontest_direct_workflow_uses_the_provider_native_agent
( skill: Path, monkeypatch: pytest.MonkeyPatch, provider: str, credential: str, expected_a
tests/tier3/test_workflow.py:157
Functiontest_directly_negated_reads_remain_fair
(negation)
tests/tier3/test_dataset_baseline_fairness.py:757
Methodtest_directories_left_out_that_the_line_does_not_name_are_ignored
The scanned copy leaves out ``evals/results`` and ``.git``; the line names neither.
tests/validators/test_security.py:10341
Methodtest_directory_alias_counts_toward_path_budget
(self, skill_root: Path, monkeypatch)
tests/deduplication/utils/test_skill_collector.py:354
Methodtest_directory_alias_to_pruned_regular_directory_is_skipped
(self, tmp_path: Path, write_skill)
tests/embedding/test_extractor.py:556
Methodtest_directory_alias_to_pruned_regular_directory_is_skipped
(self, skill_root: Path)
tests/deduplication/utils/test_skill_collector.py:370
Functiontest_directory_and_file_inside_it_load_once
(tmp_path: Path)
tests/reporting/test_dashboard_app.py:341
Functiontest_directory_enumeration_stops_before_materializing_all_entries
(tmp_path: Path, monkeypatch)
tests/reporting/test_dashboard_data.py:928
Functiontest_directory_file_budget_counts_malformed_reports
(tmp_path: Path, monkeypatch)
tests/reporting/test_dashboard_data.py:1010
Methodtest_directory_form_markdown_links_are_not_nested_references
(self, tmp_path: Path, target: str)
tests/validators/test_quality_score.py:1365
Methodtest_directory_form_readme_links_do_not_trigger_reference_finding
(self, tmp_path: Path, target: str)
tests/validators/test_quality_score.py:1122
Functiontest_directory_listing_stops_once_the_budget_cannot_fit
(tmp_path: Path)
tests/utils/test_secure_fs.py:534
Functiontest_directory_mutation_after_scandir_snapshot_fails_closed
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/utils/test_secure_fs.py:457
Functiontest_directory_reconstruction_keeps_links_from_recorded_engine_result
(tmp_path)
tests/reporting/test_recorded_harbor_uploads.py:158
Methodtest_directory_replacement_during_validation_never_reads_outside_manifest
( self, tmp_path: Path, monkeypatch )
tests/embedding/test_extractor.py:231
Functiontest_directory_skips_symlinks
(tmp_path: Path)
tests/reporting/test_dashboard_data.py:641
Functiontest_directory_swap_to_external_link_is_never_traversed
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/utils/test_secure_fs.py:221
Functiontest_directory_swap_to_external_symlink_after_manifest_is_rejected
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_secure_copy.py:308
Functiontest_directory_symlink_cycle_is_rejected
(tmp_path: Path)
tests/test_harbor_secure_copy.py:110
Methodtest_directory_traversal_error_is_actionable
(self, skill_root: Path, monkeypatch)
tests/deduplication/utils/test_skill_collector.py:595
Functiontest_dirty_repository_is_refused_before_pr_setup
(repository, release_file)
tests/test_ci_onboarding.py:281
Functiontest_disable_model_invocation_is_ignored_by_three_harnesses
(tmp_path: Path)
tests/harness/test_overlay.py:655
Functiontest_disabled_dotenv_blocks_files_even_in_the_operator_directory
(tmp_path: Path)
tests/test_harbor_dotenv_isolation.py:104
Functiontest_disabled_ecosystems_do_not_run_scanners
(audit, tmp_path)
tests/validators/test_private_dependency_audit.py:281
Functiontest_disabled_fallback_refusal_is_harbors_safety_refusal_error
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, lane: str )
tests/test_harbor_claude_refusal_fallback.py:640
Functiontest_disabled_metadata_preserves_prompt
(monkeypatch, setting)
tests/test_character_facts.py:64
Functiontest_disagreement_is_info_and_claude_errors_are_advisory
(tmp_path: Path)
tests/validators/test_private_claude_parity.py:99
Functiontest_disappearing_inventory_after_safe_manifest_is_a_secure_refusal
(modules, tmp_path, monkeypatch)
tests/deduplication/plugin/test_private_dedup.py:646
Methodtest_discover_rules_in_folder
(self, tmp_path: Path)
tests/embedding/test_extractor.py:609
Methodtest_discover_skills_in_folder
(self, tmp_path: Path, write_skill)
tests/embedding/test_extractor.py:431
Functiontest_discover_trajectories_ambiguous_folder_without_reward_uses_full_name
Without reward metadata, ambiguous ``__`` folders keep the full directory name.
tests/tier3/test_generate_dataset_results.py:219
Functiontest_discover_trajectories_finds_hermes_only_run
(tmp_path)
tests/tier3/test_generate_dataset_results.py:68
Functiontest_discover_trajectories_keeps_distinct_case_ids_with_double_underscore
Case ids containing ``__`` must not collapse to a shared prefix without metadata.
tests/tier3/test_generate_dataset_results.py:190
Functiontest_discover_trajectories_maps_harbor_trial_folder_to_case_id
Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id.
tests/tier3/test_generate_dataset_results.py:117
Functiontest_discover_trajectories_opencode_maps_harbor_folder_via_result_metadata
(tmp_path)
tests/tier3/test_generate_dataset_results.py:88
← previousnext →15,101–15,200 of 24,741, ranked by callers