MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / endpoints

Endpoints703 in github.com/NVIDIA/SkillEvaluator

Routetest_skillspector_timeout_marks_incomplete
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_scan_incomplete.py:None
Routetest_skillspector_unavailable_adds_warning
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_unknown_occurrences_preserve_visible_score_floor
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_reconciles_finding_counts
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_rejects_invalid_finding_counts
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_rejects_invalid_score_identity_fields
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_requires_completeness_fields
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_requires_finding_counts
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_contract_requires_finding_id
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_report_rejects_duplicate_component_identities
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_report_rejects_incomplete_component_evidence
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_report_requires_component_inventory
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_versioned_report_requires_issue_path
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skillspector_zero_confidence_does_not_consume_occurrence_weight
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_skipped_stale_publication_never_deploys_or_comments
pytest.mark.parametrize( ("event", "attempt"), [("workflow_run", "1"), ("workflow_dispatch", "1"), ("workf
tests/ci/test_publisher_workflow.py:None
Routetest_small_accuracy_and_goal_budgets_retain_final_response_excerpt
pytest.mark.parametrize("budget_val", [100, 200])
tests/tier3/test_evidence_compiler.py:None
Routetest_staged_digest_excluded_task_inputs_do_not_bind_publication
pytest.mark.parametrize( ("authored_parent", "excluded_input"), [ ("evals/files", "node_module
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_staged_scan_classifies_against_the_original_skill
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_scope.py:None
Routetest_staged_scan_preserves_shipped_bytecode_for_sc8
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_staged_scan_verifies_images_in_the_original_tree
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_staging_rejects_output_that_contains_evaluator_sources_before_mutation
pytest.mark.parametrize("output_kind", ["evals", "skill"])
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_standalone_efficiency_preserves_its_smaller_output_shape
pytest.mark.parametrize("calls", [[], [{"action": "bash", "action_input": {"command": "pwd"}}]])
tests/tier3/test_shared_check_adapters.py:None
Routetest_static_and_llm_stages_report_one_critical
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_streaming_redactor_repeated_prefix_scan_is_linear
pytest.mark.parametrize( ("secret_length", "output_size"), [(1024, 20_000), (4096, 5_000)], )
tests/test_harbor_secure_docker_environment.py:None
Routetest_streaming_redactor_scale_is_linear_in_input
pytest.mark.parametrize( ("secret_count", "output_size"), [(100, 100_000), (500, 20_000)], )
tests/test_harbor_secure_docker_environment.py:None
Routetest_strict_rejection_is_not_retried_or_handed_on
pytest.mark.parametrize( ("code", "body"), [ (400, b"content_filter"), (401, b"unautho
tests/tier3/test_judge_policies.py:None
Routetest_strict_structured_stop_errors_end_the_judge
pytest.mark.parametrize( "body", [ b'{"error":{"code":"budget_exceeded","message":"Budget has
tests/tier3/test_judge_policies.py:None
Routetest_structured_target_outside_the_scan_is_dropped
pytest.mark.parametrize( "target", ["/outside.pl", "../outside.pl", "scripts/../../outside.pl", "C:/ou
tests/validators/test_skillspector_reference_diagnostics.py:None
Routetest_subshell_cd_option_or_attached_semicolon_still_reads_the_script
pytest.mark.parametrize(("event", "matcher"), [("PostToolUse", "Edit|Write"), ("SessionStart", None), ("Stop",
tests/validators/test_plugin_component_risk.py:None
Routetest_successful_llm_enrichment_cannot_erase_deterministic_findings
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_successful_output_is_not_failure
pytest.mark.parametrize( "observation", [ _codex_output(0, "0 errors"), "exit_code=0\n
tests/tier3/test_error_recovery.py:None
Routetest_suite_validates_declarative_entry_checks
pytest.mark.parametrize( "checks", [ None, {}, [{"type": "output_matches", "pa
tests/tier3/test_grader_validation.py:None
Routetest_supported_activation_preserves_subcheck_average
pytest.mark.parametrize( ("call", "options", "activation_score", "execution_score"), [ ({"acti
tests/tier3/test_skill_activation_scoring.py:None
Routetest_symlink_evidence_blocks_traversal_inside_output
pytest.mark.parametrize( "link_command", [ "ln -s / /workspace/output/root", "python -
tests/tier3/test_agent_output_cleanup_scope.py:None
Routetest_system_npm_installer_still_provisions_node_and_npm
pytest.mark.parametrize("manager", ["apt-get", "apk"])
tests/tier3/test_installed_agent_bootstrap.py:None
Routetest_target_frontmatter_errors_fail_cleanly_without_writing
pytest.mark.parametrize( ("frontmatter", "message"), [ pytest.param( "name: yes\nd
tests/tier3/test_decoy_commands.py:None
Routetest_task_text_that_shares_the_skill_name_or_file_does_not_warn
pytest.mark.parametrize( "entry", [ # A skill named after the CLI it wraps: the baseline can r
tests/tier3/test_dataset_baseline_fairness.py:None
Routetest_template_safe_network_commands_match_eval_core
pytest.mark.parametrize( "cmd", [ "echo curl -d 'hello' https://example.com", "curl -s
tests/tier3/test_harbor_template_secret_patterns.py:None
Routetest_template_unsafe_network_commands_match_eval_core
pytest.mark.parametrize( "cmd", [ "curl -d 'hello' https://example.com", 'curl -H "Aut
tests/tier3/test_harbor_template_secret_patterns.py:None
Routetest_text_that_does_not_run_a_skill_script_does_not_warn
pytest.mark.parametrize( "assertion", [ "Saves report.pdf next to report.docx", "The a
tests/tier3/test_dataset_baseline_fairness.py:None
Routetest_the_lines_after_an_unquoted_value_keep_no_credential
pytest.mark.parametrize( "next_line", [ pytest.param(f"DB_PASSWORD={_BASIC_PASSWORD}", id="wor
tests/tier3/test_security_finding_evidence.py:None
Routetest_threshold_passed_to_find_duplicates
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_tier1_rejects_scanners_older_than_the_supported_release
pytest.mark.parametrize( ("command", "output", "status"), [ ("bandit", "bandit 1.9.3\n", "fail
tests/test_doctor.py:None
Routetest_tier1_scope_passes_where_the_general_scan_fails
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_scope.py:None
Routetest_tier3_output_in_the_skill_counts_as_not_inspected
pytest.mark.parametrize( ("line", "left_out"), [ ("Write the meeting summary to `r
tests/validators/test_security.py:None
Routetest_tier3_platform_jobs_are_skipped_for_unrelated_source_and_tests
pytest.mark.parametrize( "paths", [ [b"src/skillevaluator/reporting/html.py"], [b"src/
tests/test_ci_change_classifier.py:None
Routetest_tool_error_message_wins_over_provider_message
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_tool_names_map_to_declared_servers
pytest.mark.parametrize( ("agent", "fn", "servers", "tool"), [ ("claude-code", "my
tests/tier3/test_plugin_signals.py:None
Routetest_tool_only_trajectory_retains_latest_verification_failure
pytest.mark.parametrize( "budgets", [None, (7000, 11000)], ids=["defaults", "custom"] )
tests/tier3/test_evidence_compiler.py:None
Routetest_tool_strictness_is_preserved_in_chat_tools
pytest.mark.parametrize( ("translator", "payload"), [ ( responses_to_chat_request,
tests/test_nvidia_build_bridge.py:None
Routetest_transport_and_response_errors_do_not_echo_raw_content
pytest.mark.parametrize( "exception, expected", [ (APIConnectionError(request=httpx.Request("P
tests/inference/test_diagnostics.py:None
Routetest_truncated_report_stays_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unattributed_native_wrapper_observation_is_scanned_once
pytest.mark.parametrize( ("rendered", "expected_status"), [ ("text(one.output); text(two.outpu
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_unavailable_llm_report_metadata_is_incomplete_and_redacted_everywhere
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unbounded_json_integer_fails_closed
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_uncertain_evidence_keeps_generic_error
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unexpected_exit_does_not_echo_scanner_stdout
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_secrets_structured.py:None
Routetest_unexpected_llm_exit_redacts_provider_diagnostic_from_all_reports
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unexpected_skillspector_exit_fails_closed
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unexpected_skillspector_suppression_fails_closed
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_uninspectable_source_folder_is_unresolved_not_missing
pytest.mark.skipif(hasattr(os, "geteuid") and os.geteuid() == 0, reason="root ignores permission bits")
tests/test_plugin_manifest.py:None
Routetest_unique_content_passes
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_unknown_directories_cannot_supply_output_roots
pytest.mark.parametrize( "commands", [ ({"action": "bash", "action_input": {"command": "rm -rf
tests/tier3/test_agent_output_cleanup_scope.py:None
Routetest_unreadable_condition_trials_keep_other_rows
pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, reason="Needs POSIX permissions enforced for this use
tests/reporting/test_dashboard_data.py:None
Routetest_unreadable_structured_target_needs_the_ledger
pytest.mark.parametrize("target", [["not a string"], {"unexpected": "object"}, True, "bad\npath"])
tests/validators/test_skillspector_reference_diagnostics.py:None
Routetest_unreadable_subdirectory_does_not_hide_other_results
pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, reason="Needs POSIX permissions enforced for this use
tests/reporting/test_dashboard_app.py:None
Routetest_unrecognized_origin_never_receives_generic_gitlab_token
pytest.mark.parametrize("command", [["status"], ["demo", "--no-input"]])
tests/test_ci_onboarding_all_tiers.py:None
Routetest_unrecovered_agent_fault_report_reason
pytest.mark.parametrize( "observation", [ "ERROR: test failed\nexit code 1", "Tracebac
tests/tier3/test_error_recovery.py:None
Routetest_unrelated_distant_success_does_not_recover
pytest.mark.parametrize( "command,retry,inspections,observation", [ ("pytest -q", "test -f rep
tests/tier3/test_error_recovery.py:None
Routetest_unrelated_link_does_not_fail_the_manifest_lookup
pytest.mark.parametrize( ("link", "target"), [("AGENTS.md", "CLAUDE.md"), ("LICENSE", "../LICENSE"), (
tests/test_plugin_manifest.py:None
Routetest_unresolved_tmp_expansions_keep_a_finding
pytest.mark.parametrize("target", ["/tmp/$VAR", "/tmp/build-$$", "/tmp/{a,b}", "/tmp/out-${RUN_ID}"])
tests/tier3/test_destructive_command_paths.py:None
Routetest_unsafe_answering_model_is_reported_as_unresolved
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_unsafe_footer_target_is_readable_without_hyperlink
pytest.mark.parametrize( "target", ["https://[invalid", "javascript:alert(1)", "/tmp/name\x1b]8;;https://e
tests/reporting/test_terminal_polish.py:None
Routetest_unsafe_results_roots_are_refused_without_creation
pytest.mark.parametrize("target", ["source", "ancestor", "linked"])
tests/tier3/test_plugin_runtime_identity.py:None
Routetest_unscoped_multi_wrapper_observation_is_not_duplicated_for_atomic_calls
pytest.mark.parametrize( "source", [ pytest.param( 'const result = await tools.exe
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_unsupported_transports
pytest.mark.parametrize( ("target", "status"), [ ({"name": "ws", "url": "wss://mcp.example.com
tests/tier3/test_mcp_proof.py:None
Routetest_untrusted_baseline_timeouts_keep_the_agent_failed
pytest.mark.parametrize( ("failed", "code"), [ ({"output_tokens": None}, "no_progress_evidence
tests/tier3/test_baseline_timeouts.py:None
Routetest_unusable_skillspector_baseline_blocks_the_scan
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_unverified_image_gap_stays_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_valid_363_chunk_skill_fits_scalar_budget
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_validate_copy_repo_rejects_repo_root_report_output_before_tier3
pytest.mark.parametrize("output_dir", [".", "missing/.."])
tests/test_commands.py:None
Routetest_validate_on_a_folder_of_skills_keeps_each_skill_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_validate_security_only_with_llm
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_validate_security_only_without_llm
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_verdict_without_a_model_adds_no_model_fields
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_verified_image_paths_keep_scanner_findings
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_verified_png_outside_agent_control_directories
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_verified_png_reference_completes_the_llm_stage
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_verified_png_reference_completes_the_scan
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_verifier_security_uses_staged_workdir
pytest.mark.parametrize( ("configured", "native", "command", "score"), [ ("/home/runner/trial-
tests/tier3/test_security_workdir_provenance.py:None
Routetest_warnings_quote_a_bounded_amount_of_dataset_text
pytest.mark.parametrize("field", ["prompt", "expected_output", "assertions"])
tests/tier3/test_dataset_baseline_fairness.py:None
Routetest_wider_embeddings_report_their_effective_chunk_limit
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_windows_junction_sources_are_rejected
pytest.mark.parametrize("location", ["runtime", "input"])
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_windows_walker_consumes_the_path_budget_like_the_posix_walker
pytest.mark.parametrize( ("make_tree", "max_paths", "relative_path", "actual"), [ (_ordered_bu
tests/utils/test_secure_fs_platform.py:None
Routetest_windows_walker_refuses_redirects_and_closes_every_handle
pytest.mark.parametrize( ("make_link", "match"), [ (lambda root, outside: (root / "linked").sy
tests/utils/test_secure_fs_platform.py:None
Routetest_zero_activation_cannot_receive_execution_credit
pytest.mark.parametrize( "call", [ {"action": "Read", "action_input": {"file_path": "/workspac
tests/tier3/test_skill_activation_scoring.py:None
Routetier3_history
click.option( "--output", type=click.Path(dir_okay=False, path_type=Path), default=None, help=
src/skillevaluator/cli.py:None
Routetier3_import_metrics
click.option( "-o", "--output-dir", type=click.Path(file_okay=False, path_type=Path), required
src/skillevaluator/cli.py:None
Routetier3_import_traces
click.option( "--output", "-o", type=click.Path(file_okay=False, path_type=Path), help="Review
src/skillevaluator/cli.py:None
Routetier3_migrate_dataset
click.option( "--skill-name", default=None, help="Target skill identity; required when it cannot be inferr
src/skillevaluator/cli.py:None
← previousnext →601–700 of 703, ranked by callers