MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / endpoints

Endpoints703 in github.com/NVIDIA/SkillEvaluator

Routetest_extended_codex_exit_one_record_keeps_the_quiet_probe_exemption
pytest.mark.parametrize( ("command", "output", "clean"), [ ("grep foo file.txt", "", True),
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_extended_codex_incomplete_allsettled_observation_falls_back_to_atomic
pytest.mark.parametrize( "observation", [ pytest.param("", id="empty"), pytest.param("
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_extended_codex_inline_and_single_quoted_wrappers_normalize
pytest.mark.parametrize( "source", [ pytest.param('text(await tools.exec_command({cmd:"pwd"}))
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_external_and_parent_markdown_links_are_not_nested_references
pytest.mark.parametrize( "target", [ "https://agentskills.io/specification.md",
tests/validators/test_quality_score.py:None
Routetest_external_baseline_keeps_its_path_for_a_staged_scan
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_failed_build_preserves_committed_index_and_coverage
pytest.mark.parametrize("failure", ["provider", "preflight", "minimum", "vector", "input"])
tests/embedding/test_collection_coverage.py:None
Routetest_failed_cluster_adds_no_model_beside_a_fallback_verdict
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_failed_execution_with_policy_exit_code_gets_provider_message
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_failed_llm_enrichment_preserves_deterministic_findings_and_is_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_failed_or_incomplete_inventory_is_incomplete
pytest.mark.parametrize( "output,exit_code", [ ("oops", 0), ('{"dependencies":[]}', 0)
tests/validators/test_target_dependencies.py:None
Routetest_failed_scan_does_not_create_or_replace_saved_index
pytest.mark.parametrize("existing_output", [False, True])
tests/embedding/test_pairwise_preflight.py:None
Routetest_failure_details_never_echo_argument_values
pytest.mark.parametrize( ("command", "secret"), [ ("git clone https://deploy:S3cr3
tests/tier3/test_plugin_signals.py:None
Routetest_failure_detection_agrees
pytest.mark.parametrize( "observation", [ _codex_output(7, "All done"), "exit_code=127
tests/tier3/test_error_recovery.py:None
Routetest_fairness_diagnostics_bound_untrusted_matched_text
pytest.mark.parametrize("field", ["prompt", "expected_output", "assertions"])
tests/tier3/test_dataset_baseline_fairness.py:None
Routetest_fallback_explains_safe_failure_and_labels_starter
pytest.mark.parametrize( "error, expected", [ ( APIStatusError( "p
tests/tier3/test_dataset_generation_progress.py:None
Routetest_fetch_then_parse_is_not_remote_code
pytest.mark.parametrize( "command", [ "curl -s https://api.example.com/status | python3 -m jso
tests/validators/test_plugin_static_risk_review_fixes.py:None
Routetest_finding_evidence_masks_a_value_escaped_again_next_to_a_marker_of_another_kind
pytest.mark.parametrize( ("output", "masked_output"), [ pytest.param(f"session {_JWT} saved",
tests/tier3/test_security_finding_evidence.py:None
Routetest_finding_evidence_redaction_keeps_ordinary_text
pytest.mark.parametrize( "text", [ "python -u train.py && pip install --user requests && mysql
tests/tier3/test_security_finding_evidence.py:None
Routetest_finding_message_names_at_most_64_locations_and_files
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_finding_metadata_populated
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_finding_on_a_symlink_target_stays_blocking
pytest.mark.parametrize( ("alias", "target", "skill_text"), [ ("SKILL.md", "docs/b
tests/validators/test_skillspector_scope.py:None
Routetest_finding_uses_explanation_and_remediation_from_skillspector
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_findings_map_back_to_original_paths
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_folder_scan_totals_applicability_across_skills
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_scope.py:None
Routetest_folder_without_skills_is_scanned_as_a_whole
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_force_only_rm_policy_survives_parser_limits
pytest.mark.parametrize(("target", "expected_score"), [("out.txt", 1.0), ("/etc/hosts", 0.0)])
tests/tier3/test_destructive_command_paths.py:None
Routetest_force_only_rm_resolves_relative_targets_after_cd
pytest.mark.parametrize( ("commands", "expected_score"), [ (["cd /etc && rm -f hosts"], 0.0),
tests/tier3/test_destructive_command_paths.py:None
Routetest_forced_tool_choice_translates_to_chat_contract
pytest.mark.parametrize( ("translator", "payload", "expected"), [ ( responses_to_c
tests/test_nvidia_build_bridge.py:None
Routetest_foreign_skipped_tier3_payload_cannot_waive_or_certify_target_policy
pytest.mark.parametrize( ("policy_location", "expected_required"), [("foreign_payload", True), ("targe
tests/reporting/test_advisory_skip_verdict.py:None
Routetest_format_judges_receive_complete_final_response_separate_from_tool_and_reference
pytest.mark.parametrize( ("requirement", "final_response"), [ ("Return only a number.", "83"),
tests/tier3/test_judge_contracts.py:None
Routetest_frontmatter_merge_chain_fails_closed_before_embedding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_full_body_flag_passed_to_registry
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_generated_benchmark_report_duplicate_does_not_produce_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_generated_eval_output_is_still_classified_as_generated
pytest.mark.parametrize("output_dir", ["results", ".results"])
tests/validators/test_security_authored_evals.py:None
Routetest_generated_skill_card_duplicate_does_not_produce_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_generated_tasks_reject_workdir_inside_runtime_projection
pytest.mark.parametrize( "agent_workdir", [ "/workspace/input", "/workspace/input/subd
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_generic_atif_exec_call_with_input_is_unchanged
pytest.mark.parametrize( "source", ( pytest.param("list repository files", id="plain-input"),
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_github_overwrites_unverifiable_bot_marker_instead_of_deferring
pytest.mark.parametrize( "marker", [ "99999999999:1", # no such run "124:1", # anoth
tests/ci/test_feedback.py:None
Routetest_github_skips_fork_and_superseded_producer_runs_without_failing
pytest.mark.parametrize( "change", ["fork", "deleted-fork", "rerun-started", "rerun-completed", "rerun
tests/ci/test_feedback.py:None
Routetest_gitlab_aggregate_exports_the_paid_tier_decision
pytest.mark.parametrize( ("recomputed", "exported"), [("echo false", "false"), ("echo true", "true"), ("ex
tests/ci/test_gateway_benchmark_workflows.py:None
Routetest_gitlab_aggregate_recomputes_the_skill_change_skip
pytest.mark.parametrize(("recomputed", "expected"), [("echo false", "keyless"), ("echo true", "enriched")])
tests/ci/test_gateway_benchmark_workflows.py:None
Routetest_gitlab_paid_tier_runs_when_a_skill_changed_or_the_answer_is_unknown
pytest.mark.parametrize("recomputed", ["echo true; exit 0", "exit 2"])
tests/ci/test_gateway_benchmark_workflows.py:None
Routetest_gitlab_rejects_missing_malformed_or_cross_project_persisted_mr
pytest.mark.parametrize( "linked_mr", [ None, [], {}, {"iid": 7, "sour
tests/ci/test_feedback.py:None
Routetest_gitleaks_git_history_keeps_resolved_source
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scanner_path_resolution.py:None
Routetest_gitleaks_malformed_findings_report_is_incomplete
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scan_incomplete.py:None
Routetest_gitleaks_no_git_direct_file_uses_safe_relative_source
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scanner_path_resolution.py:None
Routetest_gitleaks_no_git_does_not_expose_allowlisted_ancestors
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scanner_path_resolution.py:None
Routetest_gitleaks_no_git_uses_dot_source_from_resolved_skill_dir
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scanner_path_resolution.py:None
Routetest_gitleaks_timeout_marks_incomplete
patch.object(Tools.gitleaks, "_path", "/usr/bin/gitleaks")
tests/validators/test_scan_incomplete.py:None
Routetest_gke_environment_exec_never_replays_dispatched_command
pytest.mark.parametrize( ("scenario", "statuses", "command_stderr", "expected_handshakes", "expected_runs"
tests/test_harbor_gke_environment.py:None
Routetest_gke_environment_seam_routes_untargeted_exec_to_main_and_probes_trusted_sidecar
pytest.mark.parametrize( ("probe_container_rc", "expect_isolation_failure"), [ # SC-1: Probe c
tests/test_harbor_gke_environment.py:None
Routetest_gke_environment_wait_for_pod_ready_fails_closed_on_init_probe_failure
pytest.mark.parametrize( ("init_exit_code", "init_waiting_reason", "pod_phase", "expected_error_snippet"),
tests/test_harbor_gke_environment.py:None
Routetest_gps_pretty_printed_geojson_geometry_positions_flagged
pytest.mark.parametrize( ("contents", "expected_lines"), [ ( '{\n
tests/validators/test_security.py:None
Routetest_gym_cli_probe_accepts_source_shaped_argparse_help_and_requires_all_flags
pytest.mark.parametrize( ("positional", "omitted_flag", "missing"), [ ("TARGET", None, None),
tests/test_nemo_gym_backend.py:None
Routetest_handled_input_failures_write_only_to_stderr
pytest.mark.parametrize( ("name", "args", "expected_error"), [ ("call_api", ("--url", "https:/
tests/test_reference_skill_scripts.py:None
Routetest_harbor_output_and_upload_controls_are_reserved
pytest.mark.parametrize( ("name", "value"), [ ("stream", False), ("enable_environment_
tests/test_harbor_environment_contract.py:None
Routetest_harbor_verifier_and_environment_failures_are_unscored_not_agent_failures
pytest.mark.parametrize( "exception_type", ["VerifierTimeoutError", "EnvironmentStartTimeoutError", "V
tests/test_harbor_collector_runtime_failures.py:None
Routetest_hook_url_query_component_shaped_like_a_secret_is_critical
pytest.mark.parametrize( "hooks", [ _http_hook(f"https://hooks.example.com/notify?{_BARE_QUERY
tests/validators/test_plugin_component_risk.py:None
Routetest_hooks_beside_a_missing_reference_in_a_folder_job_stay_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_hooks_beside_a_missing_reference_in_a_folder_scan_stay_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_hooks_beside_a_missing_reference_keep_the_reference_warning
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_hooks_beside_a_referenced_image_keep_the_image_note
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_hooks_do_not_stop_a_folder_scan
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_hooks_fail_with_a_critical_instead_of_an_incomplete_scan
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_html_anchor_links_are_extracted
pytest.mark.parametrize( ("content", "target"), [ ('<a href="README.md">docs</a>', "README.md"
tests/validators/test_markdown.py:None
Routetest_html_and_ci_use_the_same_not_applicable_evidence
pytest.mark.parametrize( "check,message,metadata,expected", [ ("code_file_discovery", "No code
tests/reporting/test_retained_html.py:None
Routetest_ignores_non_reads_and_searches
pytest.mark.parametrize( "cmd", [ "grep SKILL config.json", # search tool, not a viewer (the
tests/test_checks_routing.py:None
Routetest_image_targets_are_opt_in
pytest.mark.parametrize( ("content", "target"), [ ("![diagram](inline.png)", "inline.png"),
tests/validators/test_markdown.py:None
Routetest_image_verification_is_bounded
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_import_only_modules_are_skipped_in_any_position
pytest.mark.parametrize( "helper", [ pytest.param(HELPER, id="functions-and-docstring"),
tests/tier3/test_dataset_primary_script.py:None
Routetest_in_pod_metadata_isolation_probe_script_resists_proxy_and_binary_shims
pytest.mark.parametrize( ("scenario", "http_status", "with_dead_proxy", "tool_mode", "expected_rc"), [
tests/test_harbor_gke_environment.py:None
Routetest_in_tree_baseline_is_excluded_from_the_staged_scan
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_in_tree_baseline_keeps_its_path_when_scanned_in_place
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_incomplete_llm_stage_does_not_repeat_the_critical
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_hook_surface.py:None
Routetest_incomplete_llm_stage_keeps_each_findings_warning
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_incomplete_required_guidance_stops_before_judge
pytest.mark.parametrize( "case", [ "missing", "escape", "e
tests/validators/test_rubric_eval.py:None
Routetest_inline_map_printed_output_proves_the_call_settled
pytest.mark.parametrize( ("body", "is_error"), [ pytest.param('{"exit_code":0,"output":"# Exam
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_install_words_in_file_text_are_not_commands
pytest.mark.parametrize( "call", [ _call("Edit", raw=_README_PATCH), _call("Write", fi
tests/tier3/test_tool_productivity.py:None
Routetest_intentional_detail_produces_no_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_invalid_catalog_is_not_reclassified_as_provider_failure
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_invalid_model_rejected
pytest.mark.parametrize("model", ["nemotron", "openrouter/nvidia/model", "/model", "publisher/"])
tests/test_hermes_agent.py:None
Routetest_invalid_skillspector_json_fails_closed
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_json_escaped_config_keys_keep_ordinary_text
pytest.mark.parametrize( "command", [ 'git config --global "credential.helper" store && rm -rf
tests/tier3/test_security_finding_evidence.py:None
Routetest_json_preflight_decodes_a_body_within_twelve_characters_per_allowed_character
pytest.mark.parametrize( ("body", "max_string_chars"), [ # Twelve input characters decode to o
tests/utils/test_structured_data.py:None
Routetest_judge_evidence_masks_a_value_escaped_again_next_to_a_marker_of_another_kind
pytest.mark.parametrize( ("output", "masked_output"), [ pytest.param(f"session {_JWT} saved",
tests/tier3/test_judge_contracts.py:None
Routetest_judge_evidence_masks_json_escaped_tool_call_headers
pytest.mark.parametrize( ("command", "secret"), [ pytest.param( """python -c 'requ
tests/tier3/test_judge_contracts.py:None
Routetest_keeps_advice_that_names_a_case_id_but_edits_the_implementation
pytest.mark.parametrize( "case_id,message", [ ("positive-001", "Remove the special
tests/evaluation/test_judges.py:None
Routetest_keeps_advice_that_names_an_identifier_that_is_not_a_known_case_id
pytest.mark.parametrize( "message", [ "Remove the special case for UTF-8 and use t
tests/evaluation/test_judges.py:None
Routetest_keeps_advice_that_removes_implementation_cases_and_case_failures
pytest.mark.parametrize( "message", [ "Remove the special case in parser.py for po
tests/evaluation/test_judges.py:None
Routetest_keeps_advice_whose_negator_governs_the_verb_through_modifiers
pytest.mark.parametrize( "case_id,message", [ ("smoke", "Implement the missing fie
tests/evaluation/test_judges.py:None
Routetest_keeps_genuine_prohibitions_next_to_additive_wording
pytest.mark.parametrize( "case_id,message", [ ("smoke", "You should not lower the
tests/evaluation/test_judges.py:None
Routetest_keeps_mismatch_fixes_negated_removals_and_ordinary_case_prose
pytest.mark.parametrize( "message", [ "Remove the expected output mismatch in posi
tests/evaluation/test_judges.py:None
Routetest_large_cluster_still_respects_member_and_prompt_limits
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_large_member_cluster_is_reviewed_in_full_when_prompt_fits
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_ledger_rows_must_match_an_in_scope_target
pytest.mark.parametrize("target", ["../outside.pl", "/outside.pl", "scripts/\x00helper.pl", "scripts/other.pl"
tests/validators/test_skillspector_reference_diagnostics.py:None
Routetest_line_naming_a_file_left_out_of_the_scan_stays_incomplete_with_a_row
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_linked_repo_cannot_overlay_agent_discovery_roots
pytest.mark.parametrize( ("linked_path", "linked_target"), [ ("skills/target-skill/SKILL.md",
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_llm_auth_failure_hidden_behind_clean_report_is_incomplete
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_llm_cluster_limit_is_incomplete_before_llm_analysis
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_llm_enrichment_requires_explicit_positive_execution_metadata
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
← previousnext →201–300 of 703, ranked by callers