MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / endpoints

Endpoints703 in github.com/NVIDIA/SkillEvaluator

Routetest_caveats_keep_the_findings
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_caveats_report_completes_the_llm_stage
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_caveats_report_completes_the_scan
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_caveats_status_before_2_13_4_is_not_recognized
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_caveats_that_contradict_the_status_drop_the_report
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_changes_within_skill_select_only_that_skill
pytest.mark.parametrize("operation", ["modify", "delete-file", "rename-file"])
tests/ci/test_select_changed_skills.py:None
Routetest_character_facts_keep_numeric_dataset_answers_scoreable
pytest.mark.parametrize( ("filename", "contents"), [ ("evals.json", '[{"question": "What is 6*
tests/tier3/test_judge_failure_artifacts.py:None
Routetest_check_failures
pytest.mark.parametrize( ("expected", "scored", "slots", "failures", "code"), [ (2, 1, _slots(
tests/tier3/test_baseline_timeouts.py:None
Routetest_check_pod_terminated_fails_fast_when_pod_deleted_or_preempted
pytest.mark.parametrize( ("read_effect", "should_raise", "expected_match"), [ # 1. Running pod
tests/test_harbor_gke_environment.py:None
Routetest_chunk_limit_is_incomplete_before_embedding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_ci_branch_overrides_detached_head
patch("skillevaluator.utils.helpers.subprocess.check_output")
tests/utils/test_helpers.py:None
Routetest_claude_code_error_flag_still_fails
pytest.mark.parametrize( "command,output,exit_code", [ ("pytest -q", "F.\nFAILED tests/test_a.
tests/tier3/test_error_recovery.py:None
Routetest_claude_code_refusal_fallback_is_off_for_every_provider_route
pytest.mark.parametrize("route", sorted(_ROUTES))
tests/test_harbor_claude_refusal_fallback.py:None
Routetest_claude_code_success_flag_skips_text_check
pytest.mark.parametrize( "command,output", [ ( 'python3 -m py_compile repository.p
tests/tier3/test_error_recovery.py:None
Routetest_claude_markdown_declarations_supplement_packaged_defaults
pytest.mark.parametrize( ("field_name", "component_type", "default_dir"), [ ("agents", "agent"
tests/test_plugin_discovery.py:None
Routetest_clean_scan_is_not_marked
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_scan_incomplete.py:None
Routetest_clean_skill_scanned_in_place
patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector")
tests/validators/test_artifact_dir_exclusion.py:None
Routetest_codex_apply_patch_input_reaches_file_changes
pytest.mark.parametrize( ("patch", "expected_paths"), [ pytest.param(_SINGLE_FILE_PATCH, ["src
tests/tier3/test_behavior_evidence.py:None
Routetest_codex_optional_headers_preserve_output
pytest.mark.parametrize("output", ["", "Permission denied", "status=failed", "Original token count: 0"])
tests/tier3/test_error_recovery.py:None
Routetest_codex_probe_header_does_not_hide_real_failures
pytest.mark.parametrize( "command,exit_code,output", [ ("rg TODO src; python run.py", 1, ""),
tests/tier3/test_error_recovery.py:None
Routetest_codex_probe_output_is_never_stripped_as_metadata
pytest.mark.parametrize( "output", [ "Permission denied", "status=failed", "ex
tests/tier3/test_error_recovery.py:None
Routetest_codex_write_stdin_without_exec_header_is_rejected
pytest.mark.parametrize( "content", [ "write_stdin failed: stdin is closed for this session",
tests/tier3/test_trajectory_timing.py:None
Routetest_commands_without_credentials_keep_their_whole_evidence
pytest.mark.parametrize( ("command", "finding_type"), [ pytest.param( 'grep -rh "t
tests/tier3/test_security_finding_evidence.py:None
Routetest_commonmark_dead_links_are_detected
pytest.mark.parametrize( ("content", "target"), [ ("See [the guide][guide].\n\n[gu
tests/validators/test_hygiene.py:None
Routetest_commonmark_links_are_extracted
pytest.mark.parametrize( ("content", "target"), [ ("[guide](inline.md)", "inline.md"),
tests/validators/test_markdown.py:None
Routetest_compact_behavior_conversation_section_layout_permutations
pytest.mark.parametrize( "case_name,limit,final_limit_env,text_builder,expected_substrings", [
tests/tier3/test_behavior_evidence.py:None
Routetest_compacted_report_cannot_hide_a_control_path_match
patch("skillevaluator.validators.security.Tools")
tests/validators/test_skillspector_scope.py:None
Routetest_comparison_budget_exhaustion_keeps_the_finding
pytest.mark.parametrize( ("budget_name", "guidance"), [ # The pair limit counts ev
tests/validators/test_skillspector_scope.py:None
Routetest_comparison_syntax_is_not_reported_as_xml
pytest.mark.parametrize( "description", [ "Use when mapping kernel -> module relat
tests/validators/test_quality_score.py:None
Routetest_compose_stream_redaction_marker_cannot_disclose_or_synthesize_secrets
pytest.mark.parametrize( ("secrets", "raw_output"), [ (("REDACTED", "[REDACTED]"), "REDACTED [
tests/test_harbor_secure_docker_environment.py:None
Routetest_computed_allow_decision_is_auto_approve
pytest.mark.parametrize( ("name", "interpreter", "body"), [ ( "variable.py",
tests/validators/test_plugin_component_risk.py:None
Routetest_contained_directory_alias_target_rejects_root_escape_and_cycle_claims
pytest.mark.parametrize( ("relative", "target"), [ pytest.param(".agents", ".", id="root"),
tests/utils/test_tier2_paths.py:None
Routetest_contained_directory_alias_target_returns_lexical_in_root_claim
pytest.mark.parametrize( ("relative", "target", "expected"), [ pytest.param(".agents", ".claud
tests/utils/test_tier2_paths.py:None
Routetest_copies_and_same_name_skills_outside_the_catalog_tree_are_still_reported
pytest.mark.parametrize( "make_target", [ _copy_alpha_to("candidates/alpha"),
tests/embedding/test_catalog.py:None
Routetest_coverage_uses_admitted_slots
pytest.mark.parametrize("reason", ["budget_exhausted", None, "execution_failed"])
tests/tier3/test_budget_admitted_ordinals.py:None
Routetest_credit_card_next_to_technical_or_test_value_still_flagged
pytest.mark.parametrize( ("contents", "card"), [ ("timestamp_microseconds: 1707938
tests/validators/test_security.py:None
Routetest_critical_duplicate_fails
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_current_default_branch_writer_is_eligible
pytest.mark.parametrize("event_name", ["push", "workflow_dispatch"])
tests/ci/test_github_admission_workflow.py:None
Routetest_custom_compose_rejects_short_and_long_host_bind_mounts
pytest.mark.parametrize( "volume", ( "/:/host-root:ro", "./docker.sock:/var/run/docker
tests/test_harbor_custom_compose_safety.py:None
Routetest_custom_dockerfile_rejects_noncanonical_input_case
pytest.mark.parametrize("source", ["INPUT/", "Input/fixture.txt"])
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_deceptive_openai_urls_keep_max_tokens
pytest.mark.parametrize( ("provider", "request_url"), [ ("nv_build", "https://api.openai.com/v
tests/test_llm_judge.py:None
Routetest_declared_component_references_never_show_their_credentials
pytest.mark.parametrize( ("manifest_rel", "fields"), [ ( ".claude-plugin/plugin.js
tests/test_plugin_discovery.py:None
Routetest_deeply_nested_skillspector_json_fails_closed
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_default_branch_execution_requires_current_commit
pytest.mark.parametrize("event_name", ["push", "workflow_dispatch"])
tests/ci/test_github_admission_workflow.py:None
Routetest_default_v2_rerender_respects_recorded_or_historical_policy
pytest.mark.parametrize( ("stored_overall", "recorded_policy", "expected_policy", "expected_score"), [
tests/reporting/test_unified_tier3_report.py:None
Routetest_deny_or_ask_only_hook_is_not_auto_approve
pytest.mark.parametrize( "body", [ "import json\nprint(json.dumps({'hookSpecificOutput': {'per
tests/validators/test_plugin_component_risk.py:None
Routetest_deploy_is_skipped_only_when_an_automatic_first_attempt_committed_nothing
pytest.mark.parametrize( ("committed", "event", "attempt", "deploys"), [ ("true", "workflow_ru
tests/ci/test_publisher_workflow.py:None
Routetest_deprecated_cache_alias_loads_catalog
patch("skillevaluator.validators.similarity.extract_from_skill")
tests/validators/test_similarity.py:None
Routetest_deprecated_cache_alias_missing_file_fails_closed
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_deprecated_save_cache_alias_saves_catalog
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_destructive_removals_still_score
pytest.mark.parametrize( "command", [ # Outside the agent's output directory. "rm -rf
tests/tier3/test_issue192_security_false_positives.py:None
Routetest_deterministic_skillspector_stage_rejects_llm_metadata
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_direct_rule_scan_requires_two_items
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_direct_tier_json_names_the_clean_skill_and_records_the_target_as_given
pytest.mark.parametrize( ("directory", "target"), [("parent", "sample"), ("parent", "sample/"), ("skil
tests/cli/test_tier_workflows.py:None
Routetest_directories_left_out_that_the_line_does_not_name_are_ignored
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_directory_form_markdown_links_are_not_nested_references
pytest.mark.parametrize( "target", [ "advanced.md/", "advanced.md/.",
tests/validators/test_quality_score.py:None
Routetest_directory_form_readme_links_do_not_trigger_reference_finding
pytest.mark.parametrize( "target", [ "README.md/", "README.md/.",
tests/validators/test_quality_score.py:None
Routetest_distinct_cluster_errors_are_grouped_in_deterministic_order
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_dollar_n_is_a_price_or_an_argument_not_both
pytest.mark.parametrize( ("line", "prices", "placeholders"), [ ("Budget is $5.", 1, 0),
tests/harness/test_overlay.py:None
Routetest_drops_a_removal_that_a_negator_with_modifiers_does_not_govern
pytest.mark.parametrize( "case_id,message", [ ("smoke", "Run without failures by d
tests/evaluation/test_judges.py:None
Routetest_drops_a_removal_when_the_negation_word_governs_something_else
pytest.mark.parametrize( "case_id,message", [ ("positive-001", "Run without failur
tests/evaluation/test_judges.py:None
Routetest_drops_additive_wording_that_still_recommends_the_removal
pytest.mark.parametrize( "case_id,message", [ ("smoke", "Make smoke pass by not on
tests/evaluation/test_judges.py:None
Routetest_drops_advice_that_deletes_a_dataset_case
pytest.mark.parametrize( "message", [ "Remove the expected output mismatch by dele
tests/evaluation/test_judges.py:None
Routetest_drops_advice_that_deletes_a_dataset_case_named_by_its_id
pytest.mark.parametrize( "case_id,message", [ ("positive-001", "Delete the case fo
tests/evaluation/test_judges.py:None
Routetest_drops_advice_that_weakens_requirements
pytest.mark.parametrize( "message", [ "Loosen the expected output of positive-001
tests/evaluation/test_judges.py:None
Routetest_drops_an_unlisted_identifier_when_the_sentence_points_at_the_dataset
pytest.mark.parametrize( "message", [ "Delete the utf-16 case from evals/evals.jso
tests/evaluation/test_judges.py:None
Routetest_duplicate_finding_names_the_model_that_answered
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_duplicate_price_keys_are_rejected
pytest.mark.parametrize( ("contents", "duplicate_key"), [ ( '{"model":{"input_per_
tests/tier3/test_price_file_validation.py:None
Routetest_duplicate_shell_comments_at_eof_are_compared
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_duplicate_verdict_produces_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_duplicate_without_confidence_is_medium_advisory_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_dynamic_computed_codex_tool_member_fails_closed
pytest.mark.parametrize( "tool_reference", [ pytest.param("tools[name]", id="computed-member")
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_dynamic_map_forms_it_cannot_prove_stay_unsupported
pytest.mark.parametrize( "source", [ # Ambiguous ownership: each callback prints when its own
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_dynamic_map_incomplete_observation_stays_unsupported
pytest.mark.parametrize( "observation", [ pytest.param("", id="empty"), pytest.param("
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_dynamic_map_record_carries_its_own_outcome
pytest.mark.parametrize( ("record", "is_error"), [ pytest.param({"i": 0, "status": "fulfilled"
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_embedding_error_is_incomplete_without_skill_finding
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_empty_directory_fails_actionably
patch("skillevaluator.validators.similarity.EmbeddingRegistry")
tests/validators/test_similarity.py:None
Routetest_empty_skillspector_baseline_is_an_empty_baseline
patch("skillevaluator.validators.security.Tools")
tests/validators/test_security.py:None
Routetest_encoded_local_markdown_paths_remain_nested_references
pytest.mark.parametrize( "target", [ "advanced%2Emd", "docs/%2E%2E/adv
tests/validators/test_quality_score.py:None
Routetest_endpoint_url_controls_gpt5_token_key
pytest.mark.parametrize( ("base_url", "expected_key"), [ ( OPENAI_
tests/inference/test_llm_client.py:None
Routetest_eof_comments_keep_existing_filters
pytest.mark.parametrize( "content", [ "", "#!/bin/sh\n", "
tests/deduplication/utils/test_chunker.py:None
Routetest_error_recovery_is_not_clean_when_codex_observation_evidence_is_untrusted
pytest.mark.parametrize( ("source", "unsupported_evidence"), [ ( 'const failed = a
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_evaluator_only_assets_cannot_be_declared_as_task_input
pytest.mark.parametrize( "relative_path,is_directory", [ ("evals.json", False),
tests/test_harbor_input_staging.py:None
Routetest_every_judge_graded_string_leaf_is_checked_after_loading
pytest.mark.parametrize( "fields", [ {"assertions": [{"text": "The agent read SKILL.md"}]},
tests/tier3/test_dataset_baseline_fairness.py:None
Routetest_every_spelling_of_a_declared_server_read_credits_the_member
pytest.mark.parametrize( "fn", ["mcp__fs__get_document", "fs__get_document", "fs.get_document", "mcp_f
tests/tier3/test_plugin_signals.py:None
Routetest_exact_pairwise_budget_allows_cross_batch_duplicate
pytest.mark.parametrize("budget,completes", [(PAIRWISE_WORK - 1, False), (PAIRWISE_WORK, True)])
tests/embedding/test_pairwise_preflight.py:None
Routetest_existing_mutable_branch_or_dynamic_wrapper_is_not_silently_approved
pytest.mark.parametrize("ref", ["main", "refs/heads/v3", "${{ inputs.version }}", "v3/unsafe"])
tests/test_ci_onboarding_existing.py:None
Routetest_existing_nonmanifest_output_survives_real_writer
pytest.mark.parametrize( ("destination", "contents"), [ ("skills/target/evals/oracle.txt", "pr
tests/tier3/test_decoy_commands.py:None
Routetest_existing_write_argument_shapes_are_unchanged
pytest.mark.parametrize( ("args", "expected_path", "expected_body"), [ pytest.param({"file_pat
tests/tier3/test_behavior_evidence.py:None
Routetest_exit_without_output_is_retained
pytest.mark.parametrize("output", [None, ""])
tests/tier3/test_opencode_exit_status.py:None
Routetest_explicit_budget_preserves_tool_history_with_long_final_response
pytest.mark.parametrize("budget_env", ["8000", "6000"])
tests/tier3/test_behavior_evidence.py:None
Routetest_explicit_budget_reports_its_own_effective_chunk_limit
patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
tests/deduplication/intra_skill/test_intra_skill_validator.py:None
Routetest_explicit_delegation_publishes_new_pinned_leaf_from_private_wrapper
pytest.mark.parametrize("event", ["push", "pull_request", "workflow_dispatch"])
tests/ci/test_publish_reports.py:None
Routetest_explicit_empty_input_keeps_authored_copy_input_buildable
pytest.mark.parametrize( "copy_instruction", [ "COPY input/ /workspace/input/", "COPY
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_explicit_empty_rejects_specific_custom_docker_input_child
pytest.mark.parametrize( "copy_instruction", [ "COPY input/missing.txt /workspace/input/missin
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_explicit_empty_rejects_specific_source_from_named_input_context
pytest.mark.parametrize( "instruction", [ "COPY --from=taskinput missing.txt /payload/",
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_explicit_empty_resolves_static_dynamic_input_directory
pytest.mark.parametrize("declaration", ["ARG INPUT_DIR=input", "ENV INPUT_DIR=input"])
tests/test_harbor_runtime_skill_isolation.py:None
Routetest_extended_codex_apply_patch_string_literal_normalizes
pytest.mark.parametrize( "source", [ pytest.param( r"const p=await tools.apply_pat
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_extended_codex_bound_forms_it_cannot_split_keep_the_upstream_result
pytest.mark.parametrize( "source", [ 'const a=await tools.exec_command({cmd:"pwd"});const b=aw
tests/tier3/test_codex_tool_call_normalization.py:None
Routetest_extended_codex_dynamic_and_ambiguous_forms_stay_atomic
pytest.mark.parametrize( "source", [ pytest.param('const cmd="pwd";text(await tools.exec_comma
tests/tier3/test_codex_tool_call_normalization.py:None
← previousnext →101–200 of 703, ranked by callers