MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_judge_evidence_keeps_the_quote_after_a_continuation_that_ends_a_string
()
tests/tier3/test_judge_contracts.py:1676
Functiontest_judge_evidence_masking_escaped_again_only_adds_markers
(monkeypatch: pytest.MonkeyPatch)
tests/tier3/test_judge_contracts.py:1746
Functiontest_judge_evidence_masking_escaped_again_stays_linear_on_adversarial_input
(unit: str)
tests/tier3/test_judge_contracts.py:1771
Functiontest_judge_evidence_masks_a_partly_masked_value_escaped_twice_as_one_value
(value: str)
tests/tier3/test_judge_contracts.py:1650
Functiontest_judge_evidence_masks_a_value_escaped_again_next_to_a_marker_of_another_kind
( output: str, masked_output: str, monkeypatch: pytest.MonkeyPatch )
tests/tier3/test_judge_contracts.py:1719
Functiontest_judge_evidence_masks_an_argv_value_with_an_escaped_quote_escaped_twice
()
tests/tier3/test_judge_contracts.py:1550
Functiontest_judge_evidence_masks_an_unclosed_quoted_value_to_the_end_of_its_string
(times: int)
tests/tier3/test_judge_contracts.py:1664
Functiontest_judge_evidence_masks_argv_lines_escaped_twice
()
tests/tier3/test_judge_contracts.py:1560
Functiontest_judge_evidence_masks_argv_lists_escaped_again
(argv: str, masked: str)
tests/tier3/test_judge_contracts.py:1541
Functiontest_judge_evidence_masks_consecutive_credential_lines_in_json_escaped_text
(text: str, second: str)
tests/tier3/test_judge_contracts.py:1350
Functiontest_judge_evidence_masks_credentials_after_json_escaped_line_breaks
( line: str, secret: str, kept: str, line_break: str )
tests/tier3/test_judge_contracts.py:1147
Functiontest_judge_evidence_masks_credentials_after_json_escaped_tabs
(text: str, secret: str)
tests/tier3/test_judge_contracts.py:1169
Functiontest_judge_evidence_masks_credentials_in_json_content_escaped_twice
()
tests/tier3/test_judge_contracts.py:1573
Functiontest_judge_evidence_masks_credentials_without_a_key_shape
(text: str, secret: str, kept: str)
tests/tier3/test_judge_contracts.py:1065
Functiontest_judge_evidence_masks_json_escaped_quoted_values
(text: str)
tests/tier3/test_judge_contracts.py:1312
Functiontest_judge_evidence_masks_json_escaped_tool_call_headers
(command: str, secret: str)
tests/tier3/test_judge_contracts.py:1104
Functiontest_judge_evidence_masks_multi_line_tool_call_arguments
()
tests/tier3/test_judge_contracts.py:1261
Functiontest_judge_evidence_masks_non_ascii_values_at_json_escaped_line_starts
(text: str)
tests/tier3/test_judge_contracts.py:1250
Functiontest_judge_evidence_masks_passphrase_after_bare_p_element
(argv: str)
tests/tier3/test_judge_contracts.py:1464
Functiontest_judge_evidence_masks_private_key_blocks_in_json_escaped_text
(label: str)
tests/tier3/test_judge_contracts.py:1198
Functiontest_judge_evidence_masks_quoted_values_escaped_again
(text: str, value: str, times: int)
tests/tier3/test_judge_contracts.py:1629
Functiontest_judge_evidence_masks_twice_escaped_attached_password
(context: str)
tests/tier3/test_judge_contracts.py:1487
Functiontest_judge_evidence_masks_values_that_hold_escapes
(text: str)
tests/tier3/test_judge_contracts.py:1228
Functiontest_judge_evidence_places_the_text_a_reading_keeps_before_its_first_marker
(text: str, expected: str)
tests/tier3/test_judge_contracts.py:1738
Functiontest_judge_evidence_reads_a_prompt_with_an_escaped_nul_again
()
tests/tier3/test_judge_contracts.py:1687
Functiontest_judge_evidence_write_failure_never_changes_the_score
(tmp_path: Path, monkeypatch)
tests/tier3/test_judge_contracts.py:1874
Functiontest_judge_exclusion_cannot_remove_security
()
tests/tier3/test_private_comparison_contracts.py:205
Functiontest_judge_failure_does_not_change_genuine_scored_failure_advice
(tmp_path: Path)
tests/test_tier3_case_summary.py:903
Functiontest_judge_failure_of_a_native_task_gets_judge_advice_under_the_case_the_collector_recorded
( tmp_path: Path, )
tests/test_tier3_case_summary.py:1122
Functiontest_judge_failures_are_unscored_infrastructure_advice
( tmp_path: Path, should_trigger: bool, status: str, record_failure: bool )
tests/test_tier3_case_summary.py:873
Functiontest_judge_filtered_lift_respects_activation_eligibility
(tmp_path: Path, activated: bool)
tests/test_judge_lift_integrity.py:158
Functiontest_judge_free_cases_retain_collector_lift
(tmp_path: Path, native: bool)
tests/test_judge_lift_integrity.py:262
Functiontest_judge_header_format_preservation_masks_escaped_secret_suffixes
(text: str)
tests/tier3/test_judge_contracts.py:1987
Functiontest_judge_mismatch_does_not_enter_paired_interval_or_ladder
()
tests/tier3/test_private_comparison_contracts.py:49
Functiontest_judge_model_config_distinguishes_provider_default_from_explicit_model
( monkeypatch: pytest.MonkeyPatch, configured_model: str | None, expected_source: str, )
tests/test_harbor_runner_environment.py:571
Functiontest_judge_model_config_records_the_judge_reasoning_effort
( provider_name: str, judge_model: str, effort: str | None )
tests/test_harbor_runner_environment.py:603
Functiontest_judge_model_overrides_are_forwarded_only_as_harbor_verifier_env
()
tests/test_tier3_public_runtime.py:585
Functiontest_judge_redaction_preserves_long_evidence_and_masks_credentials_past_finding_scan_limit
()
tests/tier3/test_security_finding_evidence.py:1897
Methodtest_judge_returns_the_second_reply_as_parsed
(self)
tests/validators/test_rubric_eval.py:1664
Functiontest_judge_schema_not_spoofed_by_transcript_containing_accuracy_tokens
Ensure judge_goal_accuracy and judge_behavior_check pass explicit schemas and are not spoofed by SKILL_IDENTIFIED/ACTION_CORRECT in the agent transcri
tests/test_llm_judge.py:909
Functiontest_judge_sidecar_scan_rejects_native_windows_verifier_junction
(tmp_path: Path)
tests/test_issue55_collector_truth.py:4205
Functiontest_judge_token_or_cost_fields_do_not_feed_agent_pricing
(tmp_path)
tests/tier3/test_price_provenance.py:96
Functiontest_judges_accept_non_string_fields_with_character_facts
(monkeypatch, implementation, judge_name, ground_truth)
tests/test_character_facts.py:106
Functiontest_judges_outside_the_verifier_run_save_no_evidence
(tmp_path: Path, monkeypatch)
tests/tier3/test_judge_contracts.py:1906
Functiontest_junit_keeps_plugin_evidence_out_of_the_xml
(with_runtime)
tests/reporting/test_plugin_reports_runtime_composition.py:89
Methodtest_junit_marks_incomplete_as_error_not_passed
(self)
tests/reporting/test_incomplete_scans.py:127
Functiontest_jwks_uses_only_fixed_host_without_credentials
(monkeypatch)
tests/ci/test_verify_provenance.py:173
Functiontest_jwt_like_env_and_arg_values_are_still_inline_secrets
(value: str)
tests/validators/test_mcp_static.py:840
Functiontest_kata_accepts_literal_dns_servers
(dns: object)
tests/test_harbor_environment_contract.py:186
Functiontest_kata_default_runtime_must_be_registered
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_environment_contract.py:243
Functiontest_kata_kwargs_keep_the_microvm_boundary
(environment_kwargs: dict[str, object], message: str)
tests/test_harbor_environment_contract.py:174
Functiontest_kata_prerequisites_require_the_selected_runtime
( monkeypatch: pytest.MonkeyPatch, registered: list[str], ready: bool, )
tests/test_harbor_environment_contract.py:210
Functiontest_kata_rejects_compose_sidecars_before_harbor_starts
(tmp_path: Path)
tests/test_harbor_environment_contract.py:262
Functiontest_kata_runs_are_described_as_microvm_attempts
()
tests/test_harbor_environment_contract.py:282
Functiontest_kata_runs_the_hardened_kata_backend
()
tests/test_harbor_environment_contract.py:135
Functiontest_kata_uses_the_docker_nvidia_build_handoff_and_host_environment
()
tests/test_harbor_environment_contract.py:289
Functiontest_kata_uses_the_shared_docker_base_image
(monkeypatch: pytest.MonkeyPatch, tmp_path: Path)
tests/test_tier3_config_key_parity.py:420
Functiontest_keep_flag_retains_harbor_artifacts
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_tier3_progress.py:4870
Methodtest_keeps_advice_that_clarifies_requirements
(self)
tests/evaluation/test_judges.py:988
Methodtest_keeps_advice_that_names_a_case_id_but_edits_the_implementation
(self, case_id: str, message: str)
tests/evaluation/test_judges.py:1118
Methodtest_keeps_advice_that_names_an_identifier_that_is_not_a_known_case_id
(self, message: str)
tests/evaluation/test_judges.py:1308
Methodtest_keeps_advice_that_removes_implementation_cases_and_case_failures
(self, message: str)
tests/evaluation/test_judges.py:1040
Methodtest_keeps_advice_whose_negation_word_governs_the_removal_verb
(self, case_id: str, message: str)
tests/evaluation/test_judges.py:1176
Methodtest_keeps_advice_whose_negator_governs_the_verb_through_modifiers
(self, case_id: str, message: str)
tests/evaluation/test_judges.py:1200
Methodtest_keeps_genuine_prohibitions_next_to_additive_wording
(self, case_id: str, message: str)
tests/evaluation/test_judges.py:1280
Methodtest_keeps_meta_folders
Standard meta-folders (``references``, ``scripts``, ``assets``) stay in scope. The exclusion is targeted at evaluation/version artifacts only
tests/deduplication/utils/test_skill_collector.py:785
Methodtest_keeps_mismatch_fixes_negated_removals_and_ordinary_case_prose
(self, message: str)
tests/evaluation/test_judges.py:1413
Methodtest_keeps_negated_and_non_requirement_uses_of_weakening_verbs
(self)
tests/evaluation/test_judges.py:1428
Methodtest_keeps_negated_instead_of_and_implementation_advice_for_the_wider_verbs
( self, case_id: str, message: str )
tests/evaluation/test_judges.py:1148
Methodtest_keeps_plural_benchmarks_report
``benchmarks.md`` is not a generated artifact.
tests/deduplication/utils/test_skill_collector.py:934
Methodtest_kept_match_repeated_across_files_counts_every_file
(self, tmp_path: Path)
tests/validators/test_skillspector_scope.py:396
Methodtest_kept_mixed_severity_rule_weights_the_most_severe_finding_first
SkillSpector weights a rule's findings by severity first, as P9 and TM1 emit mixed severities.
tests/validators/test_skillspector_scope.py:415
Functiontest_key_glued_to_word_char_is_still_redacted
()
tests/tier3/test_secret_redaction_patterns.py:49
Functiontest_key_override_inside_skill_is_rejected_before_mutation
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_output_provenance.py:621
Functiontest_keyboard_navigation_reveals_nested_agent_and_finding_evidence
(page, reports)
tests/reporting/test_report_browser.py:659
Functiontest_keyed_gym_generates_with_its_endpoint_and_one_compatibility_probe
Harbor credential or Docker checks must not block a compatible Gym worker.
tests/cli/test_nemo_gym_autopilot_preflight.py:21
Functiontest_keyed_incompatible_gym_does_not_generate_a_dataset
(tmp_path: Path, monkeypatch: pytest.MonkeyPatch)
tests/cli/test_nemo_gym_autopilot_preflight.py:100
Functiontest_keyless_tier1_reports_unrun_llm_scope
(skill, tmp_path)
tests/cli/test_tier_workflows.py:148
Functiontest_keyword_rule_evidence_skips_the_sourced_file
(check_security)
tests/tier3/test_security_home_paths.py:604
Functiontest_keyword_rule_with_many_sourced_files
()
tests/tier3/test_security_home_paths.py:610
Functiontest_known_hook_output_is_captured_with_inventory_instead_of_reread
(tmp_path)
tests/test_plugin_context_cost.py:165
Functiontest_known_legacy_policy_lift_uses_condition_scores_not_stale_lift_artifact
()
tests/reporting/test_unified_tier3_report.py:1279
Functiontest_known_legacy_policy_overall_uses_raw_metrics_when_stored_value_disagrees
()
tests/reporting/test_unified_tier3_report.py:1257
Functiontest_known_legacy_policy_uses_metrics_even_if_stored_overall_disagrees
()
tests/test_harbor_metrics_truth.py:313
Methodtest_known_manifest_identity_does_not_bypass_unsafe_path_rejection
(self, tmp_path, unsafe_alias)
tests/validators/test_rubric_eval.py:516
Functiontest_known_never_loaded_classification_accepts_legacy_tallies
(tally)
tests/tier3/test_private_evidence_review2.py:221
Functiontest_known_pattern_redactor_rejects_unknown_candidate_kind
()
tests/test_harbor_local_mode.py:259
Functiontest_known_replacement_cannot_create_a_visible_exact_secret
()
tests/test_harbor_local_mode.py:490
Methodtest_known_test_card_in_any_grouping_not_flagged
Known test card numbers are excepted whether written with spaces, dashes, or no separators.
tests/validators/test_security.py:7826
Functiontest_ksh_mksh_and_ash_are_shells
They run a script named as their operand and carry a ``-c`` payload into the walk like sh, with sh's options: ``-n`` only parses, ``-x`` traces
tests/tier3/test_script_invocation_evidence.py:1207
Functiontest_ksh_stops_on_unset_given_no_name
``unset`` given no name prints its usage and stops ksh, unless ``command`` runs it or a subshell holds it; the other shells go on. The target of a
tests/tier3/test_script_invocation_evidence.py:1981
Functiontest_labelled_logo_has_unique_prefixed_ids
()
tests/reporting/test_report_logo.py:29
Functiontest_ladder_uses_one_shared_bootstrap_draw_stream
(monkeypatch)
tests/tier3/test_private_comparison_contracts.py:32
Functiontest_large_adversarial_patch_is_scanned_in_linear_time
(run)
tests/tier3/test_apply_patch_security.py:1202
Functiontest_large_case_sets_publish_bounded_samples_without_losing_exact_pairing
()
tests/test_harbor_metrics_truth.py:829
Methodtest_large_cluster_still_respects_member_and_prompt_limits
( self, mock_embed, mock_clusters, mock_llm, tmp_path: Path, m
tests/deduplication/intra_skill/test_intra_skill_validator.py:739
Functiontest_large_coverage_has_explicit_omissions_with_bounded_output
()
tests/reporting/test_private_runtime_projection.py:202
Functiontest_large_declarative_run_bounds_coverage_before_scored_report_details
()
tests/reporting/test_declarative_coverage.py:442
Functiontest_large_dense_short_secret_has_the_same_completed_outcome_with_callback
(tmp_path: Path)
tests/test_harbor_local_mode.py:3181
Functiontest_large_exact_pairing_respects_active_integer_string_limit_in_subprocess
()
tests/test_harbor_metrics_truth.py:1245
← previousnext →17,701–17,800 of 24,741, ranked by callers