MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Methodtest_report_metric_is_the_packaged_limit_without_a_custom_file
(self, tmp_path: Path)
tests/validators/test_quality_score.py:1746
Methodtest_report_names_the_model_that_answered_the_retry
(self, tmp_path, monkeypatch: pytest.MonkeyPatch)
tests/validators/test_rubric_eval.py:1699
Functiontest_report_notices_are_plain_text
(tmp_path: Path)
tests/reporting/test_dashboard_app.py:315
Functiontest_report_path_announcement_renders_literally
( payload: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch )
tests/reporting/test_rich_markup_escaping.py:409
Methodtest_report_path_is_lexical_and_contained
(self, value: object, expected: str | None)
tests/validators/test_skillspector_reference_diagnostics.py:532
Functiontest_report_paths_cannot_escape_retained_root
(tmp_path, unsafe)
tests/ci/test_benchmark_reports.py:189
Functiontest_report_payload_preserves_paths_and_shortens_input_hashes
()
tests/reporting/test_dataset_generation_provenance.py:129
Functiontest_report_policy_text_matches_current_or_historical_declared_flags
(tmp_path, required_paid)
tests/ci/test_tier_manifest.py:107
Functiontest_report_presents_baseline_calibration_without_a_skill_verdict
(tmp_path: Path, skill: Path)
tests/tier3/test_skip_with_skill.py:493
Functiontest_report_preserves_collector_declared_hidden_execution_error_count
(tmp_path: Path)
tests/reporting/test_report_data_bounds.py:552
Functiontest_report_provenance_and_attempt_scope_must_agree
(mutation)
tests/tier3/test_grader_policy.py:340
Functiontest_report_reasons_still_omit_criteria_excluded_by_older_rewards
()
tests/tier3/test_judge_result_semantics.py:180
Functiontest_report_redacts_configured_secret_before_bounding_reason
Redact configured secrets before truncating long judge reason strings in report.
tests/tier3/test_judge_failure_artifacts.py:1203
Functiontest_report_renders_aggregate_and_trial_failure_details
(tmp_path: Path)
tests/test_harbor_html_report_failures.py:80
Functiontest_report_replaces_stale_formula_when_recorded_policy_is_unknown
()
tests/reporting/test_unified_tier3_report.py:1397
Functiontest_report_retains_judge_filtered_regression
(tmp_path: Path, native: bool)
tests/test_judge_lift_integrity.py:176
Functiontest_report_says_nothing_to_fix_when_every_case_passes_and_meets_its_goal
(tmp_path)
tests/reporting/test_case_findings_report.py:65
Methodtest_report_serializes_every_finding_counts_occurrence_records
(self, report: dict, serialized: bool)
tests/validators/test_skillspector_scope.py:706
Functiontest_report_style_blocks_match_recorded_digests
Snapshot of this template's style blocks: a CSS edit must update the digests on purpose.
tests/reporting/test_report_redesign.py:498
Functiontest_report_surfaces_share_failure_precedence
()
tests/reporting/test_shared_run_status.py:62
Functiontest_report_tables_mark_the_with_skill_arm_skipped
(tmp_path: Path, skill: Path)
tests/tier3/test_skip_with_skill.py:569
Functiontest_report_text_and_redact_secrets_drop_userinfo_after_any_run_of_slashes
Regression: WHATWG reads 'http:/user:pw@host' as userinfo, but only '//' (and, for redact_secrets, no slash) introduced it here, so the password s
tests/validators/test_url_policy.py:369
Functiontest_report_text_marks_a_text_cut_by_the_redaction_window_as_truncated
()
tests/validators/test_url_policy.py:273
Functiontest_report_text_never_shows_a_token_cut_at_the_redaction_window
Regression: a token cut by the redaction window was too short to match its pattern, and the redacted JWT before it shrank the text enough to bring
tests/validators/test_url_policy.py:257
Functiontest_report_text_redacts_a_whole_jwt
Regression: the JWT's header and payload were redacted, but its signature was shown.
tests/validators/test_url_policy.py:246
Functiontest_report_text_redacts_a_whole_private_key
Regression: only the BEGIN line was redacted, so the key body and its END line were shown.
tests/validators/test_url_policy.py:230
Functiontest_report_text_redacts_every_known_secret_shape
()
tests/validators/test_url_policy.py:215
Functiontest_report_text_withholds_everything_after_a_private_key_it_cannot_read_as_a_block
()
tests/validators/test_url_policy.py:239
Functiontest_report_trials_and_usage_only_attempts_name_only_token_counters_in_one_order
(tmp_path: Path)
tests/reporting/test_tier3_token_usage_reports.py:338
Methodtest_report_truncation_counts_occurrence_records
(self, report: dict, truncated: bool)
tests/validators/test_skillspector_scope.py:680
Functiontest_report_url_rejects_external_or_credential_bearing_sites
(repository, release_file, url)
tests/test_ci_onboarding.py:253
Functiontest_report_uses_persisted_mixed_contract_flag_when_reward_sample_is_truncated
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, )
tests/reporting/test_report_data_bounds.py:485
Functiontest_reporter_pushes_case_rows_once_run_dir_is_known
(tmp_path)
tests/reporting/test_tier3_validate_cases.py:217
Functiontest_reporter_sanitizes_absolute_paths_inside_finding_messages
( tmp_path: Path, private_path: str, )
tests/test_public_benchmark_gate.py:1554
Functiontest_reporter_sanitizes_finding_before_publication_gate
(tmp_path: Path)
tests/test_public_benchmark_gate.py:1482
Functiontest_reporter_sanitizes_requested_environment_from_advisory_payload
(tmp_path: Path)
tests/test_public_benchmark_gate.py:1504
Functiontest_reporter_without_generation_timestamp_passes_publication_gate
(tmp_path: Path)
tests/test_public_benchmark_gate.py:1471
Functiontest_reporters_bound_and_flatten_untrusted_publication_conflict_metadata
( reporter_name: str, )
tests/reporting/test_advisory_skip_verdict.py:4020
Functiontest_reporters_bound_wide_agents_before_selection_and_assessment
( monkeypatch: pytest.MonkeyPatch, *, explicit_failure: bool, expected_tier3_status: str, )
tests/reporting/test_advisory_skip_verdict.py:1752
Functiontest_reporters_normalize_bounded_plain_text_publication_conflict_marker
( reporter_name: str, )
tests/reporting/test_advisory_skip_verdict.py:4044
Functiontest_reporters_omit_malformed_or_unbounded_publication_target
( reporter_name: str, malformation: str, )
tests/reporting/test_advisory_skip_verdict.py:3945
Functiontest_reporters_project_nested_tier3_publication_identity_metadata
( reporter_name: str, )
tests/reporting/test_advisory_skip_verdict.py:4060
Functiontest_reporters_project_publication_target_to_validated_canonical_fields
( reporter_name: str, )
tests/reporting/test_advisory_skip_verdict.py:3921
Functiontest_reporters_redact_exact_values_and_secret_shaped_details
()
tests/test_tier3_progress.py:2597
Functiontest_reporters_share_one_environment_and_compile_each_template_once
(payload, monkeypatch)
tests/reporting/test_retained_html.py:235
Functiontest_reports_are_written_for_a_file_name_that_is_not_utf8
(tmp_path: Path, report_format: str)
tests/reporting/test_reporters.py:3156
Methodtest_reports_drop_unsafe_provenance_but_keep_provider
(self)
tests/reporting/test_reporters.py:2970
Functiontest_reports_name_tolerated_agents_without_a_completed_agent_result
(tmp_path: Path, codex_short: bool)
tests/tier3/test_baseline_timeouts.py:1420
Functiontest_reports_never_fall_back_to_a_tolerated_agent
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:831
Functiontest_reports_rank_tolerated_agents_below_their_complete_peer
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:790
Functiontest_reports_reload_the_tolerated_agent_from_disk
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:319
Functiontest_reports_rendered_again_from_the_json_drop_malformed_counts
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, summary: object )
tests/reporting/test_skillspector_reference_summary.py:177
Functiontest_reports_rendered_again_from_the_json_show_the_counts
CI renders its HTML and Markdown reports again from the JSON report; a failed result keeps the line.
tests/reporting/test_skillspector_reference_summary.py:149
Functiontest_reports_rendered_from_redacted_json_accept_an_undecodable_file_name
(tmp_path)
tests/ci/test_secret_redaction.py:933
Functiontest_reports_retain_both_bundled_library_licenses
()
tests/reporting/test_chart_dependency.py:95
Functiontest_reports_show_only_manifest_checks_as_not_applicable
(tmp_path: Path)
tests/test_artifact_applicability.py:405
Methodtest_reports_surface_suppressed_counts
(self, tmp_path: Path)
tests/validators/test_skillspector_scope.py:1773
Methodtest_reports_without_ae1_evidence_have_no_summary
(self)
tests/validators/test_skillspector_reference_diagnostics.py:800
Methodtest_reports_without_provenance_add_nothing
(self)
tests/reporting/test_reporters.py:2985
Functiontest_repository_allowlist_anchor_must_follow_code_on_the_same_line
(tmp_path: Path)
tests/test_oss_boundary.py:303
Functiontest_repository_allowlist_anchor_must_match_exactly_one_configured_rule
(tmp_path: Path)
tests/test_oss_boundary.py:216
Functiontest_repository_allowlist_anchor_must_share_the_finding_line
(tmp_path: Path)
tests/test_oss_boundary.py:287
Functiontest_repository_allowlist_counts_separately_reconstructed_matches_on_one_line
(tmp_path: Path)
tests/test_oss_boundary.py:253
Functiontest_repository_allowlist_rejects_duplicate_comment_anchors
(tmp_path: Path)
tests/test_oss_boundary.py:199
Functiontest_repository_allowlist_rejects_other_rule_matches_on_the_anchor_line
(tmp_path: Path)
tests/test_oss_boundary.py:270
Functiontest_repository_allowlist_rejects_two_same_rule_matches_on_the_anchor_line
(tmp_path: Path)
tests/test_oss_boundary.py:237
Functiontest_repository_allowlist_requires_a_real_python_comment_anchor
(tmp_path: Path)
tests/test_oss_boundary.py:183
Functiontest_repository_callers_pin_a_released_evaluator_revision
The pinned runtime must stay resolvable after the pull request that set it merges. GitHub resolves ``uses: owner/repo/path@<sha>`` only for a com
tests/ci/test_repository_skill_ci.py:108
Functiontest_repository_diff_commands_and_fsmonitor_are_not_executed
(repository, tmp_path, monkeypatch)
tests/ci/test_select_changed_skills.py:202
Functiontest_repository_feedback_and_benchmarks_approve_exact_caller_and_preserve_public_publisher
()
tests/ci/test_repository_skill_ci.py:71
Functiontest_repository_identity_runs_two_git_commands
The top-level and the origin remote; the branch and HEAD a browse link needs are never asked for.
tests/validators/test_plugin_dependencies.py:107
Functiontest_repository_privacy_must_be_explicit_boolean_false
(value)
tests/test_ci_onboarding_pages.py:35
Functiontest_repository_remote_suffix_matches_reference_identity
(tmp_path: Path, suffix: str)
tests/validators/test_plugin_dependencies.py:94
Functiontest_repository_root_skill_lists_each_fix_first_location_once
Code Risk Analysis records absolute scan paths; other scanners record skill-relative ones.
tests/ci/test_native_reports.py:339
Functiontest_repository_scan_covers_all_tracked_github_configuration
(tmp_path: Path)
tests/test_oss_boundary.py:613
Functiontest_repository_scan_covers_every_tracked_non_binary_file
(tmp_path: Path)
tests/test_oss_boundary.py:640
Functiontest_repository_scan_covers_tracked_root_release_and_config_files
(tmp_path: Path, root_name: str)
tests/test_oss_boundary.py:627
Functiontest_repository_scan_fails_closed_for_a_missing_root
(tmp_path: Path)
tests/test_oss_boundary.py:1296
Functiontest_repository_scan_fails_closed_for_undecodable_tracked_unknown_file
(tmp_path: Path)
tests/test_oss_boundary.py:667
Functiontest_repository_scan_fails_closed_when_anchor_tokenization_fails
(tmp_path: Path)
tests/test_oss_boundary.py:330
Functiontest_repository_scan_fails_closed_when_anchored_python_cannot_be_parsed
(tmp_path: Path)
tests/test_oss_boundary.py:347
Functiontest_repository_scan_rejects_a_malformed_anchor_comment
(tmp_path: Path)
tests/test_oss_boundary.py:315
Functiontest_repository_scan_rejects_duplicate_anchor_ids_across_files
(tmp_path: Path)
tests/test_oss_boundary.py:365
Functiontest_repository_scan_uses_tracked_scopes_and_stable_allowlist_anchor
(tmp_path: Path)
tests/test_oss_boundary.py:114
Functiontest_repository_skill_checks_use_reviewed_full_three_tier_configuration
()
tests/ci/test_repository_skill_ci.py:20
Functiontest_repository_skill_has_committed_complete_dataset_prerequisites
()
tests/ci/test_repository_skill_ci.py:98
Methodtest_request_budget_includes_the_system_prompt
(self, make_chunk, monkeypatch)
tests/deduplication/intra_skill/test_intra_llm_analyzer.py:123
Functiontest_requested_environment_is_not_completed_run_provenance
( tmp_path: Path, requested_environment_location: str, )
tests/test_public_benchmark_gate.py:1523
Functiontest_requested_llm_analysis_is_still_required
()
tests/validators/test_security_reference_caveats.py:171
Functiontest_requested_second_attempt_gets_only_its_own_artifact
(tmp_path)
tests/ci/test_publish_reports.py:638
Methodtest_requested_skillspector_baseline_accepts_audited_suppressions
( self, mock_tools, sample_skill_dir: Path, tmp_path: Path )
tests/validators/test_security.py:1918
Methodtest_requested_skillspector_baseline_requires_suppression_accounting
( self, mock_tools, sample_skill_dir: Path, tmp_path: Path )
tests/validators/test_security.py:2044
Functiontest_requests_the_stub_misses_never_reach_the_network
( no_network: list[Any], monkeypatch: pytest.MonkeyPatch, variable: str | None, proxy: str | None )
tests/tier3/test_ragas_judge_evidence.py:820
Methodtest_requests_the_verdict_schema_and_reads_rationale
(self, make_cluster)
tests/deduplication/intra_skill/test_intra_llm_analyzer.py:468
Functiontest_require_bounded_string_enforces_length_and_nonempty
()
tests/utils/test_structured_data.py:289
Functiontest_require_bounded_string_never_stringifies_containers
(value: object)
tests/utils/test_structured_data.py:284
Functiontest_require_files_fails_empty_tree
(tmp_path: Path, capsys)
tests/test_public_benchmark_gate.py:3357
Methodtest_require_mode_remediation_offers_prefer_and_off
(self, monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_local_sandbox.py:152
Functiontest_required_behavior_judge_wrapper_keeps_inventory_on_unexpected_exception
(keyword_arguments)
tests/tier3/test_judge_result_semantics.py:99
Functiontest_required_check_counts_deduplicate_goal_and_security_failures
()
tests/test_tier3_attempt_verdict.py:187
← previousnext →21,201–21,300 of 24,741, ranked by callers