MCPcopy Create free account

hub / github.com/NVIDIA/SkillEvaluator / functions

Functions24,741 in github.com/NVIDIA/SkillEvaluator

Functiontest_timeout_multiplier_is_validated_and_recorded
( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, keep_staged_tasks: None )
tests/test_nemo_gym_backend.py:1332
Functiontest_timeout_multiplier_scales_staged_task_timeouts
(tmp_path: Path)
tests/test_nemo_gym_backend.py:1275
Methodtest_timeout_preserves_partial_output
(self)
tests/utils/test_tool_runner.py:291
Functiontest_timing_comparison_does_not_require_token_counters
()
tests/tier3/test_resource_usage.py:149
Functiontest_timing_pairs_arms_and_derives_ms_per_token
()
tests/tier3/test_resource_usage.py:100
Functiontest_timing_totals_are_rounded_to_milliseconds
()
tests/tier3/test_resource_usage.py:120
Methodtest_title_is_required
Test that title field is required for Rules.
tests/validators/test_rules_schema.py:253
Methodtest_title_is_required
Test that title field is required for Workflows.
tests/validators/test_workflows_schema.py:359
Functiontest_tls_options_are_openclaw_specific
()
tests/validators/test_openclaw_mcp.py:161
Functiontest_to_agentskills_dataset_preserves_aces_metadata
()
tests/tier3/test_generate_dataset_results.py:300
Methodtest_to_dict
(self)
tests/validators/test_quality_score.py:189
Functiontest_token_and_cost_admission
()
tests/tier3/test_run_budget.py:29
Functiontest_token_case_rows_go_before_any_score_evidence_under_the_size_budget
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path )
tests/reporting/test_tier3_token_usage_reports.py:529
Functiontest_token_chart_uses_both_arms_without_counting_cached_reads_twice
()
tests/reporting/test_telemetry_view.py:167
Functiontest_token_count_flags_remain_readable_in_saved_findings
(copy: str)
tests/tier3/test_security_finding_evidence.py:2143
Functiontest_token_efficiency_half_life_matches_constants
()
tests/test_metrics_reference_docs.py:181
Functiontest_token_estimate_divisor_matches_constants
()
tests/test_metrics_reference_docs.py:395
Functiontest_token_estimate_labels_survive_redaction
No quality metric, including the estimate labels, is rewritten as a secret.
tests/validators/test_quality_score.py:2114
Functiontest_token_shaped_url_path_segment_is_an_inline_secret
(url: str)
tests/validators/test_mcp_static.py:972
Functiontest_token_suffix_credential_keys_still_flagged
(key)
tests/validators/test_mcp_static.py:798
Functiontest_token_usage_never_pushes_a_payload_over_the_report_node_limit
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, room: str )
tests/reporting/test_tier3_token_usage_reports.py:552
Functiontest_tokenless_trajectory_still_reports_authentication_failure
(metrics: dict[str, int] | None)
tests/test_harbor_collector_runtime_failures.py:338
Functiontest_tolerance_needs_a_baseline_and_dimension_grading
(tmp_path: Path, kwargs: dict[str, Any])
tests/tier3/test_baseline_timeouts.py:1145
Functiontest_tolerated_agent_stays_scored_on_its_with_skill_arm
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:240
Functiontest_toml_quote_rejects_lone_surrogates
()
tests/test_harbor_scaffold_transactions.py:487
Functiontest_toml_quote_round_trips_valid_strings
(value: str)
tests/test_harbor_scaffold_transactions.py:483
Functiontest_toml_value_rejects_non_string_dict_elements
Reject dictionaries with non-string keys or values.
tests/test_harbor_adapter_safety.py:583
Functiontest_toml_value_serializes_dict_headers
Serialize string dictionaries as inline TOML tables.
tests/test_harbor_adapter_safety.py:575
Functiontest_too_few_valid_entries_still_reports_exclusions
(tmp_path: Path, monkeypatch)
tests/embedding/test_collection_coverage.py:149
Methodtest_tool_error_message_wins_over_provider_message
A runner error on the LLM stage is reported as is, even with an outage report on exit 2.
tests/validators/test_security.py:7330
Methodtest_tool_names_map_to_declared_servers
( self, agent: str, fn: str, servers: list[str], tool: str | None )
tests/tier3/test_plugin_signals.py:377
Functiontest_tool_only_trajectory_retains_latest_verification_failure
(monkeypatch, implementation, budgets, metric)
tests/tier3/test_evidence_compiler.py:320
Functiontest_tool_result_checks_match_recorded_content_text_only
()
tests/tier3/test_grader_validation.py:214
Functiontest_tool_step_followed_by_a_user_or_system_step_is_unmeasured
(source: str)
tests/tier3/test_trajectory_timing.py:226
Functiontest_tool_strictness_is_preserved_in_chat_tools
(translator: object, payload: dict[str, object])
tests/test_nvidia_build_bridge.py:2722
Functiontest_tool_total_requires_every_trial
()
tests/tier3/test_resource_usage.py:131
Methodtest_tools_dir_is_script_based
(self, tmp_path)
tests/validators/test_quality_score.py:222
Methodtest_tools_directory_scripts_reach_the_judge_without_colliding_with_scripts
(self, tmp_path)
tests/validators/test_rubric_eval.py:1758
Methodtest_tools_registry
Test Tools registry has expected tools.
tests/validators/test_security_validators.py:1011
Functiontest_top_issue_is_bounded_single_line_plain_text_from_highest_severity_only
()
tests/ci/test_summary_metrics.py:73
Functiontest_top_issues_count_a_folded_pii_finding_as_one_record
(tmp_path)
tests/reporting/test_reporters.py:1343
Methodtest_top_issues_cover_every_tier_while_the_html_panel_lists_tier1
( self, success_result: ValidationResult )
tests/reporting/test_reporters.py:647
Functiontest_top_level_directory_link_must_stay_within_allowed_root
(tmp_path: Path)
tests/test_harbor_secure_copy.py:137
Functiontest_top_level_help_groups_commands_by_workflow
()
tests/test_cli.py:80
Functiontest_top_level_help_lists_primary_commands
()
tests/test_cli.py:23
Functiontest_top_level_help_shows_branded_header
()
tests/test_cli.py:43
Functiontest_top_level_source_symlink_is_rejected
(tmp_path: Path)
tests/test_harbor_secure_copy.py:194
Functiontest_top_level_version_is_ignored_when_metadata_version_missing
Legacy top-level ``version`` (e.g., ``1.0``) must not trigger semver checks. Pre-proposal, quality-score recommended a top-level ``version`` with
tests/validators/test_version.py:143
Functiontest_top_level_version_is_ignored_when_metadata_version_present
A top-level ``version`` is shadowed by ``metadata.version``. Only ``metadata.version`` is authoritative, even when a legacy top-level ``versi
tests/validators/test_version.py:159
Functiontest_top_words_keep_numbers_and_inner_punctuation
()
tests/test_reference_text_analyzer.py:33
Functiontest_top_words_skip_punctuation_only_tokens
()
tests/test_reference_text_analyzer.py:26
Methodtest_total_byte_limit_is_enforced_before_embedding
(self, tmp_path: Path, write_skill, monkeypatch)
tests/embedding/test_registry.py:344
Functiontest_trace_aborts_when_no_trajectory_id_matches_existing_cases
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:195
Functiontest_trace_fingerprints_use_redacted_prompt_and_fixtures
()
tests/test_trace_import.py:407
Functiontest_trace_grounding_fails_when_dry_run_has_no_trajectory
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:551
Functiontest_trace_preserves_authored_ground_truth_and_flags_mismatches
()
tests/tier3/test_dataset_grounding.py:309
Functiontest_trace_proposes_unauthored_observed_answer_and_flags_review
()
tests/tier3/test_dataset_grounding.py:388
Functiontest_trace_refinement_keeps_template_mode_and_records_refined
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:633
Functiontest_trace_restores_authored_dataset_when_collection_fails
(tmp_path, monkeypatch)
tests/tier3/test_dataset_grounding.py:216
Functiontest_tracked_symlink_manifest_is_rejected
(repository, tmp_path)
tests/test_ci_onboarding_git.py:58
Functiontest_trailing_backslashes_render_literally
(monkeypatch: pytest.MonkeyPatch)
tests/reporting/test_rich_markup_escaping.py:269
Functiontest_trailing_backslashes_render_literally
(context: str, text: str)
tests/utils/test_rich_markup.py:74
Methodtest_trailing_hyphen
Test trailing hyphen is rejected by kebab-case pattern.
tests/validators/test_naming_utils.py:38
Functiontest_trailing_tool_step_without_a_later_timestamp_is_rejected
()
tests/tier3/test_trajectory_timing.py:221
Functiontest_trajectory_completion_tokens_prove_progress
(tmp_path: Path)
tests/tier3/test_baseline_timeouts.py:403
Functiontest_trajectory_load_failure_replaces_default_evidence_reason
()
tests/tier3/test_plain_language_findings.py:91
Functiontest_trajectory_load_metadata_is_not_reported_as_a_load_failure
()
tests/tier3/test_plain_language_findings.py:102
Functiontest_trajectory_reason_does_not_hide_missing_trial
()
tests/reporting/test_dashboard_data.py:626
Functiontest_trajectory_redaction_masks_plural_and_nonnumeric_token_fields
(tmp_path: Path)
tests/test_harbor_collector_multistep_trajectory.py:2241
Functiontest_trajectory_redaction_masks_uri_userinfo_and_remains_atif_valid
(tmp_path: Path)
tests/test_harbor_collector_multistep_trajectory.py:2222
Functiontest_trajectory_redaction_omits_token_id_arrays_and_remains_atif_valid
(tmp_path: Path)
tests/test_harbor_collector_multistep_trajectory.py:2195
Functiontest_trajectory_without_tool_calls_measures_zero_tool_time
()
tests/tier3/test_trajectory_timing.py:242
Functiontest_transient_429_preserves_schema_and_does_not_memoize_unsupported
Verify HTTP 429 retries with backoff while keeping response_format intact.
tests/test_llm_judge.py:637
Functiontest_transient_installer_failure_is_retried
(tmp_path: Path, agent_class: type, sleeps: list[float])
tests/tier3/test_installed_agent_install_retry.py:151
Functiontest_transient_system_package_failure_is_retried
(tmp_path: Path, agent_class: type, sleeps: list[float])
tests/tier3/test_installed_agent_install_retry.py:239
Methodtest_transitive_import_retains_finding
(self, tmp_path: Path)
tests/validators/test_skillspector_scope.py:910
Methodtest_transitive_relative_import_retains_finding
( self, tmp_path: Path, entrypoint: str, entrypoint_contents: str, helper: str )
tests/validators/test_skillspector_scope.py:933
Functiontest_translated_request_expansion_is_rejected_before_build_credentials_are_used
( bridge_services: _BridgeServices, )
tests/test_nvidia_build_bridge.py:1611
Functiontest_transport_and_malformed_responses_fail_without_credential_leaks
(gate, monkeypatch, capsys, payload)
tests/ci/test_gitlab_default_head_admission.py:151
Functiontest_transport_and_response_errors_do_not_echo_raw_content
(exception, expected)
tests/inference/test_diagnostics.py:75
Functiontest_transport_failure_does_not_print_server_data
(request_env, monkeypatch, capsys, error_type)
tests/ci/test_gitlab_feedback_request.py:168
Functiontest_transport_kind_mismatch_command_http_blocked
()
tests/validators/test_mcp_static.py:759
Functiontest_transport_kind_mismatch_url_stdio_blocked
()
tests/validators/test_mcp_static.py:764
Functiontest_transport_precedence_and_sse_default
(tmp_path: Path, server: dict, transport: str)
tests/validators/test_openclaw_mcp.py:63
Functiontest_transport_uppercase_casing_blocked
()
tests/validators/test_mcp_static.py:769
Functiontest_transport_url_sse_allowed
()
tests/validators/test_mcp_static.py:774
Functiontest_transport_value_error_is_a_safe_gateway_error
(bridge_services: _BridgeServices)
tests/test_nvidia_build_bridge.py:1578
Methodtest_traversal_outside_evals_blocked
(self, tmp_path: Path)
tests/test_harbor_input_staging.py:386
Functiontest_trial_artifact_symlinks_cannot_supply_advice
(tmp_path: Path)
tests/test_tier3_case_summary.py:1896
Functiontest_trial_cap_bounds_reward_and_trajectory_work_and_marks_truncation
(tmp_path, monkeypatch)
tests/tier3/test_plugin_evidence.py:466
Functiontest_trial_charts_include_agent_condition_series
()
tests/reporting/test_tier3_baseline_trial_views.py:197
Functiontest_trial_charts_preserve_token_and_step_counter_semantics
()
tests/reporting/test_unified_tier3_report.py:1008
Functiontest_trial_failure_names_docker_address_pool_exhaustion
(tmp_path: Path)
tests/test_harbor_collector_runtime_failures.py:2904
Functiontest_trial_names_cannot_escape_the_run_directory
(tmp_path: Path)
tests/test_tier3_case_summary.py:1021
Functiontest_trial_report_omits_token_chart_for_incomplete_agent_telemetry
()
tests/reporting/test_unified_tier3_report.py:865
Functiontest_trial_report_omits_token_chart_when_any_agent_telemetry_is_incomplete
()
tests/reporting/test_unified_tier3_report.py:912
Functiontest_trial_report_omits_token_chart_when_reported_agent_has_no_trials
()
tests/reporting/test_unified_tier3_report.py:965
Functiontest_trial_report_renders_unavailable_tokens_as_unavailable
()
tests/reporting/test_unified_tier3_report.py:785
Methodtest_trial_sampling_preserves_multi_agent_and_attempt_identity_without_trial_ids
(self)
tests/evaluation/test_judges.py:246
Methodtest_trial_sampling_prioritizes_trusted_negative_control_failure_evidence
(self)
tests/evaluation/test_judges.py:265
← previousnext →23,401–23,500 of 24,741, ranked by callers