Code
Hub
Workspaces
Following
Trending
Connect
MCP
copy
Create free account
hub
/
github.com/NVIDIA/SkillEvaluator
/ functions
Functions
24,741 in github.com/NVIDIA/SkillEvaluator
⨍
Functions
24,741
◇
Types & classes
1,292
↳
Endpoints
703
Function
test_bedrock_converse_transport_exhausts_configured_retries_on_persistent_429
Verify _call_bedrock with SKILL_EVAL_LLM_MAX_RETRIES=3 sends 4 total requests on persistent 429.
tests/test_retry.py:519
Function
test_bedrock_converse_transport_fails_fast_on_non_retriable_400
Verify _call_bedrock does not retry non-retriable HTTP 400 ValidationException.
tests/test_retry.py:537
Function
test_bedrock_converse_transport_positive_retries_overrides_aws_max_attempts_one
Verify _call_bedrock with SKILL_EVAL_LLM_MAX_RETRIES=3 retries 429 and succeeds even when AWS_MAX_ATTEMPTS=1.
tests/test_retry.py:495
Function
test_bedrock_converse_transport_respects_retry_after_and_max_delay_cap
Verify _call_bedrock honors Retry-After within max_delay and aborts immediately when Retry-After exceeds max_delay.
tests/test_retry.py:559
Function
test_bedrock_converse_transport_retries_transient_connection_error
Verify _call_bedrock retries transient botocore EndpointConnectionError up to SKILL_EVAL_LLM_MAX_RETRIES.
tests/test_retry.py:594
Function
test_bedrock_converse_transport_zero_retries_overrides_aws_max_attempts
Verify _call_bedrock with SKILL_EVAL_LLM_MAX_RETRIES=0 sends only 1 request even when AWS_MAX_ATTEMPTS=4.
tests/test_retry.py:473
Function
test_bedrock_custom_endpoint_metadata_cannot_make_custom_hosts_authoritative
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:1432
Function
test_bedrock_custom_endpoint_rules_cannot_make_custom_hosts_authoritative
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:1505
Method
test_bedrock_empty_response_uses_typed_error
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:402
Method
test_bedrock_forwards_only_an_explicit_timeout
( self, monkeypatch: pytest.MonkeyPatch, timeout: float | None, expected: dict )
tests/inference/test_llm_client.py:568
Method
test_bedrock_history_is_sent_through_litellm
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:1418
Function
test_bedrock_host_credential_allowlist_is_staged_for_the_verifier
()
tests/test_harbor_runner_environment.py:2319
Function
test_bedrock_model_probe_allows_distinct_routes_to_overlap
(monkeypatch)
tests/test_harbor_runtime_preflight.py:3529
Function
test_bedrock_model_probe_classifies_client_errors
( monkeypatch, error_code: str, http_status: int, expected_kind: str, expected_disposition
tests/test_harbor_runtime_preflight.py:1702
Function
test_bedrock_model_probe_classifies_credential_bootstrap_client_errors
( monkeypatch: pytest.MonkeyPatch, operation_name: str, error_code: str, http_status: int,
tests/test_harbor_runtime_preflight.py:1754
Function
test_bedrock_model_probe_classifies_invalid_compressed_local_sdk_data_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2547
Function
test_bedrock_model_probe_classifies_invalid_local_credential_token_encoding_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, credential_source: str, )
tests/test_harbor_runtime_preflight.py:2612
Function
test_bedrock_model_probe_classifies_malformed_assume_role_endpoint_rules_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2716
Function
test_bedrock_model_probe_classifies_malformed_assume_role_service_data_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2675
Function
test_bedrock_model_probe_classifies_malformed_credential_process_without_blame_unused_service_data
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2991
Function
test_bedrock_model_probe_classifies_malformed_effective_proxy_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, proxy_url: str, )
tests/test_harbor_runtime_preflight.py:2025
Function
test_bedrock_model_probe_classifies_malformed_local_endpoints_data_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2799
Function
test_bedrock_model_probe_classifies_malformed_local_login_token_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2936
Function
test_bedrock_model_probe_classifies_malformed_local_sso_token_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, expires_at: object, expected_error: str, )
tests/test_harbor_runtime_preflight.py:2886
Function
test_bedrock_model_probe_classifies_malformed_nested_sso_service_data_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2834
Function
test_bedrock_model_probe_classifies_malformed_runtime_endpoint_rules_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, service_name: str, api_version: str, )
tests/test_harbor_runtime_preflight.py:2761
Function
test_bedrock_model_probe_classifies_out_of_range_profile_max_attempts_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, max_attempts: str, )
tests/test_harbor_runtime_preflight.py:2472
Function
test_bedrock_model_probe_classifies_proven_invalid_ca_bundle_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, bundle_state: str, configuration_source: st
tests/test_harbor_runtime_preflight.py:3030
Function
test_bedrock_model_probe_classifies_real_invalid_assume_role_configuration_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, environment: dict[str, str], expected_error
tests/test_harbor_runtime_preflight.py:3316
Function
test_bedrock_model_probe_classifies_real_invalid_assume_role_parameters_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3356
Function
test_bedrock_model_probe_classifies_real_invalid_aws_environment_as_fatal
( monkeypatch: pytest.MonkeyPatch, environment: dict[str, str], expected_error: str, )
tests/test_harbor_runtime_preflight.py:2373
Function
test_bedrock_model_probe_classifies_real_invalid_credential_process_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3142
Function
test_bedrock_model_probe_classifies_real_invalid_profile_api_version_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2400
Function
test_bedrock_model_probe_classifies_real_invalid_profile_max_attempts_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2436
Function
test_bedrock_model_probe_classifies_real_invalid_region_as_fatal
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_preflight.py:2326
Function
test_bedrock_model_probe_classifies_real_malformed_local_service_data_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, service_name: str, )
tests/test_harbor_runtime_preflight.py:2509
Function
test_bedrock_model_probe_classifies_real_unexecutable_credential_process_as_fatal
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3172
Function
test_bedrock_model_probe_classifies_structural_credential_process_output_causally
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, payload: dict[str, object] | None, exit_cod
tests/test_harbor_runtime_preflight.py:3225
Function
test_bedrock_model_probe_deadline_caps_blocked_workers
(monkeypatch)
tests/test_harbor_runtime_preflight.py:3595
Function
test_bedrock_model_probe_deadline_covers_blocked_credential_provider
(monkeypatch)
tests/test_harbor_runtime_preflight.py:3565
Function
test_bedrock_model_probe_degrades_malformed_catalog_timestamp
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3393
Function
test_bedrock_model_probe_degrades_malformed_container_credential_timestamp
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3455
Function
test_bedrock_model_probe_degrades_malformed_imds_credential_responses
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, credential_body: bytes, )
tests/test_harbor_runtime_preflight.py:1911
Function
test_bedrock_model_probe_degrades_malformed_imds_role_listing
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:1968
Function
test_bedrock_model_probe_degrades_transient_credential_process_spawn_error
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3267
Function
test_bedrock_model_probe_degrades_transport_sdk_errors
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_runtime_preflight.py:2304
Function
test_bedrock_model_probe_degrades_unattributed_tls_errors
(monkeypatch: pytest.MonkeyPatch)
tests/test_harbor_runtime_preflight.py:3080
Function
test_bedrock_model_probe_degrades_unattributed_value_errors
( monkeypatch: pytest.MonkeyPatch, phase: str, )
tests/test_harbor_runtime_preflight.py:2269
Function
test_bedrock_model_probe_degrades_valid_but_unavailable_proxy
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2061
Function
test_bedrock_model_probe_degrades_when_active_imds_is_unavailable
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:1790
Function
test_bedrock_model_probe_distinguishes_transient_imds_from_no_role
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int, expected_kind: str, e
tests/test_harbor_runtime_preflight.py:1852
Function
test_bedrock_model_probe_does_not_blame_proxy_bypassed_by_no_proxy
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:2088
Function
test_bedrock_model_probe_does_not_blame_remote_bad_gzip_as_local_configuration
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_preflight.py:2583
Function
test_bedrock_model_probe_does_not_blame_remote_unicode_error_as_local_configuration
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runtime_preflight.py:2647
Function
test_bedrock_model_probe_fails_on_deterministic_sdk_errors
( monkeypatch: pytest.MonkeyPatch, error: Exception, expected_kind: str, )
tests/test_harbor_runtime_preflight.py:2240
Function
test_bedrock_model_probe_fails_when_credentials_are_absent_and_imds_is_disabled
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, bearer_token: str | None, )
tests/test_harbor_runtime_preflight.py:1821
Function
test_bedrock_model_probe_rejects_invalid_ca_bundle_before_https_runtime
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, )
tests/test_harbor_runtime_preflight.py:3098
Function
test_bedrock_model_probe_rejects_unsafe_bearer_token_characters
( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, bearer_token: str, )
tests/test_harbor_runtime_preflight.py:1665
Function
test_bedrock_model_probe_releases_slot_when_worker_creation_fails
( monkeypatch: pytest.MonkeyPatch, failure_phase: str, )
tests/test_harbor_runtime_preflight.py:3625
Function
test_bedrock_model_probe_uses_a_fresh_session_per_route
(monkeypatch)
tests/test_harbor_runtime_preflight.py:3504
Method
test_bedrock_opus5_preserves_token_limits_and_omits_temperature
( self, monkeypatch: pytest.MonkeyPatch, max_tokens: int | None, )
tests/inference/test_llm_client.py:545
Method
test_bedrock_prompt_only_reply_with_preamble_is_recovered
(self, monkeypatch: pytest.MonkeyPatch)
tests/inference/test_llm_client.py:968
Function
test_bedrock_provider_uses_claude_opus_5_5_us_profile_by_default
()
tests/test_provider_config.py:63
Function
test_bedrock_refusal_fallback_equal_to_the_request_model_is_not_sent
(verifier, monkeypatch)
tests/tier3/test_judge_policies.py:238
Function
test_bedrock_refusal_fallback_uses_a_qualified_bedrock_id_as_given
(verifier, monkeypatch, value, expected_calls)
tests/tier3/test_judge_policies.py:265
Function
test_bedrock_subprocess_environment_keeps_only_explicit_aws_provider_values
( monkeypatch: pytest.MonkeyPatch, )
tests/test_harbor_runner_environment.py:2270
Function
test_behavior_abstention_requires_a_reason
(module, reason)
tests/tier3/test_judge_result_semantics.py:134
Function
test_behavior_accepts_unique_object_before_unmatched_trailing_prose_without_retry
( judge_module, monkeypatch, )
tests/tier3/test_judge_result_semantics.py:1053
Function
test_behavior_accepts_valid_zero_and_boolean_results
( judge_module, monkeypatch, payload: dict, expected_behaviors: list[str], expected: float
tests/tier3/test_judge_result_semantics.py:1035
Function
test_behavior_aggregate_expected_class_uses_suite_threshold
()
tests/tier3/test_grader_policy.py:233
Function
test_behavior_bundle_complete_evidence_is_not_reported_truncated
()
tests/tier3/test_evidence_compiler.py:470
Function
test_behavior_bundle_completeness_follows_the_final_response_limit_env_var
( monkeypatch: pytest.MonkeyPatch, )
tests/tier3/test_evidence_compiler.py:446
Function
test_behavior_bundle_final_answer_past_history_repeat_is_truncated
()
tests/tier3/test_evidence_compiler.py:501
Function
test_behavior_bundle_final_answer_split_across_history_and_section
( monkeypatch: pytest.MonkeyPatch, length: int, truncated: bool, cut_char: str )
tests/tier3/test_evidence_compiler.py:531
Function
test_behavior_bundle_keeps_a_realistic_final_answer_at_the_default_limit
()
tests/tier3/test_evidence_compiler.py:392
Function
test_behavior_bundle_mid_length_final_answer_in_history_is_not_truncated
()
tests/tier3/test_evidence_compiler.py:482
Function
test_behavior_bundle_reason_names_each_setting_that_cut_evidence
( monkeypatch: pytest.MonkeyPatch, env: dict[str, str], reason: str, incomplete: list[str] )
tests/tier3/test_evidence_compiler.py:571
Function
test_behavior_bundle_reports_final_response_cap_loss
()
tests/tier3/test_evidence_compiler.py:374
Function
test_behavior_bundle_reports_sections_cut_by_their_own_per_entry_caps
()
tests/tier3/test_evidence_compiler.py:409
Function
test_behavior_check_budget_and_compactor_env_override
(monkeypatch)
tests/tier3/test_behavior_evidence.py:879
Function
test_behavior_check_budget_auto_reconciles_tool_history_headroom
(monkeypatch)
tests/tier3/test_behavior_evidence.py:896
Function
test_behavior_check_does_not_recut_compiled_evidence
(monkeypatch)
tests/tier3/test_judge_uses_evidence.py:35
Function
test_behavior_check_failure_warns_without_gating_attempt
()
tests/test_tier3_attempt_verdict.py:393
Function
test_behavior_coverage_and_failure_reasons_reach_report_details
()
tests/tier3/test_declarative_checks.py:212
Function
test_behavior_deep_nesting_is_scoreless_instead_of_raising
(judge_module, monkeypatch)
tests/tier3/test_judge_result_semantics.py:1204
Function
test_behavior_evidence_and_compactor_handle_tiny_limits
(monkeypatch, tiny_limit: int)
tests/tier3/test_behavior_evidence.py:1067
Function
test_behavior_evidence_default_keeps_a_realistic_final_answer
()
tests/tier3/test_behavior_evidence.py:814
Function
test_behavior_evidence_does_not_treat_read_like_tool_names_as_writes
()
tests/tier3/test_behavior_evidence.py:639
Function
test_behavior_evidence_final_response_default_and_env_override
(monkeypatch)
tests/tier3/test_behavior_evidence.py:796
Function
test_behavior_evidence_final_response_explicit_arg
()
tests/tier3/test_behavior_evidence.py:827
Function
test_behavior_evidence_gap_treats_an_absent_verified_flag_as_unknown
(judge_module, monkeypatch)
tests/tier3/test_judge_contracts.py:580
Function
test_behavior_evidence_ignores_non_write_shell_gt
()
tests/tier3/test_behavior_evidence.py:721
Function
test_behavior_evidence_keeps_write_when_prompt_is_long
()
tests/tier3/test_behavior_evidence.py:554
Function
test_behavior_evidence_prioritizes_late_write_tool_calls
()
tests/tier3/test_behavior_evidence.py:98
Function
test_behavior_evidence_standalone_reconciles_max_chars
(monkeypatch)
tests/tier3/test_behavior_evidence.py:846
Function
test_behavior_evidence_user_request_allowance_is_capped_above_the_old_800
()
tests/tier3/test_behavior_evidence.py:581
Function
test_behavior_evidence_user_request_limit_never_exceeds_the_final_response_limit
(monkeypatch)
tests/tier3/test_behavior_evidence.py:627
Function
test_behavior_expected_class_uses_suite_threshold
()
tests/tier3/test_grader_validation.py:445
Function
test_behavior_facts_describe_only_the_compacted_evidence
(monkeypatch, implementation)
tests/test_character_facts.py:197
Function
test_behavior_first_provider_error_is_scoreless_without_retry
(judge_module, monkeypatch)
tests/tier3/test_judge_result_semantics.py:927
← previous
next →
13,101–13,200 of 24,741, ranked by callers