| 65 | |
| 66 | @dataclass |
| 67 | class BaseConfig: |
| 68 | |
| 69 | mode: SEARCH_CHOICES = field( |
| 70 | default="mcts", metadata={"help": "search mode for inference"} |
| 71 | ) |
| 72 | model_dir: Optional[str] = field( |
| 73 | default=None, metadata={"help": "llm model dir"} |
| 74 | ) |
| 75 | few_shot_path: Optional[str] = field( |
| 76 | default=None, metadata={"help": "few shot data json"} |
| 77 | ) |
| 78 | prompt_path: Optional[str] = field( |
| 79 | default=None, metadata={"help": "prompt config json"} |
| 80 | ) |
| 81 | num_few_shot: int = field( |
| 82 | default=2, metadata={"help": "the number of few-shot examples"} |
| 83 | ) |
| 84 | create_local_llm: bool = field( |
| 85 | default=False, metadata={"help": "not for batch inference"} |
| 86 | ) |
| 87 | # prompt args |
| 88 | prompt_wrap: PROMPT_CHOICES = field( |
| 89 | default="react", metadata={"help": "prompt wrap type"} |
| 90 | ) |
| 91 | result_unwrap: PROMPT_CHOICES = field( |
| 92 | default="react", metadata={"help": "result unwrap"} |
| 93 | ) |
| 94 | step_delim: str = field( |
| 95 | default="\n\n", metadata={"help": "delimiter between two steps"} |
| 96 | ) |
| 97 | # vllm args |
| 98 | temperature: float = field( |
| 99 | default=0.7, metadata={"help": "control diversity of llm generation"} |
| 100 | ) |
| 101 | top_p: float = field( |
| 102 | default=1.0, metadata={"help": "Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens."} |
| 103 | ) |
| 104 | top_k: float = field( |
| 105 | default=-1.0, metadata={"help": "Float that controls the probability of other highly-scored candidates to be chosen"} |
| 106 | ) |
| 107 | use_beam_search: bool = field( |
| 108 | default=False, metadata={"help": "whether to enable beam search decoding"} |
| 109 | ) |
| 110 | best_of: int = field( |
| 111 | default=1, metadata={"help": "Integer that controls the number of candidate considered in the beam search decoding process"} |
| 112 | ) |
| 113 | max_tokens: int = field( |
| 114 | default=2000, metadata={"help": "Maximum number of tokens to generate per output sequence."} |
| 115 | ) |
| 116 | seed: Optional[int] = field( |
| 117 | default=None, metadata={"help": "seed of llm generation for reproducible"} |
| 118 | ) |
| 119 | swap_space: Optional[int] = field( |
| 120 | default=8, metadata={"help": "swap space for vllm"} |
| 121 | ) |
| 122 | n_generate_sample: int = field( |
| 123 | default=1, metadata={"help": "how many samples generated for each step. B2 in paper."} |
| 124 | ) |