| 90 | |
| 91 | |
| 92 | def generate_answer(): |
| 93 | |
| 94 | user_message = st.session_state.input_text |
| 95 | formatted_text = "{}\n<|Human|>: {}<eoh>\n<|MOSS|>:".format(st.session_state.prefix, user_message) |
| 96 | # st.info(formatted_text) |
| 97 | with st.spinner('MOSS is responding...'): |
| 98 | inference_start_time = time.time() |
| 99 | input_ids = tokenizer(formatted_text, return_tensors="pt").input_ids |
| 100 | input_ids = input_ids.cuda() |
| 101 | generated_ids = model.generate( |
| 102 | input_ids, |
| 103 | max_length=max_length+st.session_state.input_len, |
| 104 | temperature=temperature, |
| 105 | length_penalty=length_penalty, |
| 106 | max_time=max_time, |
| 107 | repetition_penalty=repetition_penalty, |
| 108 | stopping_criteria=stopping_criteria_list, |
| 109 | ) |
| 110 | st.session_state.input_len = len(generated_ids[0]) |
| 111 | # st.info(tokenizer.decode(generated_ids[0], skip_special_tokens=False)) |
| 112 | result = tokenizer.decode(generated_ids[0][input_ids.shape[1]:], skip_special_tokens=True) |
| 113 | inference_elapsed_time = time.time() - inference_start_time |
| 114 | |
| 115 | st.session_state.history.append( |
| 116 | {"message": user_message, "is_user": True} |
| 117 | ) |
| 118 | st.session_state.history.append( |
| 119 | {"message": result, "is_user": False, "time": inference_elapsed_time} |
| 120 | ) |
| 121 | |
| 122 | st.session_state.prefix = "{}{}<eom>".format(formatted_text, result) |
| 123 | st.session_state.num_queries += 1 |
| 124 | |
| 125 | |
| 126 | def clear_history(): |