This method tests if all sampling parameters are correctly processed NOTE: this does NOT test for correctness, just that we received a compatible response
(self)
| 78 | self.tearDown() |
| 79 | |
| 80 | def test_sampling_params(self): |
| 81 | """ |
| 82 | This method tests if all sampling parameters are correctly processed |
| 83 | NOTE: this does NOT test for correctness, just that we received a compatible response |
| 84 | """ |
| 85 | try: |
| 86 | self.setUp() |
| 87 | with grpc.insecure_channel("localhost:50051") as channel: |
| 88 | stub = backend_pb2_grpc.BackendStub(channel) |
| 89 | response = stub.LoadModel(backend_pb2.ModelOptions(Model="mlx-community/Llama-3.2-1B-Instruct-4bit")) |
| 90 | self.assertTrue(response.success) |
| 91 | |
| 92 | req = backend_pb2.PredictOptions( |
| 93 | Prompt="The capital of France is", |
| 94 | TopP=0.8, |
| 95 | Tokens=50, |
| 96 | Temperature=0.7, |
| 97 | TopK=40, |
| 98 | PresencePenalty=0.1, |
| 99 | FrequencyPenalty=0.2, |
| 100 | MinP=0.05, |
| 101 | Seed=42, |
| 102 | StopPrompts=["\n"], |
| 103 | IgnoreEOS=True, |
| 104 | ) |
| 105 | resp = stub.Predict(req) |
| 106 | self.assertIsNotNone(resp.message) |
| 107 | except Exception as err: |
| 108 | print(err) |
| 109 | self.fail("sampling params service failed") |
| 110 | finally: |
| 111 | self.tearDown() |
| 112 | |
| 113 | |
| 114 | def test_embedding(self): |