Method validate_data: validates that all files in the results folder are correct (have the correct name contents). Validates also that there are no missing files in the folder. If some error detected, the method raises the error
(gtFilePath, submFilePath)
| 34 | |
| 35 | |
| 36 | def validate_data(gtFilePath, submFilePath): |
| 37 | """ |
| 38 | Method validate_data: validates that all files in the results folder are correct (have the correct name contents). |
| 39 | Validates also that there are no missing files in the folder. |
| 40 | If some error detected, the method raises the error |
| 41 | """ |
| 42 | |
| 43 | gtJson = json.load(open(gtFilePath,'rb')); |
| 44 | submJson = json.load(open(submFilePath,'rb')); |
| 45 | |
| 46 | if not 'data' in gtJson: |
| 47 | raise Exception("The GT file is not valid (no data key)") |
| 48 | |
| 49 | if not 'dataset_name' in gtJson: |
| 50 | raise Exception("The GT file is not valid (no dataset_name key)") |
| 51 | |
| 52 | if isinstance(submJson, list) == False : |
| 53 | raise Exception("The Det file is not valid (root item must be an array)") |
| 54 | |
| 55 | if len(submJson) != len(gtJson['data']) : |
| 56 | raise Exception("The Det file is not valid (invalid number of answers. Expected:" + str(len(gtJson['data'])) + " Found:" + str(len(submJson)) + ")") |
| 57 | |
| 58 | gtQuestions = sorted([r['questionId'] for r in gtJson['data']]) |
| 59 | res_id_to_index = {int(r['questionId']): ix for ix, r in enumerate(submJson)} |
| 60 | detQuestions = sorted([r['questionId'] for r in submJson]) |
| 61 | |
| 62 | if( (gtQuestions == detQuestions) == False ): |
| 63 | raise Exception("The Det file is not valid. Question IDs must much GT") |
| 64 | |
| 65 | for gtObject in gtJson['data']: |
| 66 | |
| 67 | try: |
| 68 | q_id = int(gtObject['questionId']); |
| 69 | res_ix = res_id_to_index[q_id]; |
| 70 | |
| 71 | except: |
| 72 | raise Exception("The Det file is not valid. Question " + str(gtObject['questionId']) + " not present") |
| 73 | |
| 74 | else: |
| 75 | detObject = submJson[res_ix]; |
| 76 | |
| 77 | # if detObject['questionId'] != gtObject['questionId'] : |
| 78 | # raise Exception("Answer #" + str(i) + " not valid (invalid question ID. Expected:" + str(gtObject['questionId']) + "Found:" + detObject['questionId'] + ")") |
| 79 | |
| 80 | if not 'answer' in detObject: |
| 81 | raise Exception("Question " + str(gtObject['questionId']) + " not valid (no answer key)") |
| 82 | |
| 83 | if isinstance(detObject['answer'], list) == True : |
| 84 | raise Exception("Question " + str(gtObject['questionId']) + " not valid (answer key has to be a single string)") |
| 85 | |
| 86 | |
| 87 | def evaluate_method(gtFilePath, submFilePath, evaluationParams): |
no outgoing calls
no test coverage detected