This class controls the query generator. Generates new schedule_items regularly and places them into the schedule directory. Schedule_items can also be generated by other means (for example front_end.py), so it checks schedule directory regularly and starts running new jobs. It seemed easier a
| 48 | |
| 49 | |
| 50 | class Controller(object): |
| 51 | '''This class controls the query generator. Generates new schedule_items regularly and |
| 52 | places them into the schedule directory. Schedule_items can also be generated by other |
| 53 | means (for example front_end.py), so it checks schedule directory regularly and starts |
| 54 | running new jobs. It seemed easier and more convenient to implement the scheduling |
| 55 | mechanism this way, rather than use Jenkins. |
| 56 | |
| 57 | This is indended to be running on machine dedicated to be running the query generator. |
| 58 | TARGET_HOST environment variable should be set to the address of the host that will be |
| 59 | running Impala. The target machine should have Docker installed and configured. Each job |
| 60 | will be run in a separate Docker container. The Docker Image can be specified by setting |
| 61 | the DOCKER_IMAGE_NAME environment variable. The Image needs have Postgres installed and |
| 62 | appropriate data loaded. |
| 63 | |
| 64 | Attributes: |
| 65 | schedule_items: Keeps track of active job threads. This maps job id to the thread that |
| 66 | running it. |
| 67 | time_last_generated: Stores the time when a schedule was last generated automatically. |
| 68 | Used to control the rate at which new schedule_items are generated. |
| 69 | ''' |
| 70 | |
| 71 | def __init__(self): |
| 72 | self.check_env_vars() |
| 73 | self.make_local_dirs() |
| 74 | |
| 75 | self.schedule_items = {} |
| 76 | self.time_last_generated = 0 |
| 77 | |
| 78 | def make_local_dirs(self): |
| 79 | '''Create directories for schedule, log and results. |
| 80 | ''' |
| 81 | if not os.path.exists(PATH_TO_SCHEDULE): |
| 82 | os.makedirs(PATH_TO_SCHEDULE) |
| 83 | if DELETE_SCHEDULE_ITEMS_ON_STARTUP: |
| 84 | for job_id in os.listdir(PATH_TO_SCHEDULE): |
| 85 | os.remove(os.path.join(PATH_TO_SCHEDULE, job_id)) |
| 86 | if not os.path.exists(PATH_TO_FINISHED_JOBS): |
| 87 | os.makedirs(PATH_TO_FINISHED_JOBS) |
| 88 | if not os.path.exists(PATH_TO_REPORTS): |
| 89 | os.makedirs(PATH_TO_REPORTS) |
| 90 | try: |
| 91 | os.remove(PATH_TO_LOG) |
| 92 | except OSError: |
| 93 | # Log file could not be removed most likely because it does not exist, so this |
| 94 | # exception can be ignored. |
| 95 | pass |
| 96 | |
| 97 | def check_env_vars(self): |
| 98 | '''Check if all necessary enivornment variables have been set.''' |
| 99 | if 'DOCKER_PASSWORD' not in os.environ: |
| 100 | exit('DOCKER_PASSWORD environment variable not set') |
| 101 | if 'TARGET_HOST' not in os.environ: |
| 102 | exit('TARGET_HOST environment variable not set') |
| 103 | if 'TARGET_HOST_USERNAME' not in os.environ: |
| 104 | exit('TARGET_HOST_USERNAME environment variable not set') |
| 105 | if 'DOCKER_IMAGE_NAME' not in os.environ: |
| 106 | exit('DOCKER_IMAGE_NAME environment variable not set') |
| 107 |