Function that can interact with PyTest. Returns a dict containing log files and other output data.
(cluster, dirname, weekly)
| 770 | kwargs['environment'] = environment |
| 771 | |
| 772 | def test_func(cluster, dirname, weekly): |
| 773 | """Function that can interact with PyTest. |
| 774 | |
| 775 | Returns a dict containing log files and other output data. |
| 776 | |
| 777 | """ |
| 778 | test_name = '{}'.format(test_name_base) |
| 779 | if cluster in skip_clusters: |
| 780 | e = "test \"%s\" not supported on cluster \"%s\"" % (test_name, cluster) |
| 781 | print('Skip - ' + e) |
| 782 | pytest.skip(e) |
| 783 | |
| 784 | # Load LBANN Python frontend |
| 785 | import lbann |
| 786 | import lbann.contrib.launcher |
| 787 | |
| 788 | # Setup LBANN experiment |
| 789 | trainer, model, data_reader, optimizer, req_num_nodes = setup_func(lbann, weekly) |
| 790 | |
| 791 | if req_num_nodes: |
| 792 | kwargs['nodes'] = req_num_nodes |
| 793 | |
| 794 | # Configure kwargs to LBANN launcher |
| 795 | _kwargs = copy.deepcopy(kwargs) |
| 796 | if 'work_dir' not in _kwargs: |
| 797 | _kwargs['work_dir'] = os.path.join(os.path.dirname(test_file), |
| 798 | 'experiments', |
| 799 | test_name) |
| 800 | |
| 801 | # If the user provided a suffix for the work directory, append it |
| 802 | if 'work_subdir' in _kwargs: |
| 803 | _kwargs['work_dir'] = os.path.join(_kwargs['work_dir'], _kwargs['work_subdir']) |
| 804 | del _kwargs['work_subdir'] |
| 805 | |
| 806 | # Delete the work directory |
| 807 | #if os.path.isdir(_kwargs['work_dir']): |
| 808 | # shutil.rmtree(_kwargs['work_dir']) |
| 809 | |
| 810 | if 'job_name' not in _kwargs: |
| 811 | _kwargs['job_name'] = f'lbann_{test_name}' |
| 812 | if 'overwrite_script' not in _kwargs: |
| 813 | _kwargs['overwrite_script'] = True |
| 814 | |
| 815 | # Set a default time limit for tests |
| 816 | if 'time_limit' not in _kwargs: |
| 817 | _kwargs['time_limit'] = 2 |
| 818 | |
| 819 | # Run LBANN |
| 820 | work_dir = _kwargs['work_dir'] |
| 821 | stdout_log_file = os.path.join(work_dir, 'out.log') |
| 822 | stderr_log_file = os.path.join(work_dir, 'err.log') |
| 823 | return_code = lbann.contrib.launcher.run( |
| 824 | trainer=trainer, |
| 825 | model=model, |
| 826 | data_reader=data_reader, |
| 827 | optimizer=optimizer, |
| 828 | **_kwargs, |
| 829 | ) |