| 12 | MIN_FILE_TOKENS = 100 |
| 13 | |
| 14 | def main(): |
| 15 | if len(sys.argv) <= 3: |
| 16 | raise ValueError('Provide a language, source directory and target directory.') |
| 17 | |
| 18 | language = sys.argv[1] |
| 19 | proj_dir = sys.argv[2] |
| 20 | out_dir = sys.argv[3] |
| 21 | |
| 22 | # Use Pygments to get language extensions. |
| 23 | lexer = get_lexer_by_name(language) |
| 24 | language_extensions = set(ext.lower()[1:] for ext in lexer.filenames) |
| 25 | |
| 26 | print(f'Processing: {proj_dir}') |
| 27 | if not os.path.exists(out_dir): |
| 28 | os.makedirs(out_dir) |
| 29 | |
| 30 | files_found = 0 |
| 31 | for root, _, files in os.walk(proj_dir): |
| 32 | for file in files: |
| 33 | if any(file.endswith(ext) for ext in language_extensions): |
| 34 | in_path = os.path.join(root, file) |
| 35 | if not os.path.exists(in_path): # Can happen due to broken symlinks. |
| 36 | continue |
| 37 | if os.path.getsize(in_path) > MAX_FILE_SIZE: # Drop excessively long files. |
| 38 | continue |
| 39 | with open(in_path, errors='ignore') as f_in: |
| 40 | text = f_in.read() |
| 41 | if sum(1 for _ in pygments.lex(text, lexer)) < MIN_FILE_TOKENS: # Drop files with too few tokens. |
| 42 | continue |
| 43 | |
| 44 | # Copy all other files to the target directory using a simplified path. |
| 45 | rel_path = root[len(proj_dir)+1:].replace('/', '__') |
| 46 | out_path = os.path.join(out_dir, rel_path + ('__' if rel_path else '') + file) |
| 47 | if not os.path.exists(out_path): |
| 48 | try: |
| 49 | copyfile(in_path, out_path) |
| 50 | except Exception as e: |
| 51 | print(f'Skipping problematic file {in_path} due to: {e}') |
| 52 | files_found += 1 |
| 53 | print(f'Done processing; copied {files_found} files.') |
| 54 | |
| 55 | |
| 56 | if __name__ == '__main__': |