shuffle-merges files InFilePrefix_X, X in { 0, 1, ... NProcs } and stores the result into sm-InFilePrefix. Notes: does NOT check if the input files are available.
( InFilePrefix, NProcs, MaxLines )
| 44 | |
| 45 | |
| 46 | def ShuffleMerge( InFilePrefix, NProcs, MaxLines ): |
| 47 | """ |
| 48 | shuffle-merges files InFilePrefix_X, X in { 0, 1, ... NProcs } and |
| 49 | stores the result into sm-InFilePrefix. |
| 50 | |
| 51 | Notes: does NOT check if the input files are available. |
| 52 | """ |
| 53 | |
| 54 | NProcs = int(NProcs) |
| 55 | MaxLines = int(MaxLines) |
| 56 | |
| 57 | #-- init random seed |
| 58 | random.seed(time.time()) |
| 59 | |
| 60 | |
| 61 | OutFileName = "sm-%s" % InFilePrefix |
| 62 | OutFile = open( OutFileName, "w" ) |
| 63 | |
| 64 | InFileNames = {} |
| 65 | InFiles = {} |
| 66 | InFileFinished = {} |
| 67 | |
| 68 | ProcsIDList = range(NProcs) |
| 69 | |
| 70 | for index in ProcsIDList: |
| 71 | InFileNames[index] = "%s_%d" % (InFilePrefix, index) |
| 72 | InFiles[index] = open( InFileNames[index], "r" ) |
| 73 | InFileFinished[index] = 0 |
| 74 | |
| 75 | nReadLines = 0 |
| 76 | while 1: |
| 77 | |
| 78 | #-- make a list of all input files not finished yet |
| 79 | ListOfNotFinished = [] |
| 80 | for index in ProcsIDList: |
| 81 | if InFileFinished[index] == 0: |
| 82 | ListOfNotFinished.append(index) |
| 83 | |
| 84 | #-- randomly select an input file |
| 85 | lenListOfNotFinished = len(ListOfNotFinished) |
| 86 | if lenListOfNotFinished == 0: |
| 87 | break |
| 88 | elif lenListOfNotFinished == 1: |
| 89 | ProcID = ListOfNotFinished[0] |
| 90 | else: |
| 91 | # at least 2 elements in this list -> pick at random the proc ID |
| 92 | ProcID = ListOfNotFinished[random.randint(0, lenListOfNotFinished - 1)] |
| 93 | |
| 94 | #-- randomly copy 1 to MaxLines lines of it to the output file |
| 95 | nLinesToGet = random.randint( 1, MaxLines ) |
| 96 | try: |
| 97 | for index in range(nLinesToGet): |
| 98 | line = InFiles[ProcID].readline() |
| 99 | if len(line) > 0: |
| 100 | OutFile.write( line ) |
| 101 | nReadLines = nReadLines + 1 |
| 102 | if nReadLines % 10000 == 0: |
| 103 | print "nReadLines", nReadLines, "[last read", nLinesToGet, \ |