(p, rank=0, world_size=1)
| 79 | |
| 80 | |
| 81 | def read_from_file(p, rank=0, world_size=1): |
| 82 | with open(p, "r") as fin: |
| 83 | cnt = -1 |
| 84 | for l in fin: |
| 85 | cnt += 1 |
| 86 | if cnt % world_size != rank: |
| 87 | continue |
| 88 | yield l.strip(), cnt |
| 89 | |
| 90 | |
| 91 | def get_unique_embedder_keys_from_conditioner(conditioner): |