Split tokens and determine number of occurrences and store as vectors under id in global samples. */
| 112 | vectors under id in global samples. |
| 113 | */ |
| 114 | static void process_sample(const char *id, char *tokens) |
| 115 | { |
| 116 | // Verify uniqueness of id: |
| 117 | if (all_ids.find(id) != all_ids.end()) { |
| 118 | if (!nowarn) |
| 119 | fprintf(stderr, "(W): Non-unique id %s; sample discarded.\n", id); |
| 120 | return; |
| 121 | } |
| 122 | |
| 123 | Sample s(id); |
| 124 | unsigned num_tokens = 0; |
| 125 | // Per token instance string record all its positions: |
| 126 | typedef vector<unsigned> Positions; |
| 127 | typedef pair<unsigned, Positions> TokenPos; |
| 128 | unordered_map<unsigned, Positions> dict; |
| 129 | |
| 130 | // Split the tokens: |
| 131 | #if 1 |
| 132 | char *p = tokens; |
| 133 | do { |
| 134 | const char *token = p; |
| 135 | while (*p && *p != *delim) |
| 136 | p++; |
| 137 | // Here: *p == '\0' || *p == delim |
| 138 | if (token == p) // empty token |
| 139 | break; |
| 140 | if (*p == *delim) |
| 141 | *p++ = '\0'; |
| 142 | //use token: |
| 143 | |
| 144 | unsigned token_id; // unique id for token string |
| 145 | // Uniquely store all token strings in global vocabulary: |
| 146 | auto it = vocabulary.find(token); |
| 147 | if (it == vocabulary.end()) { // a fresh one |
| 148 | token_id = vocabulary.size(); |
| 149 | vocabulary[token] = token_id; |
| 150 | //tokid2string.push_back(token); NOT USED |
| 151 | } |
| 152 | else |
| 153 | token_id = it->second; |
| 154 | // Locally store all positions for this token: |
| 155 | dict[token_id].push_back(num_tokens); // size of second is frequency |
| 156 | num_tokens++; |
| 157 | |
| 158 | } while (true); |
| 159 | |
| 160 | #else |
| 161 | // strtok is slow |
| 162 | const char *token = strtok(tokens, delim); |
| 163 | while (token) { |
| 164 | unsigned token_id; // unique id for token string |
| 165 | // Uniquely store all token strings in global vocabulary: |
| 166 | auto it = vocabulary.find(token); |
| 167 | if (it == vocabulary.end()) { // a fresh one |
| 168 | token_id = vocabulary.size(); |
| 169 | vocabulary[token] = token_id; |
| 170 | //tokid2string.push_back(token); NOT USED |
| 171 | } |