| 198 | const nonExternal = profiles.filter((p) => p.is_external === false); |
| 199 | |
| 200 | const fetchGroup = async ( |
| 201 | group: IClickhouseProfile[], |
| 202 | withDateFilter: boolean |
| 203 | ) => { |
| 204 | for (const chunk of this.chunks(group, this.fetchChunkSize)) { |
| 205 | const tuples = chunk |
| 206 | .map( |
| 207 | (p) => |
| 208 | `(${sqlstring.escape(String(p.id))}, ${sqlstring.escape(p.project_id)})` |
| 209 | ) |
| 210 | .join(', '); |
| 211 | try { |
| 212 | // Table alias `p` is required: without it, WHERE's `last_seen_at` |
| 213 | // resolves to the SELECT-list aggregate alias `max(last_seen_at) AS |
| 214 | // last_seen_at`, which is an aggregate function and illegal in WHERE |
| 215 | // (CH ILLEGAL_AGGREGATION). Qualifying with `p.` bypasses the alias |
| 216 | // lookup and binds to the raw column. |
| 217 | const rows = await chQuery<IClickhouseProfile>( |
| 218 | `SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS} |
| 219 | FROM ${TABLE_NAMES.profiles} AS p |
| 220 | WHERE (p.id, p.project_id) IN (${tuples}) |
| 221 | ${withDateFilter ? 'AND p.last_seen_at > now() - INTERVAL 2 DAY' : ''} |
| 222 | GROUP BY p.id, p.project_id` |
| 223 | ); |
| 224 | for (const row of rows) { |
| 225 | result.set(`${row.project_id}:${row.id}`, row); |
| 226 | } |
| 227 | } catch (error) { |
| 228 | this.logger.warn( |
| 229 | { err: error, chunkSize: chunk.length }, |
| 230 | 'Failed to batch fetch profiles from Clickhouse, proceeding without existing data' |
| 231 | ); |
| 232 | } |
| 233 | } |
| 234 | }; |
| 235 | |
| 236 | await Promise.all([ |
| 237 | fetchGroup(external, false), |