Function sanitizeMessages

lib/edge-runtime/apiResponseSimplification.ts:159–212

(
  messages: ChatGPTMessage[],
  sanitizeUrlsFirst: boolean,
)

}

export function sanitizeMessages(
  messages: ChatGPTMessage[],
  sanitizeUrlsFirst: boolean,
): {
  cleanedMessages: ChatGPTMessage[];
  originalToPlaceholderMap: StringMapping;
} {
  /** Removes ids and urls from messages, replaces them with variables like ID1, URL4
   * and returns the cleaned messages and a lookup table of {value: variable} */
  let idStore: StringMapping = {},
    urlStore: StringMapping = {};
  const cleanedMessages = (
    JSON.parse(JSON.stringify(messages)) as ChatGPTMessage[]
  ).map((message) => {
    if (message.role === "function") {
      try {
        let cleanedObject = JSON.parse(message.content);
        if (sanitizeUrlsFirst) {
          ({ cleanedObject, urlStore } = removeUrlsFromObjs(
            cleanedObject,
            urlStore,
          ));
          ({ cleanedObject, idStore } = removeIdsFromObjs(
            cleanedObject,
            idStore,
          ));
        } else {
          ({ cleanedObject, idStore } = removeIdsFromObjs(
            cleanedObject,
            idStore,
          ));
          ({ cleanedObject, urlStore } = removeUrlsFromObjs(
            cleanedObject,
            urlStore,
          ));
        }
        message.content = JSON.stringify(cleanedObject);
      } catch {
        let cleanedMarkdown;
        ({ cleanedMarkdown, urlStore } = removeUrlsFromMarkdown(
          message.content,
          urlStore,
        ));
        message.content = cleanedMarkdown;
      }
    }
    return message;
  });

  return {
    cleanedMessages,
    originalToPlaceholderMap: { ...idStore, ...urlStore },
  };
}

export function repopulateVariables(
  obj: Json | Json[],