Function removeDuplicateURL

packages/ui/src/utils/genericHelper.js:840–859

(message)

}

export const removeDuplicateURL = (message) => {
    const visitedURLs = []
    const newSourceDocuments = []

    if (!message.sourceDocuments) return newSourceDocuments

    message.sourceDocuments.forEach((source) => {
        if (source && source.metadata && source.metadata.source) {
            if (isValidURL(source.metadata.source) && !visitedURLs.includes(source.metadata.source)) {
                visitedURLs.push(source.metadata.source)
                newSourceDocuments.push(source)
            } else if (!isValidURL(source.metadata.source)) {
                newSourceDocuments.push(source)
            }
        } else if (source) {
            newSourceDocuments.push(source)
        }
    })
    return newSourceDocuments
}

export const isValidURL = (url) => {
    try {