Function parseUnionPage

lib/core/common.py:1908–1955

Returns resulting items from UNION query inside provided page content >>> parseUnionPage("%sfoo%s%sbar%s" % (kb.chars.start, kb.chars.stop, kb.chars.start, kb.chars.stop)) ['foo', 'bar']

(page)

    return retVal

def parseUnionPage(page):
    """
    Returns resulting items from UNION query inside provided page content

    >>> parseUnionPage("%sfoo%s%sbar%s" % (kb.chars.start, kb.chars.stop, kb.chars.start, kb.chars.stop))
    ['foo', 'bar']
    """

    if page is None:
        return None

    if re.search(r"(?si)\A%s.*%s\Z" % (kb.chars.start, kb.chars.stop), page):
        if len(page) > LARGE_OUTPUT_THRESHOLD:
            warnMsg = "large output detected. This might take a while"
            logger.warning(warnMsg)

        data = BigArray()
        keys = set()

        for match in re.finditer(r"%s(.*?)%s" % (kb.chars.start, kb.chars.stop), page, re.DOTALL | re.IGNORECASE):
            entry = match.group(1)

            if kb.chars.start in entry:
                entry = entry.split(kb.chars.start)[-1]

            if kb.unionDuplicates:
                key = entry.lower()
                if key not in keys:
                    keys.add(key)
                else:
                    continue

            entry = entry.split(kb.chars.delimiter)

            if conf.hexConvert:
                entry = applyFunctionRecursively(entry, decodeDbmsHexValue)

            if kb.safeCharEncode:
                entry = applyFunctionRecursively(entry, safecharencode)

            data.append(entry[0] if len(entry) == 1 else entry)
    else:
        data = page

    if len(data) == 1 and isinstance(data[0], six.string_types):
        data = data[0]

    return data

def parseFilePaths(page):
    """