Function download

pattern/graph/commonsense.py:215–261

Downloads commonsense data from http://nodebox.net/perception. Saves the data as commonsense.csv which can be the input for Commonsense.load().

(path=os.path.join(MODULE, "commonsense.csv"), threshold=50)

#--- NODEBOX.NET/PERCEPTION ------------------------------------------------------------------------

def download(path=os.path.join(MODULE, "commonsense.csv"), threshold=50):
    """ Downloads commonsense data from http://nodebox.net/perception.
        Saves the data as commonsense.csv which can be the input for Commonsense.load().
    """
    s = "http://nodebox.net/perception?format=txt&robots=1"
    s = urlopen(s).read()
    s = s.decode("utf-8")
    s = s.replace("\\'", "'")
    # Group relations by author.
    a = {}
    for r in ([v.strip("'") for v in r.split(", ")] for r in s.split("\n")):
        if len(r) == 7:
            a.setdefault(r[-2], []).append(r)
    # Iterate authors sorted by number of contributions.
    # 1) Authors with 50+ contributions can define new relations and context.
    # 2) Authors with 50- contributions (or robots) can only reinforce existing relations.
    a = sorted(a.items(), cmp=lambda v1, v2: len(v2[1]) - len(v1[1]))
    r = {}
    for author, relations in a:
        if author == "" or author.startswith("robots@"):
            continue
        if len(relations) < threshold:
            break
        # Sort latest-first (we prefer more recent relation types).
        relations = sorted(relations, cmp=lambda r1, r2: r1[-1] > r2[-1])
        # 1) Define new relations.
        for concept1, relation, concept2, context, weight, author, date in relations:
            id = (concept1, relation, concept2)
            if id not in r:
                r[id] = [None, 0]
            if r[id][0] is None and context is not None:
                r[id][0] = context
    for author, relations in a:
        # 2) Reinforce existing relations.
        for concept1, relation, concept2, context, weight, author, date in relations:
            id = (concept1, relation, concept2)
            if id in r:
                r[id][1] += int(weight)
    # Export CSV-file.
    s = []
    for (concept1, relation, concept2), (context, weight) in r.items():
        s.append("\"%s\",\"%s\",\"%s\",\"%s\",%s" % (
            concept1, relation, concept2, context, weight))
    f = open(path, "w")
    f.write(BOM_UTF8)
    f.write("\n".join(s).encode("utf-8"))
    f.close()
    
def json():
    """ Returns a JSON-string with the data from commonsense.csv.