Method addOriginalDocument

JSAT/src/jsat/text/TextDataLoader.java:143–202

To be called by the #initialLoad() method. It will take in the text and add a new document vector to the data set. Once all text documents have been loaded, this method should never be called again. This method is thread safe. @param text the text of the document to add @return t

(String text)

     * zero and counts up.
     */
    protected int addOriginalDocument(String text)
    {
        if(noMoreAdding)
            throw new RuntimeException("Initial data set has been finalized");
        StringBuilder localWorkSpace = workSpace.get();
        List<String> localStorageSpace = storageSpace.get();
        Map<String, Integer> localWordCounts = wordCounts.get();
        if(localWorkSpace == null)
        {
            localWorkSpace = new StringBuilder();
            localStorageSpace = new ArrayList<String>();
            localWordCounts = new LinkedHashMap<String, Integer>();
            workSpace.set(localWorkSpace);
            storageSpace.set(localStorageSpace);
            wordCounts.set(localWordCounts);
        }

        localWorkSpace.setLength(0);
        localStorageSpace.clear();
        localWordCounts.clear();
        
        tokenizer.tokenize(text, localWorkSpace, localStorageSpace);
        
        for(String word : localStorageSpace)
        {
            Integer count = localWordCounts.get(word);
            if(count == null)
                localWordCounts.put(word, 1);
            else
                localWordCounts.put(word, count+1);
        }
        
        SparseVector vec = new SparseVector(currentLength.get()+1, localWordCounts.size());//+1 to avoid issues when its length is zero, will be corrected in finalization step anyway
        for(Iterator<Map.Entry<String, Integer>> iter = localWordCounts.entrySet().iterator(); iter.hasNext();)
        {
            Map.Entry<String, Integer> entry = iter.next();
            String word = entry.getKey();
            
            int ms_to_sleep = 1;
            while(!addWord(word, vec, entry.getValue()))//try in a loop, expoential back off 
            {
                try
                {
                    Thread.sleep(ms_to_sleep);
                    ms_to_sleep = Math.min(100, ms_to_sleep*2);
                }
                catch (InterruptedException ex)
                {
                    Logger.getLogger(TextDataLoader.class.getName()).log(Level.SEVERE, null, ex);
                }
            }
        }
        localWordCounts.clear();
        
        synchronized(vectors)
        {
            vectors.add(vec);
            return documents++;