Method addWord

JSAT/src/jsat/text/TextDataLoader.java:215–268

Does the work to add a given word to the sparse vector. May not succeed in race conditions when two ore more threads are trying to add a word at the same time. @param word the word to add to the vector @param vec the location to store the word occurrence @param entry the number of times the word oc

(String word, SparseVector vec, Integer value)

     * the word wasn't added due to a race- and should be tried again
     */
    private boolean addWord(String word, SparseVector vec, Integer value)
    {
        Integer indx = wordIndex.get(word);
        if(indx == null)//this word has never been seen before!
        {
            Integer index_for_new_word;
            if((index_for_new_word = wordIndex.putIfAbsent(word, -1)) == null)//I won the race to insert this word into the map
            {
                /*
                * we need to do this increment after words to avoid a race
                * condition where two people incrment currentLength for the
                * same word, as that will throw off other word additions
                * before we can fix the problem
                */
                index_for_new_word = currentLength.getAndIncrement();
                wordIndex.put(word, index_for_new_word);//overwrite with correct value
            }
            if(index_for_new_word < 0)
                return false;
            
            //possible race on tdf as well when two threads found same new word at the same time
            AtomicInteger termCount = new AtomicInteger(0), tmp = null;
            tmp = termDocumentFrequencys.putIfAbsent(index_for_new_word, termCount);
            if(tmp != null)
                termCount = tmp;
            termCount.incrementAndGet();
            
            int newLen = Math.max(index_for_new_word+1, vec.length());
            vec.setLength(newLen);
            vec.set(index_for_new_word, value);
        }
        else//this word has been seen before
        {
            if(indx < 0)
                return false;
            
            AtomicInteger toInc = termDocumentFrequencys.get(indx);
            if (toInc == null)
            {
                //wordIndex and termDocumnetFrequences are not updated
                //atomicly together, so could get index but not have tDF ready
                toInc = termDocumentFrequencys.putIfAbsent(indx, new AtomicInteger(1));
                if (toInc == null)//other person finished adding before we "fixed" via putIfAbsent
                    toInc = termDocumentFrequencys.get(indx);
            }
            toInc.incrementAndGet();
            
            if (vec.length() <= indx)//happens when another thread sees the word first and adds it, then get check and find it- but haven't increased our vector legnth
                vec.setLength(indx+1);
            vec.set(indx, value);
        }
        
        return true;
    }
    
    /**
     * Once all original documents have been added, this method is called so