Re: For the idiot Paul The Troll

For anyone who cares my tokenizer had a minor defect whereby it would
only accept random access iterators when it was meant to accept forward
iterators. Here is the fixed function:

    template <typename FwdIter1, typename FwdIter2, typename ResultContainer>
    inline FwdIter1 tokens(FwdIter1 aFirst, FwdIter1 aLast, FwdIter2
aDelimeterFirst, FwdIter2 aDelimiterLast, ResultContainer& aTokens,
std::size_t aMaxTokens = 0, bool aSkipEmptyTokens = true, bool
aDelimeterIsSubsequence = false)
    {
        if (aFirst == aLast)
            return aFirst;
        typedef typename ResultContainer::value_type value_type;
        if (aDelimeterFirst == aDelimiterLast)
        {
            aTokens.push_back(value_type(aFirst, aLast));
            return aLast;
        }
        FwdIter1 b = aFirst;
        FwdIter1 e = aDelimeterIsSubsequence ? std::search(b, aLast,
aDelimeterFirst, aDelimiterLast) : std::find_first_of(b, aLast,
aDelimeterFirst, aDelimiterLast);
        std::size_t tokens = 0;
        while(e != aLast && (aMaxTokens == 0 || tokens < aMaxTokens))
        {
            if (b == e && !aSkipEmptyTokens)
            {
                aTokens.push_back(value_type(b, b));
                ++tokens;
            }
            else if (b != e)
            {
                aTokens.push_back(value_type(b, e));
                ++tokens;
            }
            b = e;
            for (std::size_t i = aDelimeterIsSubsequence ?
std::distance(aDelimeterFirst, aDelimiterLast) : 1; i > 0; --i)
                ++b;
            e = aDelimeterIsSubsequence ? std::search(b, aLast, aDelimeterFirst,
aDelimiterLast) : std::find_first_of(b, aLast, aDelimeterFirst,
aDelimiterLast);
        }
        if (b != e && (aMaxTokens == 0 || tokens < aMaxTokens))
        {
            aTokens.push_back(value_type(b, e));
            b = e;
        }
        return b;
    }

    template <typename CharT, typename Traits, typename Alloc, typename
ResultContainer>
    inline void tokens(const std::basic_string<CharT, Traits, Alloc>&
aLine, const std::basic_string<CharT, Traits, Alloc>& aDelimeter,
ResultContainer& aTokens, std::size_t aMaxTokens = 0, bool
aSkipEmptyTokens = true, bool aDelimeterIsSubsequence = false)
    {
        tokens(aLine.begin(), aLine.end(), aDelimeter.begin(),
aDelimeter.end(), aTokens, aMaxTokens, aSkipEmptyTokens,
aDelimeterIsSubsequence);
    }

The change is to the first if statement replacing "aFirst >= aLast" with
"aFirst == aLast".

Two examples of use (the second version is potentially more efficient):

int main()
{
    typedef std::vector<std::string> words_t;
    words_t words;
    std::string test = "one two three";
    std::string delim = " ";
    tokens(test, delim, words);
    for (words_t::const_iterator i = words.begin(); i != words.end(); ++i)
        std::cout << "[" << *i << "] ";
}

int main()
{
    typedef std::vector<std::pair<std::string::const_iterator,
std::string::const_iterator> > words_t;
    words_t words;
    std::string test = "one two three";
    std::string delim = " ";
    tokens(test, delim, words);
    for (words_t::const_iterator i = words.begin(); i != words.end(); ++i)
    {
        std::cout << "[";
        std::copy(i->first, i->second, std::ostream_iterator<char>(std::cout));
        std::cout << "] ";
    }
}

HTH somebody.

/Leigh