Minimalistic tokenize using string.find()

accepts a single character as separator.

returns tokens as an array of strings.

        function tokenize(str,sep)
        {
                local start = 0;
                local idx;
                local charsep = sep.tochar();
                local ret = []
                while(idx = str.find(charsep,start)) {
                        ret.append(str.slice(start,idx));
                        start = idx+1;
                }
                if(start != str.len()) {
                        ret.append(str.slice(start,str.len()));
                }
                return ret;
        }


        local str = "1,2,3,123123,12312,2";


        foreach( i,val in tokenize(str,','))
        {
                ::print(val + "\n");
        }

Tokenize using a regular expression

accepts a string with multiple valid separators.

returns tokens as an array of strings.

        function rex_tokenize(str,seps)
        {
                local exp = "["
                //escapes separators
                foreach(char in seps) {
                        exp += @"\"; exp += char.tochar();
                }
                exp += "]";
                //
                local rex = regexp(exp);
                local start = 0;
                local idx;
                local ret = []
                while(idx = rex.search(str,start)) {
                        ret.append(str.slice(start,idx.begin));
                        start = idx.begin+1;
                }
                if(start != str.len()) {
                        ret.append(str.slice(start,str.len()));
                }
                return ret;
        }


        local str = "1,2,3,123123;12312.";
        foreach( i,val in rex_tokenize(str,",;."))
        {
                ::print(val + "\n");
        }