DMS Script Function GetToken returns strange result is the string contains some quote characters. It looks like quote character is sometimes interpreted as a delimiter even if it has not been set in SetTokenizerInput function.
Here is an example showing the problem.
Following script :
Dim strInternalContent As String
DIM delim AS STRING
FUNCTION TEST_GETTOKEN(Str AS STRING, Delim AS STRING) AS INTEGER
Dim strInternalToken As String
DIM i AS INTEGER
print("STRING= "+str)
print("DELIM= "+Delim)
i=1
SetTokenizerInput(str, Delim)
While GetToken(strInternalToken)
print("Token"+STR(i)+" = "+strInternalToken)
i=i+1
Wend
print("")
TEST_GETTOKEN=0
END FUNCTION
' ----------------- MAIN --------------------
strInternalContent = """TEST1"",""TEST2"",""TEST3"",""TEST4"""
delim=","
TEST_GETTOKEN(strInternalContent, delim)
delim=";"
TEST_GETTOKEN(strInternalContent, delim)
strInternalContent = """TEST1"",""TEST2""-""TEST3"",""TEST4"""
delim=","
TEST_GETTOKEN(strInternalContent, delim)
strInternalContent = """TEST1"",""TEST2""-""TEST3"",""TEST4"""
delim="-"
TEST_GETTOKEN(strInternalContent, delim)
strInternalContent = " ""TEST1"", ""TEST2"",""TEST3"",""TEST4"""
delim=","
TEST_GETTOKEN(strInternalContent, delim)
returns these wrong result :
STRING= "TEST1","TEST2","TEST3","TEST4"
DELIM= ,
Token1 = TEST1
Token2 = TEST2
Token3 = TEST3
Token4 = TEST4
STRING= "TEST1","TEST2","TEST3","TEST4"
DELIM= ;
Token1 = TEST1
Token2 = ,"TEST2","TEST3","TEST4"
STRING= "TEST1","TEST2"-"TEST3","TEST4"
DELIM= ,
Token1 = TEST1
Token2 = TEST2
Token3 = -"TEST3"
Token4 = TEST4
STRING= "TEST1","TEST2"-"TEST3","TEST4"
DELIM= -
Token1 = TEST1
Token2 = ,"TEST2"
Token3 = TEST3
Token4 = ,"TEST4"
STRING= "TEST1", "TEST2","TEST3","TEST4"
DELIM= ,
Token1 = "TEST1"
Token2 = "TEST2"
Token3 = TEST3
Token4 = TEST4