Merge pull request livecode#4291 from montegoulding/tokenoffset

Monte Goulding · web-flow · commit f863b0268ee0 · 2016-07-20T16:56:40.000+10:00
Complete tokenOffset implementation
diff --git a/engine/src/funcs.h b/engine/src/funcs.h
@@ -454,6 +454,15 @@ class MCByteOffset : public MCChunkOffset
 	}
 };
 
+class MCTokenOffset : public MCChunkOffset
+{
+public:
+    MCTokenOffset()
+    {
+        delimiter = CT_TOKEN;
+    }
+};
+
 class MCClickChar : public MCConstantFunctionCtxt<MCStringRef, MCInterfaceEvalClickChar>
 {
 public:
diff --git a/engine/src/lextable.cpp b/engine/src/lextable.cpp
@@ -1728,6 +1728,7 @@ LT factor_table[] =
         {"to", TT_TO, PT_TO},
         {"togglehilites", TT_PROPERTY, P_TOGGLE_HILITE},
         {"token", TT_CHUNK, CT_TOKEN},
+        {"tokenoffset", TT_FUNCTION, F_TOKEN_OFFSET},
         {"tokens", TT_CLASS, CT_TOKEN},
         {"tolower", TT_FUNCTION, F_TO_LOWER},
         {"tool", TT_PROPERTY, P_TOOL},
diff --git a/engine/src/newobj.cpp b/engine/src/newobj.cpp
@@ -788,7 +788,9 @@ MCExpression *MCN_new_function(int2 which)
 		return new MCTicks;
 	case F_TIME:
 		return new MCTheTime;
-	case F_TOP_STACK:
+    case F_TOKEN_OFFSET:
+        return new MCTokenOffset;
+    case F_TOP_STACK:
 		return new MCTopStack;
 	case F_TO_LOWER:
 		return new MCToLower;
diff --git a/engine/src/parsedef.h b/engine/src/parsedef.h
@@ -554,6 +554,7 @@ enum Functions {
     F_TIME,
     F_TO_LOWER,
     F_TO_UPPER,
+    F_TOKEN_OFFSET,
     F_TOP_STACK,
     F_TRANSPOSE,
     F_TRUEWORD_OFFSET,
diff --git a/tests/lcs/core/strings/tokenoffset.livecodescript b/tests/lcs/core/strings/tokenoffset.livecodescript
@@ -0,0 +1,28 @@
+﻿script "CoreStringTokenOffset"
+/*
+Copyright (C) 2016 LiveCode Ltd.
+
+This file is part of LiveCode.
+
+LiveCode is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License v3 as published by the Free
+Software Foundation.
+
+LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with LiveCode.  If not see <http://www.gnu.org/licenses/>.  */
+
+on TestTokenOffset
+   local tNeedle, tHaystack
+   put "put" && quote & "foo" & quote & "&" & quote & "bar" & quote && "into tVar" into tHaystack
+
+   local tToken = 1
+   repeat for each token tNeedle in "put foo & bar into tVar"
+      TestAssert "tokenoffset" && tNeedle && "in string", tokenOffset(tNeedle, tHaystack) is tToken
+      add 1 to tToken
+   end repeat
+end TestTokenOffset
