[Bug 15440] lc-compile: Permit string literals with embedded nuls

peter-b · peter-b · commit 95ed67174773 · 2015-11-12T15:15:05.000Z
Internally, lc-compile uses nul-terminated strings.  However, both the
compiled module file format and the LiveCode runtime allow strings to
contain nuls.

Previously, `"\u{0}"` evaluated to the empty string.  This was because
`UnescapeStringLiteral()` faithfully decoded `\u{0}` to the byte
string `{0x00, 0x00}`, which interpreted as a nul-terminated string is
empty.

When outputting a module file, compiler internal strings are converted
to libfoundation `MCString`.  So the nul-terminated strings used
internally by the compiler are never exposed (apart possibly via error
or diagnostic messages).

A trivial fix is therefore to use Modified UTF-8 encoding for
compiler-internal strings, where U+0000 is encoded as `{0xc0, 0x80}`.
diff --git a/docs/lcb/notes/15440.md b/docs/lcb/notes/15440.md
@@ -0,0 +1,4 @@
+# LiveCode Builder Tools
+# lc-compile
+
+# [15440] Preserve nul characters in string literals
diff --git a/tests/lcb/compiler/literals.lcb b/tests/lcb/compiler/literals.lcb
@@ -0,0 +1,31 @@
+/*
+Copyright (C) 2015 LiveCode Ltd.
+
+This file is part of LiveCode.
+
+LiveCode is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License v3 as published by the Free
+Software Foundation.
+
+LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with LiveCode.  If not see <http://www.gnu.org/licenses/>.  */
+
+module com.livecode.compiler.literals.tests
+
+public handler TestLiteralNul()
+	variable tString
+
+	test "single literal nul" when the number of chars in "\u{0}" is 1
+
+	put "a\u{0}b\u{0}c" into tString
+	test diagnostic "literal nuls: actual string length is" && \
+		the number of chars in tString formatted as string
+	test "literal nuls" when the number of chars in tString is 5
+end handler
+
+end module
diff --git a/tests/lcb/stdlib/typeconvert.lcb b/tests/lcb/stdlib/typeconvert.lcb
@@ -0,0 +1,29 @@
+/*
+Copyright (C) 2015 LiveCode Ltd.
+
+This file is part of LiveCode.
+
+LiveCode is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License v3 as published by the Free
+Software Foundation.
+
+LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with LiveCode.  If not see <http://www.gnu.org/licenses/>.  */
+
+module com.livecode.typeconvert.tests
+
+public handler TestSplit()
+	-- Bug 15440
+	variable tString
+	variable tList
+	put "security.selinux\u{0}user.test\u{0}user.uuid" into tString
+	split tString by "\u{0}" into tList
+	test "split (nul)" when the number of elements in tList is 3
+end handler
+
+end module
diff --git a/toolchain/lc-compile/src/literal.c b/toolchain/lc-compile/src/literal.c
@@ -113,7 +113,17 @@ static int char_to_nibble(char p_char, unsigned int *r_nibble)
 
 void append_utf8_char(char *p_string, int *x_index, int p_char)
 {
-    if (p_char < 128)
+	// If the char is NUL (i.e. U+0000) we can't represent it directly
+	// in a nul-terminated string.  However, we *can* use a 2-byte
+	// overlong encoding to represent it safely (i.e. "Modified
+	// UTF-8")
+	if (p_char == 0)
+	{
+		p_string[*x_index]     = 0xc0;
+		p_string[*x_index + 1] = 0x80;
+		(*x_index) += 2;
+	}
+	else if (p_char < 128)
     {
         p_string[*x_index] = p_char;
         (*x_index) += 1;
