illegal characters are illegal in raw strings, too

tjol · May 14, 2024 · e6b2900 · e6b2900
1 parent 71c0c08
commit e6b2900
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/src/tokenizer.c b/src/tokenizer.c
@@ -570,7 +570,9 @@ static kdl_tokenizer_status _pop_raw_string(kdl_tokenizer* self, kdl_token* dest
             return KDL_TOKENIZER_ERROR;
         }
 
-        if (end_quote_offset != 0 && c == '#') {
+        if (_kdl_is_illegal_char(self->charset, c)) {
+            return KDL_TOKENIZER_ERROR;
+        } else if (end_quote_offset != 0 && c == '#') {
             ++hashes_found;
         } else if (c == '"') {
             end_quote_offset = cur - self->document.data;

diff --git a/tests/kdlv2_test.c b/tests/kdlv2_test.c
@@ -139,6 +139,20 @@ static void test_tokenizer_illegal_codepoints(void)
     ASSERT(kdl_pop_token(tok, &token) == KDL_TOKENIZER_ERROR);
 
     kdl_destroy_tokenizer(tok);
+
+    doc = kdl_str_from_cstr("/* multi-line comments also can't have \x15 */");
+    tok = kdl_create_string_tokenizer(doc);
+
+    ASSERT(kdl_pop_token(tok, &token) == KDL_TOKENIZER_ERROR);
+
+    kdl_destroy_tokenizer(tok);
+
+    doc = kdl_str_from_cstr("###\" multi-line strings must't \x10 \"###");
+    tok = kdl_create_string_tokenizer(doc);
+
+    ASSERT(kdl_pop_token(tok, &token) == KDL_TOKENIZER_ERROR);
+
+    kdl_destroy_tokenizer(tok);
 }
 
 void TEST_MAIN(void)