Added nicely formated multi-line strings.

src/tokenize.cpp

@@ -11,6 +11,48 @@ namespace sorth::internal
     {
 
 
+        bool is_whitespace(char next)
+        {
+            return    (next == ' ')
+                    || (next == '\t')
+                    || (next == '\n');
+        }
+
+        void skip_whitespace(SourceBuffer& source_code)
+        {
+            char next = source_code.peek_next();
+
+            while (source_code && is_whitespace(next))
+            {
+                source_code.next();
+                next = source_code.peek_next();
+            }
+        }
+
+
+        void skip_whitespace_until_column(SourceBuffer& source_code, size_t target_column)
+        {
+            char next = source_code.peek_next();
+
+            while (   source_code
+                   && is_whitespace(next)
+                   && (source_code.current_location().get_column() < target_column))
+            {
+                source_code.next();
+                next = source_code.peek_next();
+            }
+        }
+
+
+        void append_newlines(std::string& text, size_t count)
+        {
+            for (size_t i = 0; i < count; ++i)
+            {
+                text += '\n';
+            }
+        }
+
+
         char process_escape_literal(SourceBuffer& source_code)
         {
             char next = source_code.next();
@@ -64,33 +106,110 @@ namespace sorth::internal
         }
 
 
-        std::string process_string(SourceBuffer& source_code)
+        std::string process_multi_line_string(const Location& start, SourceBuffer& source_code)
         {
-            Location start = source_code.current_location();
+            // Extract the *.
+            source_code.next();
 
-            std::string new_string;
+            // Extract first whitespace until we get to first real text, and record the colum we are
+            // at now.
+            skip_whitespace(source_code);
+
+            auto target_column = source_code.current_location().get_column();
             char next = 0;
 
-            source_code.next();
+            std::string new_string;
 
             while (source_code)
             {
                 next = source_code.next();
 
-                if (next == '"')
+                // Found an asterisk, check to see if the next char is a quote.  If it is, we're
+                // done with this string.
+                if (next == '*')
                 {
-                    break;
+                    if (source_code.peek_next() == '"')
+                    {
+                        next = source_code.next();
+                        break;
+                    }
+                    else
+                    {
+                        new_string += next;
+                    }
                 }
-
-                if (next == '\\')
+                else if (next == '\\')
+                {
+                    // Process the escaped character.
+                    new_string += process_escape_literal(source_code);
+                }
+                else if (next == '\n')
                 {
-                    next = process_escape_literal(source_code);
+                    new_string += next;
 
+                    // We're on a new line, so get rid of the whitespace until we either find text
+                    // or reach the column we're looking for.  Any whitespace after that column will
+                    // be included in the string.  Any skiped new lines should be added to the
+                    // string.
+                    auto start_line = source_code.current_location().get_line();
+
+                    skip_whitespace_until_column(source_code, target_column);
+
+                    auto current_line = source_code.current_location().get_line();
+
+                    if (current_line > (start_line))
+                    {
+                        append_newlines(new_string, current_line - (start_line));
+                    }
+                }
+                else
+                {
+                    // No special processing needed for this character.
+                    new_string += next;
                 }
-                new_string += next;
             }
 
-            throw_error_if(next != '"', start, "Missing end of string literal.");
+            return new_string;
+        }
+
+
+        std::string process_string(SourceBuffer& source_code)
+        {
+            Location start = source_code.current_location();
+
+            std::string new_string;
+
+            source_code.next();
+
+            if (source_code.peek_next() == '*')
+            {
+                new_string = process_multi_line_string(start, source_code);
+            }
+            else
+            {
+                char next = 0;
+
+                while (source_code)
+                {
+                    next = source_code.next();
+
+                    if (next == '"')
+                    {
+                        break;
+                    }
+
+                    throw_error_if(next == '\n', start, "Unexpected new line in string literal.");
+
+                    if (next == '\\')
+                    {
+                        next = process_escape_literal(source_code);
+
+                    }
+                    new_string += next;
+                }
+
+                throw_error_if(next != '"', start, "Missing end of string literal.");
+            }
 
             return new_string;
         }
@@ -125,7 +244,7 @@ namespace sorth::internal
     }
 
 
-     TokenList tokenize(SourceBuffer& source_code)
+    TokenList tokenize(SourceBuffer& source_code)
     {
         TokenList tokens;
 
@@ -144,13 +263,6 @@ namespace sorth::internal
                 return new_string;
             };
 
-        auto is_whitespace = [](char next) -> bool
-            {
-                return    (next == ' ')
-                       || (next == '\t')
-                       || (next == '\n');
-            };
-
         auto is_numeric = [](const std::string& text) -> bool
             {
                 if ((text[0] >= '0') && (text[0] <= '9'))
@@ -167,20 +279,9 @@ namespace sorth::internal
                 return false;
             };
 
-        auto skip_whitespace = [&]()
-            {
-                char next = source_code.peek_next();
-
-                while (source_code && (is_whitespace(next)))
-                {
-                    source_code.next();
-                    next = source_code.peek_next();
-                }
-            };
-
         while (source_code)
         {
-            skip_whitespace();
+            skip_whitespace(source_code);
 
             if (!source_code)
             {

tests/05_test_strings.f

@@ -40,3 +40,17 @@
 "Hello world!" dup string.size@ --
 swap over 1 rot string.remove "." + .cr
 "Cliché but... " swap 0 swap string.[]! .cr
+
+cr
+
+"*
+    This is a multi line string.
+    We're including extra whitespace.
+
+        * This line is indented!
+
+This line is different!
+
+    All done!
+*"
+.