Some initial sketches of AST

[getify]

Here's a sample snippet of Foi code:

   def x: hello( 40 + two );

Here's what Foi-Toy tokenizes that code as:

{ type: "WHITESPACE", value: "   ", start: 0, end: 2 }
{ type: "KEYWORD", value: "def", start: 3, end: 5 }
{ type: "WHITESPACE", value: " ", start: 6, end: 6 }
{ type: "GENERAL", value: "x", start: 7, end: 7 }
{ type: "COLON", value: ":", start: 8, end: 8 }
{ type: "WHITESPACE", value: " ", start: 9, end: 9 }
{ type: "GENERAL", value: "hello", start: 10, end: 14 }
{ type: "OPEN_PAREN", value: "(", start: 15, end: 15 }
{ type: "WHITESPACE", value: " ", start: 16, end: 16 }
{ type: "NUMBER", value: "40", start: 17, end: 18 }
{ type: "WHITESPACE", value: " ", start: 19, end: 19 }
{ type: "PLUS", value: "+", start: 20, end: 20 }
{ type: "WHITESPACE", value: " ", start: 21, end: 21 }
{ type: "GENERAL", value: "two", start: 22, end: 24 }
{ type: "WHITESPACE", value: " ", start: 25, end: 25 }
{ type: "CLOSE_PAREN", value: ")", start: 26, end: 26 }
{ type: "SEMICOLON", value: ";", start: 27, end: 27 }

And here's the proposed AST for that stream of tokens:

{
  type: "PROGRAM",
  filename: "1.foi",
  path: "./1.foi",
  statements: [
    {
      type: "DEFINE_VAR",
      line: 0,
      col: 3,
      target: {
        type: "IDENTIFIER",
        line: 0,
        col: 7,
        value: "x"
      },
      initializer: {
        type: "PREFIX_CALL",
        line: 0,
        col: 10,
        callee: {
          type: "IDENTIFIER",
          line: 0,
          col: 10,
          value: "hello",
        },
        argList: {
          type: "PAREN_COMMA_LIST",
          args: [
            {
              type: "BINARY_PLUS",
              line: 0,
              col: 17,
              left: {
                type: "NUMBER",
                line: 0,
                col: 17,
                value: "40"
              },
              right: {
                type: "IDENTIFIER",
                line: 0,
                col: 22,
                value: "two"
              },
              syntax: [
                {
                  placeholder: "left"
                },
                {
                  type: "WHITESPACE",
                  line: 0,
                  col: 19,
                  value: " "
                },
                {
                  type: "OPERATOR",
                  line: 0,
                  col: 20,
                  value: "+"
                },
                {
                  type: "WHITESPACE",
                  line: 0,
                  col: 21,
                  value: " "
                },
                {
                  placeholder: "right"
                },
              ]
            }
          ],
          syntax: [
            {
              type: "OPEN_PAREN",
              line: 0,
              col: 15,
              value: "("
            },
            {
              type: "WHITESPACE",
              line: 0,
              col: 16,
              value: " "
            },
            {
              placeholder: "args.0"
            },
            {
              type: "WHITESPACE",
              line: 0,
              col: 25,
              value: " "
            },
            {
              type: "CLOSE_PAREN",
              line: 0,
              col: 26,
              value: ")"
            },
          ]
        },
        syntax: [
          {
            placeholder: "callee"
          },
          {
            placeholder: "argList"
          }
        ]
      },
      syntax: [
        {
          type: "KEYWORD",
          value: "def",
          line: 0,
          col: 3,
        },
        {
          type: "WHITESPACE",
          value: " ",
          line: 0,
          col: 6
        },
        {
          placeholder: "target",
        },
        {
          type: "COLON",
          value: ":",
          line: 0,
          col: 8,
        },
        {
          type: "WHITESPACE",
          value: " ",
          line: 0,
          col: 9
        },
        {
          placeholder: "initializer",
        },
        {
          type: "SEMICOLON",
          value: ";",
          line: 0,
          col: 27
        }
      ],
    }    
  ],
  syntax: [
    {
      type: "WHITESPACE",
      line: 0,
      col: 0,
      value: "   "
    },
    {
      placeholder: "statements.0"
    }
  ]
}

[getify]

serializeSyntax({
  type: "PROGRAM",
  filename: "1.foi",
  path: "./1.foi",
  statements: [
    {
      type: "DEFINE_VAR",
      line: 0,
      col: 3,
      target: {
        type: "IDENTIFIER",
        line: 0,
        col: 7,
        value: "x"
      },
      initializer: {
        type: "PREFIX_CALL",
        line: 0,
        col: 10,
        callee: {
          type: "IDENTIFIER",
          line: 0,
          col: 10,
          value: "hello",
        },
        argList: {
          type: "PAREN_COMMA_LIST",
          args: [
            {
              type: "BINARY_PLUS",
              line: 0,
              col: 17,
              left: {
                type: "NUMBER",
                line: 0,
                col: 17,
                value: "40"
              },
              right: {
                type: "IDENTIFIER",
                line: 0,
                col: 22,
                value: "two"
              },
              syntax: [
                {
                  placeholder: "left"
                },
                {
                  type: "WHITESPACE",
                  line: 0,
                  col: 19,
                  value: " "
                },
                {
                  type: "OPERATOR",
                  line: 0,
                  col: 20,
                  value: "+"
                },
                {
                  type: "WHITESPACE",
                  line: 0,
                  col: 21,
                  value: " "
                },
                {
                  placeholder: "right"
                },
              ]
            }
          ],
          syntax: [
            {
              type: "OPEN_PAREN",
              line: 0,
              col: 15,
              value: "("
            },
            {
              type: "WHITESPACE",
              line: 0,
              col: 16,
              value: " "
            },
            {
              placeholder: "args.0"
            },
            {
              type: "WHITESPACE",
              line: 0,
              col: 25,
              value: " "
            },
            {
              type: "CLOSE_PAREN",
              line: 0,
              col: 26,
              value: ")"
            },
          ]
        },
        syntax: [
          {
            placeholder: "callee"
          },
          {
            placeholder: "argList"
          }
        ]
      },
      syntax: [
        {
          type: "KEYWORD",
          value: "def",
          line: 0,
          col: 3,
        },
        {
          type: "WHITESPACE",
          value: " ",
          line: 0,
          col: 6
        },
        {
          placeholder: "target",
        },
        {
          type: "COLON",
          value: ":",
          line: 0,
          col: 8,
        },
        {
          type: "WHITESPACE",
          value: " ",
          line: 0,
          col: 9
        },
        {
          placeholder: "initializer",
        },
        {
          type: "SEMICOLON",
          value: ";",
          line: 0,
          col: 27
        }
      ],
    }    
  ],
  syntax: [
    {
      type: "WHITESPACE",
      line: 0,
      col: 0,
      value: "   "
    },
    {
      placeholder: "statements.0"
    }
  ]
});


function serializeSyntax(tree) {
	var code = "";
	switch (tree.type) {
		case "PROGRAM":
		case "DEFINE_VAR":
		case "PREFIX_CALL":
		case "PAREN_COMMA_LIST":
		case "BINARY_PLUS": {
			for (let entry of tree.syntax) {
				if (entry.placeholder != null) {
					if (/^\w+\.\d+$/.test(entry.placeholder)) {
						let [,prop,index] = entry.placeholder.match(/^(\w+)\.(\d+)$/);
						code += serializeSyntax(tree[prop][Number(index)]);
					}
					else {
						code += serializeSyntax(tree[entry.placeholder]);
					}
				}
				else {
					code += serializeSyntax(entry);
				}
			}
			break;
		}
		default: {
			if (tree.value != null) {
				code = tree.value;
			}
			else {
				console.log("oops",tree);
			}
		}
	};
	return code;
}