ClickHouse: improved tokens handling (#450)

* Consider None as ending token

* Add CH identifier syntax test

* Treat None as end

* Run black formatter

sql_metadata/parser.py

@@ -463,7 +463,8 @@ def with_names(self) -> List[str]:
                         while token.next_token and not token.is_with_query_end:
                             token = token.next_token
                         is_end_of_with_block = (
-                            token.next_token_not_comment.normalized
+                            token.next_token_not_comment is None
+                            or token.next_token_not_comment.normalized
                             in WITH_ENDING_KEYWORDS
                         )
                         if is_end_of_with_block:
@@ -504,7 +505,7 @@ def with_queries(self) -> Dict[str, str]:
                 True, value_attribute="is_with_query_end", direction="right"
             )
             query_token = with_start.next_token
-            while query_token != with_end:
+            while query_token is not None and query_token != with_end:
                 current_with_query.append(query_token)
                 query_token = query_token.next_token
             with_query_text = "".join([x.stringified_token for x in current_with_query])

test/test_with_statements.py

@@ -470,3 +470,26 @@ def test_comment_between_with_and_query():
     assert parser.columns == ["column_1", "column_2"]
     assert parser.with_queries == {"cte_1": "SELECT column_1, column_2 FROM table_1"}
     assert parser.tables == ["table_1"]
+
+
+def test_identifier_syntax():
+    """
+    Specific for ClickHouse With indentifier syntax
+
+    https://clickhouse.com/docs/en/sql-reference/statements/select/with#examples
+    """
+
+    query = """
+        WITH
+            '2019-08-01 15:23:00' as ts_upper_bound
+        SELECT EventDate, EventTime
+        FROM hits
+        WHERE
+            EventDate = toDate(ts_upper_bound) AND
+            EventTime <= ts_upper_bound;
+    """
+
+    parser = Parser(query)
+
+    assert parser.tables == ["hits"]
+    assert parser.columns == ["EventDate", "EventTime", "ts_upper_bound"]