From b558db01dfa41d7bef432ecff6541dd939df9e97 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:08:40 +0000 Subject: [PATCH 01/12] Add support for numeric separators --- grammars/python.cson | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 71cb6d4..04c0dd8 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -38,51 +38,51 @@ 'include': '#language_variables' } { - 'match': '\\b(?i:(0x\\h*)L)' + 'match': '\\b(?i:(0x\\h(?:_?\\h)*)L)' 'name': 'constant.numeric.integer.long.hexadecimal.python' } { - 'match': '\\b(?i:(0x\\h*))' + 'match': '\\b(?i:(0x\\h(?:_?\\h)*)))' 'name': 'constant.numeric.integer.hexadecimal.python' } { - 'match': '\\b(?i:(0o?[0-7]+)L)' + 'match': '\\b(?i:(0o?[0-7]((?:_?[0-7])*)))L)' 'name': 'constant.numeric.integer.long.octal.python' } { - 'match': '\\b(?i:(0o?[0-7]+))' + 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*)))' 'name': 'constant.numeric.integer.octal.python' } { - 'match': '\\b(?i:(0b[01]+)L)', + 'match': '\\b(?i:(0b[01](?:_?[01])*))L)' 'name': 'constant.numeric.integer.long.binary.python' } { - 'match': '\\b(?i:(0b[01]+))', + 'match': '\\b(?i:(0b[01](?:_?[01])*)))' 'name': 'constant.numeric.integer.binary.python' } { - 'match': '\\b(?i:(((\\d+(\\.(?=[^a-zA-Z_])\\d*)?|(?<=[^0-9a-zA-Z_])\\.\\d+)(e[\\-\\+]?\\d+)?))J)' + 'match': '\\b(?i:(((\\d(?:_?\\d)*(\\.(?=[^a-zA-Z_])(\\d(?:_?\\d)*)?)?|(?<=[^0-9a-zA-Z_])\\.\\d(?:_?\\d)*)(e[\\-\\+]?\\d(?:_?\\d)*)?))J)' 'name': 'constant.numeric.complex.python' } { - 'match': '\\b(?i:(\\d+\\.\\d*(e[\\-\\+]?\\d+)?))(?=[^a-zA-Z_])' + 'match': '\\b(?i:(\\d(?:_?\\d)*\\.(\\d(?:_?\\d)*)?(e[\\-\\+]?\\d(?:_?\\d)*)?))(?=[^a-zA-Z_])' 'name': 'constant.numeric.float.python' } { - 'match': '(?<=[^0-9a-zA-Z_])(?i:(\\.\\d+(e[\\-\\+]?\\d+)?))' + 'match': '(?<=[^0-9a-zA-Z_])(?i:(\\.\\d(?:_?\\d)*(e[\\-\\+]?\\d(?:_?\\d)*)?))' 'name': 'constant.numeric.float.python' } { - 'match': '\\b(?i:(\\d+e[\\-\\+]?\\d+))' + 'match': '\\b(?i:(\\d(?:_?\\d)*e[\\-\\+]?\\d(?:_?\\d)*))' 'name': 'constant.numeric.float.python' } { - 'match': '\\b(?i:([1-9]+[0-9]*|0)L)' + 'match': '\\b(?i:([1-9](?:_?\\d)*|0)L)' 'name': 'constant.numeric.integer.long.decimal.python' } { - 'match': '\\b([1-9]+[0-9]*|0)' + 'match': '\\b(?:[1-9](?:_?\\d)*|0)' 'name': 'constant.numeric.integer.decimal.python' } { From e8ad319d30470ba26bca2b103d4ef9f249a809c3 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:08:49 +0000 Subject: [PATCH 02/12] Add specs for numbers --- spec/python-spec.coffee | 245 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 423f8c1..247d97d 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -31,6 +31,251 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: 'yield from', scopes: ['source.python', 'keyword.control.statement.python'] + describe "numbers", -> + describe "integers", -> + describe "binary", -> + it "tokenizes binary numbers", -> + {tokens} = grammar.tokenizeLine '0b01' + + expect(tokens[0]).toEqual value: '0b01', scopes: ['source.python', 'constant.numeric.integer.binary.python'] + + it "tokenizes binary numbers with separators", -> + {tokens} = grammar.tokenizeLine '0b0000_0100_1101' + + expect(tokens[0]).toEqual value: '0b0000_0100_1101', scopes: ['source.python', 'constant.numeric.integer.binary.python'] + + it "does not tokenize binary numbers with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '0b0000__0100_1101' + + expect(tokens[0]).toEqual value: '0b0000__0100_1101', scopes: ['source.python'] + + it "does not tokenize binary numbers with a leading separator", -> + {tokens} = grammar.tokenizeLine '0b_0000_0100_1101' + + expect(tokens[0]).toEqual value: '0b_0000_0100_1101', scopes: ['source.python'] + + it "does not tokenize binary numbers with a trailing separator", -> + {tokens} = grammar.tokenizeLine '0b0000_0100_1101_' + + expect(tokens[0]).toEqual value: '0b0000_0100_1101_', scopes: ['source.python'] + + it "does not tokenize non-binary numbers", -> + {tokens} = grammar.tokenizeLine '0b7' + + expect(tokens[0]).toEqual value: '0b7', scopes: ['source.python'] + + describe "octal", -> + it "tokenizes octal numbers", -> + {tokens} = grammar.tokenizeLine '0o72' + + expect(tokens[0]).toEqual value: '0o72', scopes: ['source.python', 'constant.numeric.integer.octal.python'] + + it "tokenizes octal numbers with separators", -> + {tokens} = grammar.tokenizeLine '0o236_777_644' + + expect(tokens[0]).toEqual value: '0b236_777_644', scopes: ['source.python', 'constant.numeric.integer.octal.python'] + + it "does not tokenize octal numbers with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '0b236__777_644' + + expect(tokens[0]).toEqual value: '0b236__777_644', scopes: ['source.python'] + + it "does not tokenize octal numbers with a leading separator", -> + {tokens} = grammar.tokenizeLine '0b_236_777_644' + + expect(tokens[0]).toEqual value: '0b_236_777_644', scopes: ['source.python'] + + it "does not tokenize octal numbers with a trailing separator", -> + {tokens} = grammar.tokenizeLine '0b236_777_644_' + + expect(tokens[0]).toEqual value: '0b236_777_644_', scopes: ['source.python'] + + it "does not tokenize non-octal numbers", -> + {tokens} = grammar.tokenizeLine '0o8' + + expect(tokens[0]).toEqual value: '0o8', scopes: ['source.python'] + + describe "hexadecimal", -> + it "tokenizes hexadecimal numbers", -> + {tokens} = grammar.tokenizeLine '0xe3' + + expect(tokens[0]).toEqual value: '0xe3', scopes: ['source.python', 'constant.numeric.integer.hexadecimal.python'] + + it "tokenizes hexadecimal numbers with separators", -> + {tokens} = grammar.tokenizeLine '0bdead_beef_1337' + + expect(tokens[0]).toEqual value: '0bdead_beef_1337', scopes: ['source.python', 'constant.numeric.integer.hexadecimal.python'] + + it "does not tokenize hexadecimal numbers with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '0bdead__beef_1337' + + expect(tokens[0]).toEqual value: '0bdead__beef_1337', scopes: ['source.python'] + + it "does not tokenize hexadecimal numbers with a leading separator", -> + {tokens} = grammar.tokenizeLine '0b_dead_beef_1337' + + expect(tokens[0]).toEqual value: '0b_dead_beef_1337', scopes: ['source.python'] + + it "does not tokenize hexadecimal numbers with a trailing separator", -> + {tokens} = grammar.tokenizeLine '0bdead_beef_1337_' + + expect(tokens[0]).toEqual value: '0bdead_beef_1337_', scopes: ['source.python'] + + it "does not tokenize non-hexadecimal numbers", -> + {tokens} = grammar.tokenizeLine '0xf' + + expect(tokens[0]).toEqual value: '0xf', scopes: ['source.python'] + + describe "decimal", -> + it "tokenizes decimal numbers", -> + {tokens} = grammar.tokenizeLine '92' + + expect(tokens[0]).toEqual value: '92', scopes: ['source.python', 'constant.numeric.integer.octal.python'] + + it "tokenizes decimal numbers with separators", -> + {tokens} = grammar.tokenizeLine '123_456_789' + + expect(tokens[0]).toEqual value: '123_456_789', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] + + it "does not tokenize decimal numbers with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '123__456_789' + + expect(tokens[0]).toEqual value: '123__456_789', scopes: ['source.python'] + + it "does not tokenize decimal numbers with a leading separator", -> + {tokens} = grammar.tokenizeLine '_123_456_789' + + expect(tokens[0]).toEqual value: '_123_456_789', scopes: ['source.python'] + + it "does not tokenize decimal numbers with a trailing separator", -> + {tokens} = grammar.tokenizeLine '123_456_789_' + + expect(tokens[0]).toEqual value: '123_456_789_', scopes: ['source.python'] + + it "does not tokenize non-decimal numbers", -> + {tokens} = grammar.tokenizeLine '3f' + + expect(tokens[0]).toEqual value: '3f', scopes: ['source.python'] + + describe "floats", -> + describe "decimal", -> + it "tokenizes decimal floats", -> + {tokens} = grammar.tokenizeLine '32.68' + + expect(tokens[0]).toEqual value: '32.68', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes decimal floats without a fractional portion", -> + {tokens} = grammar.tokenizeLine '32.' + + expect(tokens[0]).toEqual value: '32.', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes decimal floats without a whole portion", -> + {tokens} = grammar.tokenizeLine '.68' + + expect(tokens[0]).toEqual value: '.68', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes decimal floats with separators", -> + {tokens} = grammar.tokenizeLine '373_827.639_817' + + expect(tokens[0]).toEqual value: '373_827.639_817', scopes: ['source.python', 'constant.numeric.float.python'] + + it "does not tokenize decimal floats with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '373__827.639_817' + + expect(tokens[0]).toEqual value: '373__827.639_817', scopes: ['source.python'] + + it "does not tokenize decimal floats with a leading separator", -> + {tokens} = grammar.tokenizeLine '373_827._639_817' + + expect(tokens[0]).toEqual value: '373_827._639_817', scopes: ['source.python'] + + it "does not tokenize decimal floats with a trailing separator", -> + {tokens} = grammar.tokenizeLine '373_827_.639_817' + + expect(tokens[0]).toEqual value: '373_827_.639_817', scopes: ['source.python'] + + describe "exponential", -> + it "tokenizes exponential floats", -> + {tokens} = grammar.tokenizeLine '97e43' + + expect(tokens[0]).toEqual value: '97e43', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes exponential floats with an optional sign", -> + {tokens} = grammar.tokenizeLine '97e+43' + + expect(tokens[0]).toEqual value: '97e+43', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes exponential decimal floats", -> + {tokens} = grammar.tokenizeLine '97.22e-43' + + expect(tokens[0]).toEqual value: '97e+43', scopes: ['source.python', 'constant.numeric.float.python'] + + it "tokenizes exponential floats with separators", -> + {tokens} = grammar.tokenizeLine '62_2e-2_839' + + expect(tokens[0]).toEqual value: '62_2e-2_839', scopes: ['source.python', 'constant.numeric.float.python'] + + it "does not tokenize exponential floats with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '62_2e-2__839' + + expect(tokens[0]).toEqual value: '62_2e-2__839', scopes: ['source.python'] + + it "does not tokenize exponential floats with a leading separator", -> + {tokens} = grammar.tokenizeLine '_62_2e-2_839' + + expect(tokens[0]).toEqual value: '_62_2e-2_839', scopes: ['source.python'] + + it "does not tokenize exponential floats with a trailing separator", -> + {tokens} = grammar.tokenizeLine '62_2e-2_839_' + + expect(tokens[0]).toEqual value: '62_2e-2_839_', scopes: ['source.python'] + + it "does not tokenize exponential floats with a fractional exponent", -> + {tokens} = grammar.tokenizeLine '97.22e-43.3' + + expect(tokens[0]).toEqual value: '97.22e-43.3', scopes: ['source.python'] + + describe "complex", -> + it "tokenizes complex floats", -> + {tokens} = grammar.tokenizeLine '63j' + + expect(tokens[0]).toEqual value: '63j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + + it "tokenizes complex decimal floats", -> + {tokens} = grammar.tokenizeLine '63.72j' + + expect(tokens[0]).toEqual value: '63.72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + + it "tokenizes complex exponential floats", -> + {tokens} = grammar.tokenizeLine '63e-72j' + + expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + + it "tokenizes complex decimal exponential floats", -> + {tokens} = grammar.tokenizeLine '63.22e-72j' + + expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + + it "tokenizes complex floats with separators", -> + {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839j' + + expect(tokens[0]).toEqual value: '62_2.83_7e-2_839j', scopes: ['source.python', 'constant.numeric.float.python'] + + it "does not tokenize complex floats with more than one consecutive separator", -> + {tokens} = grammar.tokenizeLine '62_2.83_7e-2__839j' + + expect(tokens[0]).toEqual value: '62_2.83_7e-2__839j', scopes: ['source.python'] + + it "does not tokenize complex floats with a leading separator", -> + {tokens} = grammar.tokenizeLine '62_2._83_7e-2_839j' + + expect(tokens[0]).toEqual value: '_62_2._83_7e-2_839', scopes: ['source.python'] + + it "does not tokenize complex floats with a trailing separator", -> + {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839_j' + + expect(tokens[0]).toEqual value: '62_2.83_7e-2_839_j', scopes: ['source.python'] + it "tokenizes multi-line strings", -> tokens = grammar.tokenizeLines('"1\\\n2"') From 3561483d5ae1af784bcd7eae49d751b43bf2abbc Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:25:33 +0000 Subject: [PATCH 03/12] Remove extra parentheses --- grammars/python.cson | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 04c0dd8..d2216d7 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -42,7 +42,7 @@ 'name': 'constant.numeric.integer.long.hexadecimal.python' } { - 'match': '\\b(?i:(0x\\h(?:_?\\h)*)))' + 'match': '\\b(?i:(0x\\h(?:_?\\h)*))' 'name': 'constant.numeric.integer.hexadecimal.python' } { @@ -50,15 +50,15 @@ 'name': 'constant.numeric.integer.long.octal.python' } { - 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*)))' + 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*))' 'name': 'constant.numeric.integer.octal.python' } { - 'match': '\\b(?i:(0b[01](?:_?[01])*))L)' + 'match': '\\b(?i:(0b[01](?:_?[01])*)L)' 'name': 'constant.numeric.integer.long.binary.python' } { - 'match': '\\b(?i:(0b[01](?:_?[01])*)))' + 'match': '\\b(?i:(0b[01](?:_?[01])*))' 'name': 'constant.numeric.integer.binary.python' } { From 20e3fb668d0c2cb55b24c20da9bf2b1e2567f2bc Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:32:51 +0000 Subject: [PATCH 04/12] Another fix --- grammars/python.cson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grammars/python.cson b/grammars/python.cson index d2216d7..f2189e2 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -46,7 +46,7 @@ 'name': 'constant.numeric.integer.hexadecimal.python' } { - 'match': '\\b(?i:(0o?[0-7]((?:_?[0-7])*)))L)' + 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*)L)' 'name': 'constant.numeric.integer.long.octal.python' } { From 819145cf65b4b52b13fee30cebf9fec667e2bd33 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:36:34 +0000 Subject: [PATCH 05/12] Add word separators --- grammars/python.cson | 24 ++++++++++++------------ spec/python-spec.coffee | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index f2189e2..00b1910 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -38,51 +38,51 @@ 'include': '#language_variables' } { - 'match': '\\b(?i:(0x\\h(?:_?\\h)*)L)' + 'match': '\\b(?i:(0x\\h(?:_?\\h)*)L)\\b' 'name': 'constant.numeric.integer.long.hexadecimal.python' } { - 'match': '\\b(?i:(0x\\h(?:_?\\h)*))' + 'match': '\\b(?i:(0x\\h(?:_?\\h)*))\\b' 'name': 'constant.numeric.integer.hexadecimal.python' } { - 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*)L)' + 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*)L)\\b' 'name': 'constant.numeric.integer.long.octal.python' } { - 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*))' + 'match': '\\b(?i:(0o?[0-7](?:_?[0-7])*))\\b' 'name': 'constant.numeric.integer.octal.python' } { - 'match': '\\b(?i:(0b[01](?:_?[01])*)L)' + 'match': '\\b(?i:(0b[01](?:_?[01])*)L)\\b' 'name': 'constant.numeric.integer.long.binary.python' } { - 'match': '\\b(?i:(0b[01](?:_?[01])*))' + 'match': '\\b(?i:(0b[01](?:_?[01])*))\\b' 'name': 'constant.numeric.integer.binary.python' } { - 'match': '\\b(?i:(((\\d(?:_?\\d)*(\\.(?=[^a-zA-Z_])(\\d(?:_?\\d)*)?)?|(?<=[^0-9a-zA-Z_])\\.\\d(?:_?\\d)*)(e[\\-\\+]?\\d(?:_?\\d)*)?))J)' + 'match': '\\b(?i:(((\\d(?:_?\\d)*(\\.(\\d(?:_?\\d)*)?)?|(?<=[^0-9a-zA-Z_])\\.\\d(?:_?\\d)*)(e[\\-\\+]?\\d(?:_?\\d)*)?))J)\\b' 'name': 'constant.numeric.complex.python' } { - 'match': '\\b(?i:(\\d(?:_?\\d)*\\.(\\d(?:_?\\d)*)?(e[\\-\\+]?\\d(?:_?\\d)*)?))(?=[^a-zA-Z_])' + 'match': '\\b(?i:(\\d(?:_?\\d)*\\.(\\d(?:_?\\d)*)?(e[\\-\\+]?\\d(?:_?\\d)*)?))\\b' 'name': 'constant.numeric.float.python' } { - 'match': '(?<=[^0-9a-zA-Z_])(?i:(\\.\\d(?:_?\\d)*(e[\\-\\+]?\\d(?:_?\\d)*)?))' + 'match': '(?<=[^0-9a-zA-Z_])(?i:(\\.\\d(?:_?\\d)*(e[\\-\\+]?\\d(?:_?\\d)*)?))\\b' 'name': 'constant.numeric.float.python' } { - 'match': '\\b(?i:(\\d(?:_?\\d)*e[\\-\\+]?\\d(?:_?\\d)*))' + 'match': '\\b(?i:(\\d(?:_?\\d)*e[\\-\\+]?\\d(?:_?\\d)*))\\b' 'name': 'constant.numeric.float.python' } { - 'match': '\\b(?i:([1-9](?:_?\\d)*|0)L)' + 'match': '\\b(?i:([1-9](?:_?\\d)*|0)L)\\b' 'name': 'constant.numeric.integer.long.decimal.python' } { - 'match': '\\b(?:[1-9](?:_?\\d)*|0)' + 'match': '\\b(?:[1-9](?:_?\\d)*|0)\\b' 'name': 'constant.numeric.integer.decimal.python' } { diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 247d97d..518c9e1 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -252,9 +252,9 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] it "tokenizes complex decimal exponential floats", -> - {tokens} = grammar.tokenizeLine '63.22e-72j' + {tokens} = grammar.tokenizeLine '63.e-72j' - expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + expect(tokens[0]).toEqual value: '63.e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] it "tokenizes complex floats with separators", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839j' From a96f2ea135b2290d74c38f957bd71c6546bd0601 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:45:33 +0000 Subject: [PATCH 06/12] Fix spec typos --- spec/python-spec.coffee | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 518c9e1..f3ae4d3 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -76,19 +76,19 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: '0b236_777_644', scopes: ['source.python', 'constant.numeric.integer.octal.python'] it "does not tokenize octal numbers with more than one consecutive separator", -> - {tokens} = grammar.tokenizeLine '0b236__777_644' + {tokens} = grammar.tokenizeLine '0o236__777_644' - expect(tokens[0]).toEqual value: '0b236__777_644', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0o236__777_644', scopes: ['source.python'] it "does not tokenize octal numbers with a leading separator", -> - {tokens} = grammar.tokenizeLine '0b_236_777_644' + {tokens} = grammar.tokenizeLine '0o_236_777_644' - expect(tokens[0]).toEqual value: '0b_236_777_644', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0o_236_777_644', scopes: ['source.python'] it "does not tokenize octal numbers with a trailing separator", -> - {tokens} = grammar.tokenizeLine '0b236_777_644_' + {tokens} = grammar.tokenizeLine '0o236_777_644_' - expect(tokens[0]).toEqual value: '0b236_777_644_', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0o236_777_644_', scopes: ['source.python'] it "does not tokenize non-octal numbers", -> {tokens} = grammar.tokenizeLine '0o8' @@ -102,24 +102,24 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: '0xe3', scopes: ['source.python', 'constant.numeric.integer.hexadecimal.python'] it "tokenizes hexadecimal numbers with separators", -> - {tokens} = grammar.tokenizeLine '0bdead_beef_1337' + {tokens} = grammar.tokenizeLine '0xdead_beef_1337' - expect(tokens[0]).toEqual value: '0bdead_beef_1337', scopes: ['source.python', 'constant.numeric.integer.hexadecimal.python'] + expect(tokens[0]).toEqual value: '0xdead_beef_1337', scopes: ['source.python', 'constant.numeric.integer.hexadecimal.python'] it "does not tokenize hexadecimal numbers with more than one consecutive separator", -> - {tokens} = grammar.tokenizeLine '0bdead__beef_1337' + {tokens} = grammar.tokenizeLine '0xdead__beef_1337' - expect(tokens[0]).toEqual value: '0bdead__beef_1337', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0xdead__beef_1337', scopes: ['source.python'] it "does not tokenize hexadecimal numbers with a leading separator", -> - {tokens} = grammar.tokenizeLine '0b_dead_beef_1337' + {tokens} = grammar.tokenizeLine '0x_dead_beef_1337' - expect(tokens[0]).toEqual value: '0b_dead_beef_1337', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0x_dead_beef_1337', scopes: ['source.python'] it "does not tokenize hexadecimal numbers with a trailing separator", -> - {tokens} = grammar.tokenizeLine '0bdead_beef_1337_' + {tokens} = grammar.tokenizeLine '0xdead_beef_1337_' - expect(tokens[0]).toEqual value: '0bdead_beef_1337_', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0xdead_beef_1337_', scopes: ['source.python'] it "does not tokenize non-hexadecimal numbers", -> {tokens} = grammar.tokenizeLine '0xf' @@ -130,7 +130,7 @@ describe "Python grammar", -> it "tokenizes decimal numbers", -> {tokens} = grammar.tokenizeLine '92' - expect(tokens[0]).toEqual value: '92', scopes: ['source.python', 'constant.numeric.integer.octal.python'] + expect(tokens[0]).toEqual value: '92', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] it "tokenizes decimal numbers with separators", -> {tokens} = grammar.tokenizeLine '123_456_789' From 1af383d9f9c3199eea292a45ca546f13dd75f563 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:50:10 +0000 Subject: [PATCH 07/12] Caught a regression! --- grammars/python.cson | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grammars/python.cson b/grammars/python.cson index 00b1910..a84c3ab 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -66,7 +66,8 @@ 'name': 'constant.numeric.complex.python' } { - 'match': '\\b(?i:(\\d(?:_?\\d)*\\.(\\d(?:_?\\d)*)?(e[\\-\\+]?\\d(?:_?\\d)*)?))\\b' + # Use a custom word boundary as floats ending with . won't be matched by \b + 'match': '\\b(?i:(\\d(?:_?\\d)*\\.(\\d(?:_?\\d)*)?(e[\\-\\+]?\\d(?:_?\\d)*)?))(?=[^a-zA-Z_]|$)' 'name': 'constant.numeric.float.python' } { From d95191b65789d5fb6cdf8cd37eadf63d1cff002b Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:53:55 +0000 Subject: [PATCH 08/12] LOL I know my hex numbers, I promise --- spec/python-spec.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index f3ae4d3..9c30cda 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -73,7 +73,7 @@ describe "Python grammar", -> it "tokenizes octal numbers with separators", -> {tokens} = grammar.tokenizeLine '0o236_777_644' - expect(tokens[0]).toEqual value: '0b236_777_644', scopes: ['source.python', 'constant.numeric.integer.octal.python'] + expect(tokens[0]).toEqual value: '0o236_777_644', scopes: ['source.python', 'constant.numeric.integer.octal.python'] it "does not tokenize octal numbers with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '0o236__777_644' @@ -122,9 +122,9 @@ describe "Python grammar", -> expect(tokens[0]).toEqual value: '0xdead_beef_1337_', scopes: ['source.python'] it "does not tokenize non-hexadecimal numbers", -> - {tokens} = grammar.tokenizeLine '0xf' + {tokens} = grammar.tokenizeLine '0xg' - expect(tokens[0]).toEqual value: '0xf', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '0xg', scopes: ['source.python'] describe "decimal", -> it "tokenizes decimal numbers", -> From 247fcb3d76eb56702946b4a128237384e5c7a0b0 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 01:59:56 +0000 Subject: [PATCH 09/12] Expand the lookarounds a bit --- grammars/python.cson | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index a84c3ab..757a40d 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -62,7 +62,8 @@ 'name': 'constant.numeric.integer.binary.python' } { - 'match': '\\b(?i:(((\\d(?:_?\\d)*(\\.(\\d(?:_?\\d)*)?)?|(?<=[^0-9a-zA-Z_])\\.\\d(?:_?\\d)*)(e[\\-\\+]?\\d(?:_?\\d)*)?))J)\\b' + # Use a custom word boundary for the second half as floats beginning with . won't be matched by \b + 'match': '(?i:(((\\b\\d(?:_?\\d)*(\\.(\\d(?:_?\\d)*)?)?|(?<=[^0-9a-zA-Z_]|^)\\.\\d(?:_?\\d)*)(e[\\-\\+]?\\d(?:_?\\d)*)?))J)\\b' 'name': 'constant.numeric.complex.python' } { @@ -71,7 +72,8 @@ 'name': 'constant.numeric.float.python' } { - 'match': '(?<=[^0-9a-zA-Z_])(?i:(\\.\\d(?:_?\\d)*(e[\\-\\+]?\\d(?:_?\\d)*)?))\\b' + # Use a custom word boundary as floats beginning with . won't be matched by \b + 'match': '(?<=[^0-9a-zA-Z_]|^)(?i:(\\.\\d(?:_?\\d)*(e[\\-\\+]?\\d(?:_?\\d)*)?))\\b' 'name': 'constant.numeric.float.python' } { From dfe5ada91a5ac5d6ade6f2da4a5bc3ea7e52d578 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 02:09:37 +0000 Subject: [PATCH 10/12] More spec fixes --- spec/python-spec.coffee | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 9c30cda..c5a3a14 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -182,17 +182,17 @@ describe "Python grammar", -> it "does not tokenize decimal floats with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '373__827.639_817' - expect(tokens[0]).toEqual value: '373__827.639_817', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '373__827', scopes: ['source.python'] it "does not tokenize decimal floats with a leading separator", -> {tokens} = grammar.tokenizeLine '373_827._639_817' - expect(tokens[0]).toEqual value: '373_827._639_817', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '373_827', scopes: ['source.python'] it "does not tokenize decimal floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '373_827_.639_817' - expect(tokens[0]).toEqual value: '373_827_.639_817', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '373_827_', scopes: ['source.python'] describe "exponential", -> it "tokenizes exponential floats", -> @@ -208,73 +208,73 @@ describe "Python grammar", -> it "tokenizes exponential decimal floats", -> {tokens} = grammar.tokenizeLine '97.22e-43' - expect(tokens[0]).toEqual value: '97e+43', scopes: ['source.python', 'constant.numeric.float.python'] + expect(tokens[0]).toEqual value: '97.22e+43', scopes: ['source.python', 'constant.numeric.float.python'] it "tokenizes exponential floats with separators", -> {tokens} = grammar.tokenizeLine '62_2e-2_839' - expect(tokens[0]).toEqual value: '62_2e-2_839', scopes: ['source.python', 'constant.numeric.float.python'] + expect(tokens[0]).toEqual value: '62_2e', scopes: ['source.python', 'constant.numeric.float.python'] it "does not tokenize exponential floats with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '62_2e-2__839' - expect(tokens[0]).toEqual value: '62_2e-2__839', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '62_2e', scopes: ['source.python'] it "does not tokenize exponential floats with a leading separator", -> {tokens} = grammar.tokenizeLine '_62_2e-2_839' - expect(tokens[0]).toEqual value: '_62_2e-2_839', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '_62_2e', scopes: ['source.python'] it "does not tokenize exponential floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '62_2e-2_839_' - expect(tokens[0]).toEqual value: '62_2e-2_839_', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '62_2e', scopes: ['source.python'] it "does not tokenize exponential floats with a fractional exponent", -> {tokens} = grammar.tokenizeLine '97.22e-43.3' - expect(tokens[0]).toEqual value: '97.22e-43.3', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '97.22e-43', scopes: ['source.python', 'constant.numeric.float.python'] describe "complex", -> it "tokenizes complex floats", -> {tokens} = grammar.tokenizeLine '63j' - expect(tokens[0]).toEqual value: '63j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + expect(tokens[0]).toEqual value: '63j', scopes: ['source.python', 'constant.numeric.complex.python'] it "tokenizes complex decimal floats", -> {tokens} = grammar.tokenizeLine '63.72j' - expect(tokens[0]).toEqual value: '63.72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + expect(tokens[0]).toEqual value: '63.72j', scopes: ['source.python', 'constant.numeric.complex.python'] it "tokenizes complex exponential floats", -> {tokens} = grammar.tokenizeLine '63e-72j' - expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + expect(tokens[0]).toEqual value: '63e-72j', scopes: ['source.python', 'constant.numeric.complex.python'] it "tokenizes complex decimal exponential floats", -> {tokens} = grammar.tokenizeLine '63.e-72j' - expect(tokens[0]).toEqual value: '63.e-72j', scopes: ['source.python', 'constant.numeric.float.complex.python'] + expect(tokens[0]).toEqual value: '63.e-72j', scopes: ['source.python', 'constant.numeric.complex.python'] it "tokenizes complex floats with separators", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839j' - expect(tokens[0]).toEqual value: '62_2.83_7e-2_839j', scopes: ['source.python', 'constant.numeric.float.python'] + expect(tokens[0]).toEqual value: '62_2.83_7e-2_839j', scopes: ['source.python', 'constant.numeric.complex.python'] it "does not tokenize complex floats with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2__839j' - expect(tokens[0]).toEqual value: '62_2.83_7e-2__839j', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '62_2.8', scopes: ['source.python'] it "does not tokenize complex floats with a leading separator", -> {tokens} = grammar.tokenizeLine '62_2._83_7e-2_839j' - expect(tokens[0]).toEqual value: '_62_2._83_7e-2_839', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '_62_2', scopes: ['source.python'] it "does not tokenize complex floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839_j' - expect(tokens[0]).toEqual value: '62_2.83_7e-2_839_j', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '62_2.83_7e-2_83', scopes: ['source.python', 'constant.numeric.float.python'] it "tokenizes multi-line strings", -> tokens = grammar.tokenizeLines('"1\\\n2"') From ea7e3206351e0065f3ce0fe7cdde7533f191f7f8 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 02:22:24 +0000 Subject: [PATCH 11/12] Final fixes, hopefully --- spec/python-spec.coffee | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index c5a3a14..879d6f5 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -187,7 +187,7 @@ describe "Python grammar", -> it "does not tokenize decimal floats with a leading separator", -> {tokens} = grammar.tokenizeLine '373_827._639_817' - expect(tokens[0]).toEqual value: '373_827', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '373_827', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] it "does not tokenize decimal floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '373_827_.639_817' @@ -208,12 +208,12 @@ describe "Python grammar", -> it "tokenizes exponential decimal floats", -> {tokens} = grammar.tokenizeLine '97.22e-43' - expect(tokens[0]).toEqual value: '97.22e+43', scopes: ['source.python', 'constant.numeric.float.python'] + expect(tokens[0]).toEqual value: '97.22e-43', scopes: ['source.python', 'constant.numeric.float.python'] it "tokenizes exponential floats with separators", -> {tokens} = grammar.tokenizeLine '62_2e-2_839' - expect(tokens[0]).toEqual value: '62_2e', scopes: ['source.python', 'constant.numeric.float.python'] + expect(tokens[0]).toEqual value: '62_2e-2_839', scopes: ['source.python', 'constant.numeric.float.python'] it "does not tokenize exponential floats with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '62_2e-2__839' @@ -264,12 +264,12 @@ describe "Python grammar", -> it "does not tokenize complex floats with more than one consecutive separator", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2__839j' - expect(tokens[0]).toEqual value: '62_2.8', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '62_2.8', scopes: ['source.python', 'constant.numeric.float.python'] it "does not tokenize complex floats with a leading separator", -> {tokens} = grammar.tokenizeLine '62_2._83_7e-2_839j' - expect(tokens[0]).toEqual value: '_62_2', scopes: ['source.python'] + expect(tokens[0]).toEqual value: '_62_2', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] it "does not tokenize complex floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839_j' From 0f913ebd8bde2b52dfe5b46027f739d0d2827251 Mon Sep 17 00:00:00 2001 From: Winston Liu Date: Mon, 27 Sep 2021 02:24:15 +0000 Subject: [PATCH 12/12] Right-o, final fix for real this time! --- spec/python-spec.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 879d6f5..ab606e8 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -269,7 +269,7 @@ describe "Python grammar", -> it "does not tokenize complex floats with a leading separator", -> {tokens} = grammar.tokenizeLine '62_2._83_7e-2_839j' - expect(tokens[0]).toEqual value: '_62_2', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] + expect(tokens[0]).toEqual value: '62_2', scopes: ['source.python', 'constant.numeric.integer.decimal.python'] it "does not tokenize complex floats with a trailing separator", -> {tokens} = grammar.tokenizeLine '62_2.83_7e-2_839_j'