From ea70a520f94d5aa6b493509f10842af2fe658078 Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Fri, 10 Jan 2025 17:24:58 +0700 Subject: [PATCH 1/8] fix: accept completion on key tab press --- src/components/gui/sql-editor/index.tsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/components/gui/sql-editor/index.tsx b/src/components/gui/sql-editor/index.tsx index c6d786d9..3cfd75df 100644 --- a/src/components/gui/sql-editor/index.tsx +++ b/src/components/gui/sql-editor/index.tsx @@ -3,7 +3,7 @@ import CodeMirror, { Extension, ReactCodeMirrorRef, } from "@uiw/react-codemirror"; -import { LanguageSupport } from "@codemirror/language"; +import { indentUnit, LanguageSupport } from "@codemirror/language"; import { acceptCompletion, completionStatus, @@ -83,6 +83,24 @@ const SqlEditor = forwardRef( return true; }, }, + { + key: "Space", + preventDefault: true, + run: (target) => { + if (completionStatus(target.state) === "active") { + acceptCompletion(target); + } else { + target.dispatch({ + changes: { + from: target.state.selection.main.from, + insert: " ", + }, + selection: { anchor: target.state.selection.main.anchor + 1 }, + }); + } + return true; + }, + }, { key: "Ctrl-Space", mac: "Cmd-i", @@ -153,6 +171,7 @@ const SqlEditor = forwardRef( }, }), keyExtensions, + indentUnit.of(" "), sqlDialect, tooltipExtension, tableNameHighlightPlugin, @@ -184,6 +203,7 @@ const SqlEditor = forwardRef( drawSelection: false, }} theme={theme} + indentWithTab={false} value={value} height="100%" onChange={onChange} From bc585a5efa177c611a9c82c6a5d0fa36f0bd027d Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Sat, 11 Jan 2025 17:08:14 +0700 Subject: [PATCH 2/8] feat: sql tokenizer --- src/lib/sql/tokenizer.test.ts | 371 ++++++++++++++++++++++++++++++++++ src/lib/sql/tokenizer.ts | 46 +++++ 2 files changed, 417 insertions(+) create mode 100644 src/lib/sql/tokenizer.test.ts create mode 100644 src/lib/sql/tokenizer.ts diff --git a/src/lib/sql/tokenizer.test.ts b/src/lib/sql/tokenizer.test.ts new file mode 100644 index 00000000..906599a7 --- /dev/null +++ b/src/lib/sql/tokenizer.test.ts @@ -0,0 +1,371 @@ +import { tokenizeSql } from "./tokenizer"; + +test("Select with placeholder and multiple lines comment", () => { + const sql = ` + SELECT * FROM customers + WHERE customer.id = :id and customer.name != "john:wich" + /* This : is not placeholder */ + `; + const tokens = tokenizeSql(sql); + console.log(tokens); + expect(tokens).toEqual([ + { type: "WHITESPACE", value: "\n " }, + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: "\n " }, + { type: "KEYWORD", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customer.id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "PLACEHOLDER", value: ":id" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "and" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customer.name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "!=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: '"john:wich"' }, + { type: "WHITESPACE", value: "\n " }, + { type: "COMMENT", value: "/* This : is not placeholder */" }, + { type: "WHITESPACE", value: "\n " }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); +test("Select Tokenizer", () => { + const sql = "SELECT `customer.id`, customer.name FROM `customers`"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "`customer.id`" }, + { type: "PUNCTUATION", value: "," }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customer.name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "`customers`" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Insert Tokenizer", () => { + const sql = "INSERT INTO customers (id, name) VALUES (1, 'John')"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "INSERT" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "INTO" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "PUNCTUATION", value: "(" }, + { type: "IDENTIFIER", value: "id" }, + { type: "PUNCTUATION", value: "," }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "PUNCTUATION", value: ")" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "VALUES" }, + { type: "WHITESPACE", value: " " }, + { type: "PUNCTUATION", value: "(" }, + { type: "NUMBER", value: "1" }, + { type: "PUNCTUATION", value: "," }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'John'" }, + { type: "PUNCTUATION", value: ")" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Update Tokenizer", () => { + const sql = "UPDATE customers SET name = 'John' WHERE id = 1"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "UPDATE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "SET" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'John'" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "NUMBER", value: "1" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Delete Tokenizer", () => { + const sql = "DELETE FROM customers WHERE id = 1"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "DELETE" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "NUMBER", value: "1" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with DISTINCT", () => { + const sql = "SELECT DISTINCT name FROM customers"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "DISTINCT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with JOIN", () => { + const sql = + "SELECT * FROM customers INNER JOIN orders ON customers.id = orders.customer_id"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "INNER" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "JOIN" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "orders" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "ON" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers.id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "orders.customer_id" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with GROUP BY", () => { + const sql = "SELECT name, COUNT(*) FROM customers GROUP BY name"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "PUNCTUATION", value: "," }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "COUNT" }, + { type: "PUNCTUATION", value: "(" }, + { type: "OPERATOR", value: "*" }, + { type: "PUNCTUATION", value: ")" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "GROUP" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "BY" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with ORDER BY", () => { + const sql = "SELECT name FROM customers ORDER BY name DESC"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "ORDER" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "BY" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "DESC" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with LIMIT and OFFSET", () => { + const sql = "SELECT name FROM customers LIMIT 10 OFFSET 5"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "LIMIT" }, + { type: "WHITESPACE", value: " " }, + { type: "NUMBER", value: "10" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "OFFSET" }, + { type: "WHITESPACE", value: " " }, + { type: "NUMBER", value: "5" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with CASE WHEN", () => { + const sql = + "SELECT CASE WHEN id = 1 THEN 'one' ELSE 'other' END AS result FROM customers"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "CASE" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "WHEN" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "NUMBER", value: "1" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "THEN" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'one'" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "ELSE" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'other'" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "END" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "AS" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "result" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with UNION", () => { + const sql = "SELECT name FROM customers UNION SELECT name FROM orders"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "UNION" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "orders" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Select with multiple placehoder", () => { + const sql = "SELECT * FROM customers WHERE id = :id AND name = :name"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "PLACEHOLDER", value: ":id" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "AND" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "PLACEHOLDER", value: ":name" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("try to tokenize invalid sql", () => { + const sql = "SELECT * FROM customers WHERE id = :id AND name = :123name"; + const tokens = tokenizeSql(sql); + expect(tokens).toEqual([{ type: "sql", value: sql }]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("try extra whitespace stay next to each other", () => { + const sql = "SELECT * FROM customers"; + const tokens = tokenizeSql(sql); + expect(tokens.map((t) => t.value).join("")).toBe(sql); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + ]); +}); diff --git a/src/lib/sql/tokenizer.ts b/src/lib/sql/tokenizer.ts new file mode 100644 index 00000000..57109bc4 --- /dev/null +++ b/src/lib/sql/tokenizer.ts @@ -0,0 +1,46 @@ +interface Token { + type: string; + value: string; +} + +const tokenTypes: { type: string; regex: RegExp }[] = [ + { type: "WHITESPACE", regex: /^\s+/ }, + { + type: "KEYWORD", + regex: + /^(SELECT|FROM|WHERE|INSERT|INTO|VALUES|UPDATE|SET|DELETE|INNER|OUTER|JOIN|ON|AND|OR|NOT|LIKE|GROUP|BY|HAVING|ORDER|LIMIT|OFFSET|DISTINCT|AS|CASE|WHEN|THEN|ELSE|END|UNION|ALL|ANY|EXISTS|IN|IS|NULL|BETWEEN|WITH|DESC|ASC|COUNT)\b(?!\.)/i, + }, + { type: "IDENTIFIER", regex: /^(`?[a-zA-Z_][a-zA-Z0-9_.]*`?)/ }, + { type: "STRING", regex: /^(?:'(?:[^']|'')*'|"(?:[^"]|"")*")/ }, + { type: "NUMBER", regex: /^\d+(\.\d+)?/ }, + { type: "PLACEHOLDER", regex: /^:[a-zA-Z_][a-zA-Z0-9_]*/ }, + { type: "COMMENT", regex: /^(--[^\n]*|\/\*[\s\S]*?\*\/)/ }, + { type: "OPERATOR", regex: /^(=|<>|!=|<|>|<=|>=|\+|-|\*|\/)/ }, + { type: "PUNCTUATION", regex: /^[`,;()]/ }, +]; + +export function tokenizeSql(sql: string): Token[] { + const tokens: Token[] = []; + let cursor = 0; + const length = sql.length; + + while (cursor < length) { + let matched = false; + for (const { type, regex } of tokenTypes) { + const subStr = sql.substring(cursor); + const match = regex.exec(subStr); + if (match) { + tokens.push({ type, value: match[0] }); + cursor += match[0].length; + matched = true; + break; + } + } + + if (!matched) { + return [{ type: "sql", value: sql }]; + } + } + + return tokens; +} From 748c241650b6453d7c6f0b06279ec6ce597b5d6b Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Mon, 13 Jan 2025 10:25:01 +0700 Subject: [PATCH 3/8] feat: add query placeholder --- src/components/gui/tabs/query-placeholder.tsx | 75 +++++++++++++++++++ src/components/gui/tabs/query-tab.tsx | 61 ++++++++++++++- src/lib/sql/tokenizer.test.ts | 27 ++++++- src/lib/sql/tokenizer.ts | 40 +++++----- 4 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 src/components/gui/tabs/query-placeholder.tsx diff --git a/src/components/gui/tabs/query-placeholder.tsx b/src/components/gui/tabs/query-placeholder.tsx new file mode 100644 index 00000000..a3c409c4 --- /dev/null +++ b/src/components/gui/tabs/query-placeholder.tsx @@ -0,0 +1,75 @@ +import { buttonVariants } from "@/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { LucideChevronDown } from "lucide-react"; +interface Props { + placeHolders: Record; + onChange: (placeHolders: Record) => void; +} + +export function QueryPlaceholder({ + placeHolders, + onChange, +}: Props): JSX.Element { + const placeHolderTable = () => { + return ( +
+ + + + + + + + + {Object.entries(placeHolders).map(([key, value]) => ( + + + + + ))} + +
VariableValue
{key} + { + const newValue = e.currentTarget.value; + const newPlaceHolders = { + ...placeHolders, + [key]: newValue, + }; + onChange(newPlaceHolders); + }} + > +
+
+ ); + }; + return ( + + +
+ {hasPlaceHolderWithEmptyValue(placeHolders) && ( +
+
+
+ )} + Placeholders{" "} + {!!placeHolders && } +
+
+ {placeHolderTable()} +
+ ); +} + +export function hasPlaceHolderWithEmptyValue( + placeHolders: Record +) { + return Object.values(placeHolders).some((value) => value === ""); +} diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index b30f4afb..de6c440d 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -1,5 +1,5 @@ import { format } from "sql-formatter"; -import { useCallback, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { LucideGrid, LucideMessageSquareWarning, @@ -51,6 +51,9 @@ import { } from "@/components/ui/dropdown-menu"; import { isExplainQueryPlan } from "../query-explanation"; import ExplainResultTab from "../tabs-result/explain-result-tab"; +import { tokenizeSql } from "@/lib/sql/tokenizer"; +import { QueryPlaceholder } from "./query-placeholder"; +import { escapeSqlValue } from "@/drivers/sqlite/sql-helper"; interface QueryWindowProps { initialCode?: string; @@ -84,6 +87,31 @@ export default function QueryWindow({ initialNamespace ?? "Unsaved Query" ); const [savedKey, setSavedKey] = useState(initialSavedKey); + const [placeHolders, setPlaceHolders] = useState>({}); + + useEffect(() => { + const timer = setTimeout(() => { + const editorState = editorRef.current?.view?.state; + if (!editorState) return; + const finalStatements = splitSqlQuery(editorState).map((q) => q.text); + const newPlaceholders: Record = {}; + for (const statement of finalStatements) { + const token = tokenizeSql(statement); + const placeholders = token + .filter((t) => t.type === "PLACEHOLDER") + .map((t) => t.value.split(":")[1]); + for (const placeholder of placeholders) { + newPlaceholders[placeholder] = ""; + } + } + for (const newKey of Object.keys(newPlaceholders)) { + newPlaceholders[newKey] = placeHolders[newKey] ?? ""; + } + setPlaceHolders(newPlaceholders); + }, 1000); + return () => clearTimeout(timer); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [code]); const onFormatClicked = () => { try { @@ -137,6 +165,29 @@ export default function QueryWindow({ setProgress(undefined); setQueryTabIndex(0); + //inject placeholders + for (const statement of finalStatements) { + const token = tokenizeSql(statement); + const variables = token + .filter((t) => t.type === "PLACEHOLDER") + .map((t) => t.value.split(":")[1]); + if ( + variables.length > 0 && + variables.some((p) => placeHolders[p] === "") + ) { + toast.error("Please fill in all placeholders"); + return; + } + } + for (const key of Object.keys(placeHolders)) { + finalStatements = finalStatements.map((s) => + s.replace( + new RegExp(`:${key}`, "g"), + escapeSqlValue(placeHolders[key]) + ) + ); + } + multipleQuery(databaseDriver, finalStatements, (currentProgress) => { setProgress(currentProgress); }) @@ -348,6 +399,14 @@ export default function QueryWindow({
Ln {lineNumber}
Col {columnNumber + 1}
+
+ {Object.keys(placeHolders).length > 0 && ( + + )} +
diff --git a/src/lib/sql/tokenizer.test.ts b/src/lib/sql/tokenizer.test.ts index 906599a7..cacacebf 100644 --- a/src/lib/sql/tokenizer.test.ts +++ b/src/lib/sql/tokenizer.test.ts @@ -7,7 +7,6 @@ test("Select with placeholder and multiple lines comment", () => { /* This : is not placeholder */ `; const tokens = tokenizeSql(sql); - console.log(tokens); expect(tokens).toEqual([ { type: "WHITESPACE", value: "\n " }, { type: "KEYWORD", value: "SELECT" }, @@ -351,7 +350,31 @@ test("Select with multiple placehoder", () => { test("try to tokenize invalid sql", () => { const sql = "SELECT * FROM customers WHERE id = :id AND name = :123name"; const tokens = tokenizeSql(sql); - expect(tokens).toEqual([{ type: "sql", value: sql }]); + expect(tokens).toEqual([ + { type: "KEYWORD", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "id" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "PLACEHOLDER", value: ":id" }, + { type: "WHITESPACE", value: " " }, + { type: "KEYWORD", value: "AND" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: ":123name" }, + ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); diff --git a/src/lib/sql/tokenizer.ts b/src/lib/sql/tokenizer.ts index 57109bc4..e0f0cb5c 100644 --- a/src/lib/sql/tokenizer.ts +++ b/src/lib/sql/tokenizer.ts @@ -20,27 +20,31 @@ const tokenTypes: { type: string; regex: RegExp }[] = [ ]; export function tokenizeSql(sql: string): Token[] { - const tokens: Token[] = []; - let cursor = 0; - const length = sql.length; + try { + const tokens: Token[] = []; + let cursor = 0; + const length = sql.length; - while (cursor < length) { - let matched = false; - for (const { type, regex } of tokenTypes) { - const subStr = sql.substring(cursor); - const match = regex.exec(subStr); - if (match) { - tokens.push({ type, value: match[0] }); - cursor += match[0].length; - matched = true; - break; + while (cursor < length) { + let matched = false; + let subStr = ""; + for (const { type, regex } of tokenTypes) { + subStr = sql.substring(cursor); + const match = regex.exec(subStr); + if (match) { + tokens.push({ type, value: match[0] }); + cursor += match[0].length; + matched = true; + break; + } } - } - if (!matched) { - return [{ type: "sql", value: sql }]; + if (!matched) { + return [...tokens, { type: "UNKNOWN", value: subStr }]; + } } + return tokens; + } catch (e) { + return [{ type: "SQL", value: sql }]; } - - return tokens; } From 76f00a8d6451eeff8e02c047fc979733146d64f4 Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Tue, 14 Jan 2025 11:32:54 +0700 Subject: [PATCH 4/8] fix: - handle placeholder value with empty string - change query placehoder UI from popover to sheet --- src/components/gui/tabs/query-placeholder.tsx | 18 +++++++----------- src/components/gui/tabs/query-tab.tsx | 4 ++-- src/drivers/sqlite/sql-helper.ts | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/components/gui/tabs/query-placeholder.tsx b/src/components/gui/tabs/query-placeholder.tsx index a3c409c4..e49819c9 100644 --- a/src/components/gui/tabs/query-placeholder.tsx +++ b/src/components/gui/tabs/query-placeholder.tsx @@ -1,14 +1,10 @@ import { buttonVariants } from "@/components/ui/button"; -import { - Popover, - PopoverContent, - PopoverTrigger, -} from "@/components/ui/popover"; import { LucideChevronDown } from "lucide-react"; interface Props { placeHolders: Record; onChange: (placeHolders: Record) => void; } +import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; export function QueryPlaceholder({ placeHolders, @@ -16,7 +12,7 @@ export function QueryPlaceholder({ }: Props): JSX.Element { const placeHolderTable = () => { return ( -
+
@@ -51,8 +47,8 @@ export function QueryPlaceholder({ ); }; return ( - - + +
{hasPlaceHolderWithEmptyValue(placeHolders) && (
@@ -62,9 +58,9 @@ export function QueryPlaceholder({ Placeholders{" "} {!!placeHolders && }
- - {placeHolderTable()} - + + {placeHolderTable()} + ); } diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index de6c440d..c19da1ca 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -53,7 +53,7 @@ import { isExplainQueryPlan } from "../query-explanation"; import ExplainResultTab from "../tabs-result/explain-result-tab"; import { tokenizeSql } from "@/lib/sql/tokenizer"; import { QueryPlaceholder } from "./query-placeholder"; -import { escapeSqlValue } from "@/drivers/sqlite/sql-helper"; +import { escapeSqlValue, extractInputValue } from "@/drivers/sqlite/sql-helper"; interface QueryWindowProps { initialCode?: string; @@ -183,7 +183,7 @@ export default function QueryWindow({ finalStatements = finalStatements.map((s) => s.replace( new RegExp(`:${key}`, "g"), - escapeSqlValue(placeHolders[key]) + escapeSqlValue(extractInputValue(placeHolders[key])) ) ); } diff --git a/src/drivers/sqlite/sql-helper.ts b/src/drivers/sqlite/sql-helper.ts index 0606f2ba..df3d6feb 100644 --- a/src/drivers/sqlite/sql-helper.ts +++ b/src/drivers/sqlite/sql-helper.ts @@ -33,6 +33,23 @@ export function escapeSqlValue(value: unknown) { throw new Error(value.toString() + " is unrecongize type of value"); } +export function extractInputValue(input: string): string | number { + const trimmedInput = input.trim(); + if ( + (trimmedInput.startsWith('"') && trimmedInput.endsWith('"')) || + (trimmedInput.startsWith("'") && trimmedInput.endsWith("'")) + ) { + return trimmedInput.slice(1, -1).toString(); + } + + const parsedNumber = parseFloat(trimmedInput); + if (!isNaN(parsedNumber)) { + return parsedNumber; + } + + return trimmedInput.toString(); +} + export function convertSqliteType( type: string | undefined ): TableColumnDataType | undefined { From 4bec09ee17a4c2a4cdcb11d224ff1a54e2515010 Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Thu, 16 Jan 2025 13:45:58 +0700 Subject: [PATCH 5/8] feat: handle more databases token --- src/components/gui/tabs/query-tab.tsx | 4 +- src/lib/sql/tokenizer.test.ts | 438 +++++++++++++------------- src/lib/sql/tokenizer.ts | 104 ++++-- 3 files changed, 310 insertions(+), 236 deletions(-) diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index c19da1ca..e6ae6c30 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -96,7 +96,7 @@ export default function QueryWindow({ const finalStatements = splitSqlQuery(editorState).map((q) => q.text); const newPlaceholders: Record = {}; for (const statement of finalStatements) { - const token = tokenizeSql(statement); + const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); const placeholders = token .filter((t) => t.type === "PLACEHOLDER") .map((t) => t.value.split(":")[1]); @@ -167,7 +167,7 @@ export default function QueryWindow({ //inject placeholders for (const statement of finalStatements) { - const token = tokenizeSql(statement); + const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); const variables = token .filter((t) => t.type === "PLACEHOLDER") .map((t) => t.value.split(":")[1]); diff --git a/src/lib/sql/tokenizer.test.ts b/src/lib/sql/tokenizer.test.ts index cacacebf..71cd04e6 100644 --- a/src/lib/sql/tokenizer.test.ts +++ b/src/lib/sql/tokenizer.test.ts @@ -1,394 +1,408 @@ import { tokenizeSql } from "./tokenizer"; -test("Select with placeholder and multiple lines comment", () => { - const sql = ` - SELECT * FROM customers - WHERE customer.id = :id and customer.name != "john:wich" - /* This : is not placeholder */ - `; - const tokens = tokenizeSql(sql); +test("Select simple query", () => { + const sql = "SELECT * FROM customers"; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "WHITESPACE", value: "\n " }, - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "customers" }, - { type: "WHITESPACE", value: "\n " }, - { type: "KEYWORD", value: "WHERE" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); + +test("Check identifers", () => { + const sql = `SELECT customer.name, [customer].[first name], "customer"."last name" + FROM customers WHERE customers.name = 'John Doe'`; + const tokens = tokenizeSql(sql, "sqlite"); + expect(tokens).toEqual([ + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customer.id" }, + { type: "IDENTIFIER", value: "customer" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "name" }, + { type: "PUNCTUATION", value: "," }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "IDENTIFIER", value: "[customer]" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "[first name]" }, + { type: "PUNCTUATION", value: "," }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: '"customer"' }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: '"last name"' }, + { type: "WHITESPACE", value: "\n " }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "PLACEHOLDER", value: ":id" }, + { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "and" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customer.name" }, + { type: "IDENTIFIER", value: "customers" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "name" }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "!=" }, + { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "STRING", value: '"john:wich"' }, - { type: "WHITESPACE", value: "\n " }, - { type: "COMMENT", value: "/* This : is not placeholder */" }, - { type: "WHITESPACE", value: "\n " }, + { type: "STRING", value: "'John Doe'" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select Tokenizer", () => { - const sql = "SELECT `customer.id`, customer.name FROM `customers`"; - const tokens = tokenizeSql(sql); + +test("Back tick identifier", () => { + const sql = "SELECT `customer`.`name` FROM `customers`"; + const tokens = tokenizeSql(sql, "mysql"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "`customer.id`" }, - { type: "PUNCTUATION", value: "," }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customer.name" }, + { type: "IDENTIFIER", value: "`customer`" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "`name`" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "`customers`" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Insert Tokenizer", () => { - const sql = "INSERT INTO customers (id, name) VALUES (1, 'John')"; - const tokens = tokenizeSql(sql); +test("Invalid identifier", () => { + const sql = `SELECT [customer].[fist + name] FROM "customers"`; + const tokens = tokenizeSql(sql, "mysql"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "INSERT" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "INTO" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, - { type: "WHITESPACE", value: " " }, - { type: "PUNCTUATION", value: "(" }, - { type: "IDENTIFIER", value: "id" }, - { type: "PUNCTUATION", value: "," }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "[customer]" }, + { type: "PUNCTUATION", value: "." }, + { type: "UNKNOWN", value: "[" }, + { type: "IDENTIFIER", value: "fist" }, + { type: "WHITESPACE", value: "\n " }, { type: "IDENTIFIER", value: "name" }, - { type: "PUNCTUATION", value: ")" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "VALUES" }, + { type: "UNKNOWN", value: "]" }, { type: "WHITESPACE", value: " " }, - { type: "PUNCTUATION", value: "(" }, - { type: "NUMBER", value: "1" }, - { type: "PUNCTUATION", value: "," }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "STRING", value: "'John'" }, - { type: "PUNCTUATION", value: ")" }, + { type: "IDENTIFIER", value: '"customers"' }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Update Tokenizer", () => { - const sql = "UPDATE customers SET name = 'John' WHERE id = 1"; - const tokens = tokenizeSql(sql); +test("String literal", () => { + const sql = `SELECT 'Hello' AS "greeting" FROM "customers" WHERE "name" = 'John Doe'`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "UPDATE" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "STRING", value: "'Hello'" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "SET" }, + { type: "IDENTIFIER", value: "AS" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "IDENTIFIER", value: '"greeting"' }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "STRING", value: "'John'" }, + { type: "IDENTIFIER", value: '"customers"' }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "WHERE" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "id" }, + { type: "IDENTIFIER", value: '"name"' }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "NUMBER", value: "1" }, + { type: "STRING", value: "'John Doe'" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Delete Tokenizer", () => { - const sql = "DELETE FROM customers WHERE id = 1"; - const tokens = tokenizeSql(sql); +test("number literal", () => { + const sql = `SELECT 123.45 FROM "customers" WHERE "age" = 30`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "DELETE" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "NUMBER", value: "123.45" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "WHERE" }, + { type: "IDENTIFIER", value: '"customers"' }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "id" }, + { type: "IDENTIFIER", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: '"age"' }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "NUMBER", value: "1" }, + { type: "NUMBER", value: "30" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with DISTINCT", () => { - const sql = "SELECT DISTINCT name FROM customers"; - const tokens = tokenizeSql(sql); +test("Placeholder", () => { + const sql = `SELECT * FROM customers WHERE name = :name`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "DISTINCT" }, + { type: "IDENTIFIER", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "name" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "PLACEHOLDER", value: ":name" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with JOIN", () => { - const sql = - "SELECT * FROM customers INNER JOIN orders ON customers.id = orders.customer_id"; - const tokens = tokenizeSql(sql); +test("Placehoder witch have sign : in string and comment", () => { + const sql = `/* + SELECT * FROM customers WHERE name = ':name' AND "code" =:code -- only :code is a placeholder + */ + SELECT * FROM customers WHERE name = ':name' AND "code" =:code -- only :code is a placeholder`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { + type: "COMMENT", + value: + "/*\n" + + ` SELECT * FROM customers WHERE name = ':name' AND "code" =:code -- only :code is a placeholder\n` + + " */", + }, + { type: "WHITESPACE", value: "\n " }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "INNER" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "JOIN" }, + { type: "IDENTIFIER", value: "name" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "orders" }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "':name'" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "ON" }, + { type: "IDENTIFIER", value: "AND" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers.id" }, + { type: "IDENTIFIER", value: '"code"' }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "=" }, + { type: "PLACEHOLDER", value: ":code" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "orders.customer_id" }, + { type: "COMMENT", value: "-- only :code is a placeholder" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with GROUP BY", () => { - const sql = "SELECT name, COUNT(*) FROM customers GROUP BY name"; - const tokens = tokenizeSql(sql); +test("Mysql comment", () => { + const sql = `SELECT * FROM customers WHERE name = 'John Doe' + # this is a comment + -- this is also comment + /* this is a block comment */ + `; + const tokens = tokenizeSql(sql, "mysql"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, - { type: "PUNCTUATION", value: "," }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "COUNT" }, - { type: "PUNCTUATION", value: "(" }, { type: "OPERATOR", value: "*" }, - { type: "PUNCTUATION", value: ")" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "GROUP" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "BY" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'John Doe'" }, + { type: "WHITESPACE", value: "\n " }, + { type: "COMMENT", value: "# this is a comment" }, + { type: "WHITESPACE", value: "\n " }, + { type: "COMMENT", value: "-- this is also comment" }, + { type: "WHITESPACE", value: "\n " }, + { type: "COMMENT", value: "/* this is a block comment */" }, + { type: "WHITESPACE", value: "\n " }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with ORDER BY", () => { - const sql = "SELECT name FROM customers ORDER BY name DESC"; - const tokens = tokenizeSql(sql); +test("Error on invalid # comment from non-mysql", () => { + const sql = `SELECT * FROM customers WHERE name = 'John Doe' # this is a comment`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "ORDER" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "BY" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "name" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "DESC" }, - ]); - expect(tokens.map((t) => t.value).join("")).toBe(sql); -}); - -test("Select with LIMIT and OFFSET", () => { - const sql = "SELECT name FROM customers LIMIT 10 OFFSET 5"; - const tokens = tokenizeSql(sql); - expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "STRING", value: "'John Doe'" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "UNKNOWN", value: "#" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "LIMIT" }, + { type: "IDENTIFIER", value: "this" }, { type: "WHITESPACE", value: " " }, - { type: "NUMBER", value: "10" }, + { type: "IDENTIFIER", value: "is" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "OFFSET" }, + { type: "IDENTIFIER", value: "a" }, { type: "WHITESPACE", value: " " }, - { type: "NUMBER", value: "5" }, + { type: "IDENTIFIER", value: "comment" }, ]); - expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with CASE WHEN", () => { - const sql = - "SELECT CASE WHEN id = 1 THEN 'one' ELSE 'other' END AS result FROM customers"; - const tokens = tokenizeSql(sql); +test("Operator and punctuation", () => { + const sql = `SELECT * FROM customers WHERE name = 'John Doe' AND age > 30;`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "CASE" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "WHEN" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "id" }, - { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "NUMBER", value: "1" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "THEN" }, + { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "STRING", value: "'one'" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "ELSE" }, + { type: "IDENTIFIER", value: "name" }, { type: "WHITESPACE", value: " " }, - { type: "STRING", value: "'other'" }, + { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "END" }, + { type: "STRING", value: "'John Doe'" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "AS" }, + { type: "IDENTIFIER", value: "AND" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "result" }, + { type: "IDENTIFIER", value: "age" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "OPERATOR", value: ">" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "NUMBER", value: "30" }, + { type: "PUNCTUATION", value: ";" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); - -test("Select with UNION", () => { - const sql = "SELECT name FROM customers UNION SELECT name FROM orders"; - const tokens = tokenizeSql(sql); +test("Keyword WITH and subquery", () => { + const sql = `WITH subquery AS (SELECT * FROM customers) SELECT * FROM subquery`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "WITH" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "IDENTIFIER", value: "subquery" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "AS" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "PUNCTUATION", value: "(" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "UNION" }, + { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "IDENTIFIER", value: "customers" }, + { type: "PUNCTUATION", value: ")" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "orders" }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "subquery" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("Select with multiple placehoder", () => { - const sql = "SELECT * FROM customers WHERE id = :id AND name = :name"; - const tokens = tokenizeSql(sql); +test("IN clause", () => { + const sql = `SELECT * FROM customers WHERE age IN (25, 30, 35)`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "*" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "WHERE" }, + { type: "IDENTIFIER", value: "WHERE" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "id" }, - { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "IDENTIFIER", value: "age" }, { type: "WHITESPACE", value: " " }, - { type: "PLACEHOLDER", value: ":id" }, + { type: "IDENTIFIER", value: "IN" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "AND" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "PUNCTUATION", value: "(" }, + { type: "NUMBER", value: "25" }, + { type: "PUNCTUATION", value: "," }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "NUMBER", value: "30" }, + { type: "PUNCTUATION", value: "," }, { type: "WHITESPACE", value: " " }, - { type: "PLACEHOLDER", value: ":name" }, + { type: "NUMBER", value: "35" }, + { type: "PUNCTUATION", value: ")" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); -test("try to tokenize invalid sql", () => { - const sql = "SELECT * FROM customers WHERE id = :id AND name = :123name"; - const tokens = tokenizeSql(sql); +test("INNER JOIN", () => { + const sql = `SELECT customers.name, orders.id FROM customers INNER JOIN orders ON customers.id = orders.customer_id`; + const tokens = tokenizeSql(sql, "sqlite"); expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, + { type: "IDENTIFIER", value: "SELECT" }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "*" }, + { type: "IDENTIFIER", value: "customers" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "name" }, + { type: "PUNCTUATION", value: "," }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, + { type: "IDENTIFIER", value: "orders" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "id" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, + { type: "IDENTIFIER", value: "FROM" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "WHERE" }, + { type: "IDENTIFIER", value: "customers" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "id" }, + { type: "IDENTIFIER", value: "INNER" }, { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "=" }, + { type: "IDENTIFIER", value: "JOIN" }, { type: "WHITESPACE", value: " " }, - { type: "PLACEHOLDER", value: ":id" }, + { type: "IDENTIFIER", value: "orders" }, { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "AND" }, + { type: "IDENTIFIER", value: "ON" }, { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "name" }, + { type: "IDENTIFIER", value: "customers" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "id" }, { type: "WHITESPACE", value: " " }, { type: "OPERATOR", value: "=" }, { type: "WHITESPACE", value: " " }, - { type: "UNKNOWN", value: ":123name" }, + { type: "IDENTIFIER", value: "orders" }, + { type: "PUNCTUATION", value: "." }, + { type: "IDENTIFIER", value: "customer_id" }, ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); - -test("try extra whitespace stay next to each other", () => { - const sql = "SELECT * FROM customers"; - const tokens = tokenizeSql(sql); - expect(tokens.map((t) => t.value).join("")).toBe(sql); - expect(tokens).toEqual([ - { type: "KEYWORD", value: "SELECT" }, - { type: "WHITESPACE", value: " " }, - { type: "OPERATOR", value: "*" }, - { type: "WHITESPACE", value: " " }, - { type: "KEYWORD", value: "FROM" }, - { type: "WHITESPACE", value: " " }, - { type: "IDENTIFIER", value: "customers" }, - ]); -}); diff --git a/src/lib/sql/tokenizer.ts b/src/lib/sql/tokenizer.ts index e0f0cb5c..300ec94d 100644 --- a/src/lib/sql/tokenizer.ts +++ b/src/lib/sql/tokenizer.ts @@ -1,25 +1,85 @@ +import { SupportedDialect } from "@/drivers/base-driver"; + interface Token { type: string; value: string; } -const tokenTypes: { type: string; regex: RegExp }[] = [ - { type: "WHITESPACE", regex: /^\s+/ }, - { - type: "KEYWORD", - regex: - /^(SELECT|FROM|WHERE|INSERT|INTO|VALUES|UPDATE|SET|DELETE|INNER|OUTER|JOIN|ON|AND|OR|NOT|LIKE|GROUP|BY|HAVING|ORDER|LIMIT|OFFSET|DISTINCT|AS|CASE|WHEN|THEN|ELSE|END|UNION|ALL|ANY|EXISTS|IN|IS|NULL|BETWEEN|WITH|DESC|ASC|COUNT)\b(?!\.)/i, - }, - { type: "IDENTIFIER", regex: /^(`?[a-zA-Z_][a-zA-Z0-9_.]*`?)/ }, - { type: "STRING", regex: /^(?:'(?:[^']|'')*'|"(?:[^"]|"")*")/ }, - { type: "NUMBER", regex: /^\d+(\.\d+)?/ }, - { type: "PLACEHOLDER", regex: /^:[a-zA-Z_][a-zA-Z0-9_]*/ }, - { type: "COMMENT", regex: /^(--[^\n]*|\/\*[\s\S]*?\*\/)/ }, - { type: "OPERATOR", regex: /^(=|<>|!=|<|>|<=|>=|\+|-|\*|\/)/ }, - { type: "PUNCTUATION", regex: /^[`,;()]/ }, +const tokenTypes: { + type: string; + findToken: (input: string, dialect?: SupportedDialect) => string | null; +}[] = [ + { + type: "WHITESPACE", + findToken: (input) => { + const regex = /^\s+/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "IDENTIFIER", + findToken: (input) => { + const regex = + /^(`[^(`\n)]+`|"[^("\n)]+"|\[[^(\]\n)]+\]|[a-zA-Z_][a-zA-Z0-9_]*)/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "STRING", + findToken: (input) => { + const regex = /^(?:'(?:[^('\n)]|'')*'|"(?:[^("\n)]|"")*")/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + + { + type: "NUMBER", + findToken: (input) => { + const regex = /^\d+(\.\d+)?/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "PLACEHOLDER", + findToken: (input) => { + const regex = /^:[a-zA-Z_][a-zA-Z0-9_]*/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "COMMENT", + findToken: (input, dialect) => { + let regex = /^(--.*|\/\*[\s\S]*?\*\/)/; // -- comment, /* comment */ + if (dialect === "mysql") regex = /^(--.*|#.*|\/\*[\s\S]*?\*\/)/; // for mysql, # is also used for comments + + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "OPERATOR", + findToken: (input) => { + const regex = /^(=|<>|!=|<|>|<=|>=|\+|-|\*|\/)/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, + { + type: "PUNCTUATION", + findToken: (input) => { + const regex = /^[`,;().]/; + const match = regex.exec(input); + return match?.[0] ?? null; + }, + }, ]; -export function tokenizeSql(sql: string): Token[] { +export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { try { const tokens: Token[] = []; let cursor = 0; @@ -27,20 +87,20 @@ export function tokenizeSql(sql: string): Token[] { while (cursor < length) { let matched = false; - let subStr = ""; - for (const { type, regex } of tokenTypes) { - subStr = sql.substring(cursor); - const match = regex.exec(subStr); + const subStr = sql.substring(cursor); + for (const { type, findToken } of tokenTypes) { + const match = findToken(subStr, dialect); if (match) { - tokens.push({ type, value: match[0] }); - cursor += match[0].length; + tokens.push({ type, value: match }); + cursor += match.length; matched = true; break; } } if (!matched) { - return [...tokens, { type: "UNKNOWN", value: subStr }]; + tokens.push({ type: "UNKNOWN", value: subStr[0] }); + cursor++; } } return tokens; From 8ddb527467c204e4750ec6821591b1c6731daaa6 Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Fri, 17 Jan 2025 22:33:01 +0700 Subject: [PATCH 6/8] refactor: improve query placeholder --- src/components/gui/tabs/query-tab.tsx | 72 +++++++++++++++------------ src/lib/sql/tokenizer.test.ts | 37 ++++++++++++++ src/lib/sql/tokenizer.ts | 9 +++- 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index e6ae6c30..583efa61 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -87,31 +87,30 @@ export default function QueryWindow({ initialNamespace ?? "Unsaved Query" ); const [savedKey, setSavedKey] = useState(initialSavedKey); - const [placeHolders, setPlaceHolders] = useState>({}); + const [placeholders, setPlaceholders] = useState>({}); useEffect(() => { const timer = setTimeout(() => { - const editorState = editorRef.current?.view?.state; - if (!editorState) return; - const finalStatements = splitSqlQuery(editorState).map((q) => q.text); - const newPlaceholders: Record = {}; - for (const statement of finalStatements) { - const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); - const placeholders = token + setPlaceholders((prev) => { + const newPlaceholders: Record = {}; + const token = tokenizeSql(code, databaseDriver.getFlags().dialect); + const foundPlaceholders = token .filter((t) => t.type === "PLACEHOLDER") - .map((t) => t.value.split(":")[1]); - for (const placeholder of placeholders) { - newPlaceholders[placeholder] = ""; + .map((t) => t.value.slice(1)); + + for (const foundPlaceholder of foundPlaceholders) { + newPlaceholders[foundPlaceholder] = ""; } - } - for (const newKey of Object.keys(newPlaceholders)) { - newPlaceholders[newKey] = placeHolders[newKey] ?? ""; - } - setPlaceHolders(newPlaceholders); + // write old placeholders value into new placeholders + for (const newKey of Object.keys(newPlaceholders)) { + newPlaceholders[newKey] = prev[newKey] ?? ""; + } + + return { ...newPlaceholders }; + }); }, 1000); return () => clearTimeout(timer); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [code]); + }, [code, databaseDriver]); const onFormatClicked = () => { try { @@ -166,26 +165,33 @@ export default function QueryWindow({ setQueryTabIndex(0); //inject placeholders - for (const statement of finalStatements) { - const token = tokenizeSql(statement, databaseDriver.getFlags().dialect); + for (let i = 0; i < finalStatements.length; i++) { + const token = tokenizeSql( + finalStatements[i], + databaseDriver.getFlags().dialect + ); + const variables = token .filter((t) => t.type === "PLACEHOLDER") - .map((t) => t.value.split(":")[1]); + .map((t) => t.value.slice(1)); if ( variables.length > 0 && - variables.some((p) => placeHolders[p] === "") + variables.some((p) => placeholders[p] === "") ) { toast.error("Please fill in all placeholders"); return; } - } - for (const key of Object.keys(placeHolders)) { - finalStatements = finalStatements.map((s) => - s.replace( - new RegExp(`:${key}`, "g"), - escapeSqlValue(extractInputValue(placeHolders[key])) - ) - ); + + finalStatements[i] = token + .map((t) => { + if (t.type === "PLACEHOLDER") { + return escapeSqlValue( + extractInputValue(placeholders[t.value.slice(1)]) + ); + } + return t.value; + }) + .join(""); } multipleQuery(databaseDriver, finalStatements, (currentProgress) => { @@ -400,10 +406,10 @@ export default function QueryWindow({
Col {columnNumber + 1}
- {Object.keys(placeHolders).length > 0 && ( + {Object.keys(placeholders).length > 0 && ( )}
diff --git a/src/lib/sql/tokenizer.test.ts b/src/lib/sql/tokenizer.test.ts index 71cd04e6..01adc37a 100644 --- a/src/lib/sql/tokenizer.test.ts +++ b/src/lib/sql/tokenizer.test.ts @@ -406,3 +406,40 @@ test("INNER JOIN", () => { ]); expect(tokens.map((t) => t.value).join("")).toBe(sql); }); + +test("Accumulate unknown tokens", () => { + const sql = `SELECT * FROM customers WHERE name = 'John Doe' ### this # is a comment ##`; + const tokens = tokenizeSql(sql, "sqlite"); + expect(tokens).toEqual([ + { type: "IDENTIFIER", value: "SELECT" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "*" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "FROM" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "customers" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "WHERE" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "name" }, + { type: "WHITESPACE", value: " " }, + { type: "OPERATOR", value: "=" }, + { type: "WHITESPACE", value: " " }, + { type: "STRING", value: "'John Doe'" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "###" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "this" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "#" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "is" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "a" }, + { type: "WHITESPACE", value: " " }, + { type: "IDENTIFIER", value: "comment" }, + { type: "WHITESPACE", value: " " }, + { type: "UNKNOWN", value: "##" }, + ]); + expect(tokens.map((t) => t.value).join("")).toBe(sql); +}); diff --git a/src/lib/sql/tokenizer.ts b/src/lib/sql/tokenizer.ts index 300ec94d..5d82473c 100644 --- a/src/lib/sql/tokenizer.ts +++ b/src/lib/sql/tokenizer.ts @@ -84,6 +84,7 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { const tokens: Token[] = []; let cursor = 0; const length = sql.length; + let unknownAcc = ""; while (cursor < length) { let matched = false; @@ -91,6 +92,11 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { for (const { type, findToken } of tokenTypes) { const match = findToken(subStr, dialect); if (match) { + if (unknownAcc !== "") { + tokens.push({ type: "UNKNOWN", value: unknownAcc }); + unknownAcc = ""; + } + tokens.push({ type, value: match }); cursor += match.length; matched = true; @@ -99,10 +105,11 @@ export function tokenizeSql(sql: string, dialect: SupportedDialect): Token[] { } if (!matched) { - tokens.push({ type: "UNKNOWN", value: subStr[0] }); + unknownAcc += subStr[0]; cursor++; } } + if (unknownAcc !== "") tokens.push({ type: "UNKNOWN", value: unknownAcc }); return tokens; } catch (e) { return [{ type: "SQL", value: sql }]; From bef492f40535b13c08ff69719096ac193d6fef9e Mon Sep 17 00:00:00 2001 From: keppere <2keppere@gmail.com> Date: Fri, 17 Jan 2025 22:45:22 +0700 Subject: [PATCH 7/8] fix: remove acceipt completion by space --- src/components/gui/sql-editor/index.tsx | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/components/gui/sql-editor/index.tsx b/src/components/gui/sql-editor/index.tsx index 3cfd75df..936e27ae 100644 --- a/src/components/gui/sql-editor/index.tsx +++ b/src/components/gui/sql-editor/index.tsx @@ -83,24 +83,6 @@ const SqlEditor = forwardRef( return true; }, }, - { - key: "Space", - preventDefault: true, - run: (target) => { - if (completionStatus(target.state) === "active") { - acceptCompletion(target); - } else { - target.dispatch({ - changes: { - from: target.state.selection.main.from, - insert: " ", - }, - selection: { anchor: target.state.selection.main.anchor + 1 }, - }); - } - return true; - }, - }, { key: "Ctrl-Space", mac: "Cmd-i", From 3059a6ce19a59753deba49c87c0a266cb74ef838 Mon Sep 17 00:00:00 2001 From: "Visal .In" Date: Mon, 20 Jan 2025 08:02:10 +0700 Subject: [PATCH 8/8] redesign name placeholder --- src/components/gui/tabs/query-placeholder.tsx | 73 ++++++++++++------- src/components/gui/tabs/query-tab.tsx | 30 +++++--- 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/src/components/gui/tabs/query-placeholder.tsx b/src/components/gui/tabs/query-placeholder.tsx index e49819c9..287182ed 100644 --- a/src/components/gui/tabs/query-placeholder.tsx +++ b/src/components/gui/tabs/query-placeholder.tsx @@ -1,38 +1,50 @@ import { buttonVariants } from "@/components/ui/button"; -import { LucideChevronDown } from "lucide-react"; +import { + Popover, + PopoverTrigger, + PopoverContent, +} from "@/components/ui/popover"; +import { useMemo } from "react"; + interface Props { - placeHolders: Record; + placeholders: Record; onChange: (placeHolders: Record) => void; } -import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; export function QueryPlaceholder({ - placeHolders, + placeholders, onChange, }: Props): JSX.Element { - const placeHolderTable = () => { + const placeholderCount = Object.keys(placeholders).length; + const emptyPlaceholderCount = Object.values(placeholders).filter( + (v) => v === "" + ).length; + const hasEmptyPlaceholder = emptyPlaceholderCount > 0; + + const placeholderTable = useMemo(() => { return ( -
+
- - + + - {Object.entries(placeHolders).map(([key, value]) => ( + {Object.entries(placeholders).map(([key, value]) => ( - - +
VariableValueVariablesValue
{key} + {key} { const newValue = e.currentTarget.value; const newPlaceHolders = { - ...placeHolders, + ...placeholders, [key]: newValue, }; onChange(newPlaceHolders); @@ -45,27 +57,32 @@ export function QueryPlaceholder({
); - }; + }, [placeholders, onChange]); + return ( - - + +
- {hasPlaceHolderWithEmptyValue(placeHolders) && ( + {hasEmptyPlaceholder && (
-
+
)} - Placeholders{" "} - {!!placeHolders && } + Placeholders + + {placeholderCount - emptyPlaceholderCount} / {placeholderCount} +
-
- {placeHolderTable()} -
- ); -} + + + {placeholderTable} -export function hasPlaceHolderWithEmptyValue( - placeHolders: Record -) { - return Object.values(placeHolders).some((value) => value === ""); +

+ Use '' for an + empty string. If the value is a number, it will automatically be cast + to a number. To specify a numeric string, wrap it in single quote. +

+
+ + ); } diff --git a/src/components/gui/tabs/query-tab.tsx b/src/components/gui/tabs/query-tab.tsx index 583efa61..28ce784e 100644 --- a/src/components/gui/tabs/query-tab.tsx +++ b/src/components/gui/tabs/query-tab.tsx @@ -54,6 +54,7 @@ import ExplainResultTab from "../tabs-result/explain-result-tab"; import { tokenizeSql } from "@/lib/sql/tokenizer"; import { QueryPlaceholder } from "./query-placeholder"; import { escapeSqlValue, extractInputValue } from "@/drivers/sqlite/sql-helper"; +import { sendAnalyticEvents } from "@/lib/tracking"; interface QueryWindowProps { initialCode?: string; @@ -94,21 +95,19 @@ export default function QueryWindow({ setPlaceholders((prev) => { const newPlaceholders: Record = {}; const token = tokenizeSql(code, databaseDriver.getFlags().dialect); + const foundPlaceholders = token .filter((t) => t.type === "PLACEHOLDER") .map((t) => t.value.slice(1)); for (const foundPlaceholder of foundPlaceholders) { - newPlaceholders[foundPlaceholder] = ""; - } - // write old placeholders value into new placeholders - for (const newKey of Object.keys(newPlaceholders)) { - newPlaceholders[newKey] = prev[newKey] ?? ""; + newPlaceholders[foundPlaceholder] = prev[foundPlaceholder] ?? ""; } - return { ...newPlaceholders }; + return newPlaceholders; }); }, 1000); + return () => clearTimeout(timer); }, [code, databaseDriver]); @@ -164,16 +163,27 @@ export default function QueryWindow({ setProgress(undefined); setQueryTabIndex(0); - //inject placeholders for (let i = 0; i < finalStatements.length; i++) { const token = tokenizeSql( finalStatements[i], databaseDriver.getFlags().dialect ); + // Defensive measurement + if (token.join("") === finalStatements[i]) { + sendAnalyticEvents([ + { name: "tokenize_mismatch", data: { token, finalStatements } }, + ]); + + toast.error("Failed to tokenize SQL statement"); + + return; + } + const variables = token .filter((t) => t.type === "PLACEHOLDER") .map((t) => t.value.slice(1)); + if ( variables.length > 0 && variables.some((p) => placeholders[p] === "") @@ -374,7 +384,7 @@ export default function QueryWindow({
-
+
-
+
Ln {lineNumber}
Col {columnNumber + 1}
@@ -408,7 +418,7 @@ export default function QueryWindow({
{Object.keys(placeholders).length > 0 && ( )}