From c7b553ec30bcbfc323b62ae966dfeccdbc4d8e9c Mon Sep 17 00:00:00 2001 From: "remalloc.virtual@gmail.com" Date: Mon, 9 Aug 2021 15:44:56 +0800 Subject: [PATCH 1/5] Check DB weather or not support JSON --- dataset/database.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dataset/database.py b/dataset/database.py index b3ecfc2..629b2cd 100644 --- a/dataset/database.py +++ b/dataset/database.py @@ -56,6 +56,8 @@ def __init__( self.engine = create_engine(url, **engine_kwargs) self.is_postgres = self.engine.dialect.name == "postgresql" self.is_sqlite = self.engine.dialect.name == "sqlite" + self._server_version_info = None + self._is_support_json = None def _enable_sqlite_wal_mode(dbapi_con, con_record): # reference: @@ -105,6 +107,25 @@ def in_transaction(self): return False return len(self.local.tx) > 0 + @property + def server_version_info(self): + if self._server_version_info is None: + tables = self.tables # connect DB + self._server_version_info = self.engine.dialect.server_version_info + return self._server_version_info + + @property + def is_support_json(self): + if self._is_support_json is None: + support_versions = {"mysql": 5.7, "postgresql": 9.2, "sqlite": 3.9, "mssql": 13.0, "oracle": 12.1} + db_name = self.engine.dialect.name + version = float(str(self.server_version_info[0]) + "." + str(self.server_version_info[1])) + if support_versions.get(db_name) and version >= support_versions[db_name]: + self._is_support_json = True + else: + self._is_support_json = False + return self._is_support_json + def _flush_tables(self): """Clear the table metadata after transaction rollbacks.""" for table in self._tables.values(): From 80a7ba73e1407495e8e8555930657a553479ee67 Mon Sep 17 00:00:00 2001 From: "remalloc.virtual@gmail.com" Date: Mon, 9 Aug 2021 15:48:58 +0800 Subject: [PATCH 2/5] Add doc string --- dataset/database.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataset/database.py b/dataset/database.py index 629b2cd..4ad64ed 100644 --- a/dataset/database.py +++ b/dataset/database.py @@ -109,6 +109,7 @@ def in_transaction(self): @property def server_version_info(self): + """Return database version number""" if self._server_version_info is None: tables = self.tables # connect DB self._server_version_info = self.engine.dialect.server_version_info @@ -116,6 +117,7 @@ def server_version_info(self): @property def is_support_json(self): + """Check if this database version support JSON column""" if self._is_support_json is None: support_versions = {"mysql": 5.7, "postgresql": 9.2, "sqlite": 3.9, "mssql": 13.0, "oracle": 12.1} db_name = self.engine.dialect.name From 4cb82b944e6d56f5eb3811f93bf75a4be359c264 Mon Sep 17 00:00:00 2001 From: "remalloc.virtual@gmail.com" Date: Mon, 9 Aug 2021 16:57:14 +0800 Subject: [PATCH 3/5] Add find json function --- dataset/table.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/dataset/table.py b/dataset/table.py index 5a3d81c..bfdc763 100644 --- a/dataset/table.py +++ b/dataset/table.py @@ -2,6 +2,7 @@ import warnings import threading from banal import ensure_list +from decimal import Decimal from sqlalchemy import func, select, false from sqlalchemy.sql import and_, expression @@ -418,7 +419,37 @@ def _generate_clause(self, column, op, value): return self.table.c[column].like(value + "%") return false() - def _args_to_clause(self, args, clauses=()): + def _generate_json_clause(self, column, key, ops_value): + element_type = { + int: "as_integer", + float: "as_float", + bool: "as_boolean", + Decimal: "as_numeric", + str: "as_string", + list: "as_json", + dict: "as_json" + } + for op, value in ops_value.items(): + col = getattr(self.table.c[column][key], element_type[type(value)])() + if op in (">", "gt"): + yield col > value + elif op in ("<", "lt"): + yield col < value + elif op in (">=", "gte"): + yield col >= value + elif op in ("<=", "lte"): + yield col <= value + elif op in ("=", "==", "is"): + yield col == value + elif op in ("!=", "<>", "not"): + yield col != value + elif op in ("between", ".."): + start, end = value + yield self._generate_json_clause(column, key, {">=": start, "<": end}) + else: + yield false() + + def _args_to_clause(self, args, clauses=(), json_=None): clauses = list(clauses) for column, value in args.items(): column = self._get_column_name(column) @@ -431,6 +462,13 @@ def _args_to_clause(self, args, clauses=()): clauses.append(self._generate_clause(column, op, op_value)) else: clauses.append(self._generate_clause(column, "=", value)) + if json_: + if not self.db.is_support_json: + raise NotImplementedError("Current database not support json column!") + for column, value in json_.items(): + column = self._get_column_name(column) + for key, ops_value in value.items(): + clauses.extend(self._generate_json_clause(column, key, ops_value)) return and_(*clauses) def _args_to_order_by(self, order_by): @@ -597,6 +635,18 @@ def find(self, *_clauses, **kwargs): # return all rows sorted by multiple columns (descending by year) results = table.find(order_by=['country', '-year']) + Using ``_json``:: + + # Notice: selected key type depends on giving value type, + # like if given integer but stored type is float will be automatically transformed to integer. + # select json_column to find rows that keys are between 0 and 1 + results = table.find(_json={'json_column':{'key':{'>=': 0.0, '<':1.0}}}) # keys are float + results = table.find(_json={'json_column':{'key':{'>=': 0, '<':1}}}) # keys are integer + # find rows by index + results = table.find(_json={'json_column':{3:{'>=': 0, '<':1}}}) + # find rows by path + results = table.find(_json={'json_column':{('key1','key2'):{'>=': 0, '<':1}}}) + You can also submit filters based on criteria other than equality, see :ref:`advanced_filters` for details. @@ -612,11 +662,12 @@ def find(self, *_clauses, **kwargs): order_by = kwargs.pop("order_by", None) _streamed = kwargs.pop("_streamed", False) _step = kwargs.pop("_step", QUERY_STEP) + _json = kwargs.pop("_json", None) if _step is False or _step == 0: _step = None order_by = self._args_to_order_by(order_by) - args = self._args_to_clause(kwargs, clauses=_clauses) + args = self._args_to_clause(kwargs, clauses=_clauses, json_=_json) query = self.table.select(whereclause=args, limit=_limit, offset=_offset) if len(order_by): query = query.order_by(*order_by) From 202700c9f8e83a885ff593a022eaf2824c8f0baa Mon Sep 17 00:00:00 2001 From: "remalloc.virtual@gmail.com" Date: Mon, 9 Aug 2021 17:26:33 +0800 Subject: [PATCH 4/5] Add more examples about finding json --- dataset/table.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/dataset/table.py b/dataset/table.py index bfdc763..e36a6f2 100644 --- a/dataset/table.py +++ b/dataset/table.py @@ -639,13 +639,25 @@ def find(self, *_clauses, **kwargs): # Notice: selected key type depends on giving value type, # like if given integer but stored type is float will be automatically transformed to integer. - # select json_column to find rows that keys are between 0 and 1 - results = table.find(_json={'json_column':{'key':{'>=': 0.0, '<':1.0}}}) # keys are float - results = table.find(_json={'json_column':{'key':{'>=': 0, '<':1}}}) # keys are integer + # id json_column + # 0 {"key":-0.5} + # 1 {"key":0.5} + # 2 {"key":1.5} + results = table.find(_json={'json_column':{'key':{'>=': 0.0, '<':1.0}}}) # id = [1] + results = table.find(_json={'json_column':{'key':{'>=': 0, '<':1}}}) # int(-0.5)==0, id = [0,1] + + # id json_column + # 0 [0,1,2] + # 1 [0,0.5,1] + # 2 [0] # find rows by index - results = table.find(_json={'json_column':{3:{'>=': 0, '<':1}}}) + results = table.find(_json={'json_column':{1:{'>=': 0.0, '<':1.0}}}) + + # id json_column + # 0 {"key1":{"key2":-1}} + # 1 {"key1":{"key2":0.5}} # find rows by path - results = table.find(_json={'json_column':{('key1','key2'):{'>=': 0, '<':1}}}) + results = table.find(_json={'json_column':{('key1','key2'):{'>=': 0.0, '<':1.0}}}) You can also submit filters based on criteria other than equality, see :ref:`advanced_filters` for details. From c6c6c49bfbe17c258fc5557ee5c78e164011fa64 Mon Sep 17 00:00:00 2001 From: "remalloc.virtual@gmail.com" Date: Mon, 9 Aug 2021 18:08:20 +0800 Subject: [PATCH 5/5] Add doc about finding json --- dataset/table.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dataset/table.py b/dataset/table.py index e36a6f2..934e1d0 100644 --- a/dataset/table.py +++ b/dataset/table.py @@ -639,6 +639,7 @@ def find(self, *_clauses, **kwargs): # Notice: selected key type depends on giving value type, # like if given integer but stored type is float will be automatically transformed to integer. + # Support operations: >(gt), <(lt), >=(gte), <=(lte), =(==,is), !=(<>, not), between("..") # id json_column # 0 {"key":-0.5} # 1 {"key":0.5} @@ -651,13 +652,13 @@ def find(self, *_clauses, **kwargs): # 1 [0,0.5,1] # 2 [0] # find rows by index - results = table.find(_json={'json_column':{1:{'>=': 0.0, '<':1.0}}}) + results = table.find(_json={'json_column':{1:{'>=': 0.0, '<':1.0}}}) # id = [1] # id json_column # 0 {"key1":{"key2":-1}} # 1 {"key1":{"key2":0.5}} # find rows by path - results = table.find(_json={'json_column':{('key1','key2'):{'>=': 0.0, '<':1.0}}}) + results = table.find(_json={'json_column':{('key1','key2'):{'between':[0.0,1.0]}}}) # id = [1] You can also submit filters based on criteria other than equality, see :ref:`advanced_filters` for details.